mirror/stackexchange-dnscontrol
1
0
Fork 0
mirror of https://github.com/StackExchange/dnscontrol.git synced 2024-05-11 05:55:12 +00:00
Code Issues Releases Activity

Refactoring diff package interface (#22)

Browse Source 
* initial refactoring of diffing

* making cloudflare and others compile

* gandi and gcloud. no idea if gandi works anymore.

* r53

* namedotcom wasn't working.
This commit is contained in:
Craig Peterson
2017-01-11 12:38:07 -07:00
committed by GitHub
parent 1f8b0a11e0
commit 12f006441b
15 changed files with 322 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
providers/diff/diff.go
View File

@@ -3,69 +3,83 @@ package diff
import (
"fmt"
"sort"
"github.com/StackExchange/dnscontrol/models"
)
type Record interface {
GetName() string
GetType() string
GetContent() string
// Get relevant comparision data. Default implentation uses "ttl [mx priority]", but providers may insert
// provider specific metadata if needed.
GetComparisionData() string
}
type Correlation struct {
Existing Record
Desired Record
d *differ
Existing *models.RecordConfig
Desired *models.RecordConfig
}
type Changeset []Correlation
func IncrementalDiff(existing []Record, desired []Record) (unchanged, create, toDelete, modify Changeset) {
type Differ interface {
IncrementalDiff(existing []*models.RecordConfig) (unchanged, create, toDelete, modify Changeset)
}
func New(dc *models.DomainConfig, extraValues ...func(*models.RecordConfig) map[string]string) Differ {
return &differ{
dc: dc,
extraValues: extraValues,
}
}
type differ struct {
dc *models.DomainConfig
extraValues []func(*models.RecordConfig) map[string]string
}
// get normalized content for record. target, ttl, mxprio, and specified metadata
func (d *differ) content(r *models.RecordConfig) string {
content := fmt.Sprintf("%s %d", r.Target, r.TTL)
if r.Type == "MX" {
content += fmt.Sprintf(" priority=%d", r.Priority)
}
for _, f := range d.extraValues {
for k, v := range f(r) {
content += fmt.Sprintf(" %s=%s", k, v)
}
}
return content
}
func (d *differ) IncrementalDiff(existing []*models.RecordConfig) (unchanged, create, toDelete, modify Changeset) {
unchanged = Changeset{}
create = Changeset{}
toDelete = Changeset{}
modify = Changeset{}
// log.Printf("ID existing records: (%d)\n", len(existing))
// for i, d := range existing {
// log.Printf("\t%d\t%v\n", i, d)
// }
// log.Printf("ID desired records: (%d)\n", len(desired))
// for i, d := range desired {
// log.Printf("\t%d\t%v\n", i, d)
// }
desired := d.dc.Records
//sort existing and desired by name
type key struct {
name, rType string
}
existingByNameAndType := map[key][]Record{}
desiredByNameAndType := map[key][]Record{}
existingByNameAndType := map[key][]*models.RecordConfig{}
desiredByNameAndType := map[key][]*models.RecordConfig{}
for _, e := range existing {
k := key{e.GetName(), e.GetType()}
k := key{e.NameFQDN, e.Type}
existingByNameAndType[k] = append(existingByNameAndType[k], e)
}
for _, d := range desired {
k := key{d.GetName(), d.GetType()}
k := key{d.NameFQDN, d.Type}
desiredByNameAndType[k] = append(desiredByNameAndType[k], d)
}
// Look through existing records. This will give us changes and deletions and some additions
// Look through existing records. This will give us changes and deletions and some additions.
// Each iteration is only for a single type/name record set
for key, existingRecords := range existingByNameAndType {
desiredRecords := desiredByNameAndType[key]
//first look through records that are the same content on both sides. Those are either modifications or unchanged
//first look through records that are the same target on both sides. Those are either modifications or unchanged
for i := len(existingRecords) - 1; i >= 0; i-- {
ex := existingRecords[i]
for j, de := range desiredRecords {
if de.GetContent() == ex.GetContent() {
//they're either identical or should be a modification of each other
if de.GetComparisionData() == ex.GetComparisionData() {
unchanged = append(unchanged, Correlation{ex, de})
if de.Target == ex.Target {
//they're either identical or should be a modification of each other (ttl or metadata changes)
if d.content(de) == d.content(ex) {
unchanged = append(unchanged, Correlation{d, ex, de})
} else {
modify = append(modify, Correlation{ex, de})
modify = append(modify, Correlation{d, ex, de})
}
// remove from both slices by index
existingRecords = existingRecords[:i+copy(existingRecords[i:], existingRecords[i+1:])]
@@ -75,18 +89,18 @@ func IncrementalDiff(existing []Record, desired []Record) (unchanged, create, to
}
}
desiredLookup := map[string]Record{}
existingLookup := map[string]Record{}
// build index based on normalized value/ttl
desiredLookup := map[string]*models.RecordConfig{}
existingLookup := map[string]*models.RecordConfig{}
// build index based on normalized content data
for _, ex := range existingRecords {
normalized := fmt.Sprintf("%s %s", ex.GetContent(), ex.GetComparisionData())
normalized := d.content(ex)
if existingLookup[normalized] != nil {
panic(fmt.Sprintf("DUPLICATE E_RECORD FOUND: %s %s", key, normalized))
}
existingLookup[normalized] = ex
}
for _, de := range desiredRecords {
normalized := fmt.Sprintf("%s %s", de.GetContent(), de.GetComparisionData())
normalized := d.content(de)
if desiredLookup[normalized] != nil {
panic(fmt.Sprintf("DUPLICATE D_RECORD FOUND: %s %s", key, normalized))
}
@@ -95,36 +109,28 @@ func IncrementalDiff(existing []Record, desired []Record) (unchanged, create, to
// if a record is in both, it is unchanged
for norm, ex := range existingLookup {
if de, ok := desiredLookup[norm]; ok {
unchanged = append(unchanged, Correlation{ex, de})
unchanged = append(unchanged, Correlation{d, ex, de})
delete(existingLookup, norm)
delete(desiredLookup, norm)
}
}
//sort records by normalized text. Keeps behaviour deterministic
existingStrings, desiredStrings := []string{}, []string{}
for norm := range existingLookup {
existingStrings = append(existingStrings, norm)
}
for norm := range desiredLookup {
desiredStrings = append(desiredStrings, norm)
}
sort.Strings(existingStrings)
sort.Strings(desiredStrings)
existingStrings, desiredStrings := sortedKeys(existingLookup), sortedKeys(desiredLookup)
// Modifications. Take 1 from each side.
for len(desiredStrings) > 0 && len(existingStrings) > 0 {
modify = append(modify, Correlation{existingLookup[existingStrings[0]], desiredLookup[desiredStrings[0]]})
modify = append(modify, Correlation{d, existingLookup[existingStrings[0]], desiredLookup[desiredStrings[0]]})
existingStrings = existingStrings[1:]
desiredStrings = desiredStrings[1:]
}
// If desired still has things they are additions
for _, norm := range desiredStrings {
rec := desiredLookup[norm]
create = append(create, Correlation{nil, rec})
create = append(create, Correlation{d, nil, rec})
}
// if found , but not desired, delete it
for _, norm := range existingStrings {
rec := existingLookup[norm]
toDelete = append(toDelete, Correlation{rec, nil})
toDelete = append(toDelete, Correlation{d, rec, nil})
}
// remove this set from the desired list to indicate we have processed it.
delete(desiredByNameAndType, key)
@@ -136,7 +142,7 @@ func IncrementalDiff(existing []Record, desired []Record) (unchanged, create, to
}
for _, desiredList := range desiredByNameAndType {
for _, rec := range desiredList {
create = append(create, Correlation{nil, rec})
create = append(create, Correlation{d, nil, rec})
}
}
return
@@ -144,10 +150,19 @@ func IncrementalDiff(existing []Record, desired []Record) (unchanged, create, to
func (c Correlation) String() string {
if c.Existing == nil {
return fmt.Sprintf("CREATE %s %s %s %s", c.Desired.GetType(), c.Desired.GetName(), c.Desired.GetContent(), c.Desired.GetComparisionData())
return fmt.Sprintf("CREATE %s %s %s", c.Desired.Type, c.Desired.NameFQDN, c.d.content(c.Desired))
}
if c.Desired == nil {
return fmt.Sprintf("DELETE %s %s %s %s", c.Existing.GetType(), c.Existing.GetName(), c.Existing.GetContent(), c.Existing.GetComparisionData())
return fmt.Sprintf("DELETE %s %s %s", c.Existing.Type, c.Existing.NameFQDN, c.d.content(c.Existing))
}
return fmt.Sprintf("MODIFY %s %s: (%s %s) -> (%s %s)", c.Existing.GetType(), c.Existing.GetName(), c.Existing.GetContent(), c.Existing.GetComparisionData(), c.Desired.GetContent(), c.Desired.GetComparisionData())
return fmt.Sprintf("MODIFY %s %s: (%s) -> (%s)", c.Existing.Type, c.Existing.NameFQDN, c.d.content(c.Existing), c.d.content(c.Desired))
}
func sortedKeys(m map[string]*models.RecordConfig) []string {
s := []string{}
for v := range m {
s = append(s, v)
}
sort.Strings(s)
return s
}

103
providers/diff/diff_test.go
View File

@@ -1,58 +1,52 @@
package diff
import (
"fmt"
"strconv"
"strings"
"testing"
"github.com/StackExchange/dnscontrol/models"
"github.com/miekg/dns/dnsutil"
)
type myRecord string //@ A 1 1.2.3.4
func (m myRecord) GetName() string {
name := strings.SplitN(string(m), " ", 4)[0]
return dnsutil.AddOrigin(name, "example.com")
}
func (m myRecord) GetType() string {
return strings.SplitN(string(m), " ", 4)[1]
}
func (m myRecord) GetContent() string {
return strings.SplitN(string(m), " ", 4)[3]
}
func (m myRecord) GetComparisionData() string {
return fmt.Sprint(strings.SplitN(string(m), " ", 4)[2])
func myRecord(s string) *models.RecordConfig {
parts := strings.Split(s, " ")
ttl, _ := strconv.ParseUint(parts[2], 10, 32)
return &models.RecordConfig{
NameFQDN: dnsutil.AddOrigin(parts[0], "example.com"),
Type: parts[1],
TTL: uint32(ttl),
Target: parts[3],
Metadata: map[string]string{},
}
}
func TestAdditionsOnly(t *testing.T) {
desired := []Record{
desired := []*models.RecordConfig{
myRecord("@ A 1 1.2.3.4"),
}
existing := []Record{}
existing := []*models.RecordConfig{}
checkLengths(t, existing, desired, 0, 1, 0, 0)
}
func TestDeletionsOnly(t *testing.T) {
existing := []Record{
existing := []*models.RecordConfig{
myRecord("@ A 1 1.2.3.4"),
}
desired := []Record{}
desired := []*models.RecordConfig{}
checkLengths(t, existing, desired, 0, 0, 1, 0)
}
func TestModification(t *testing.T) {
existing := []Record{
existing := []*models.RecordConfig{
myRecord("www A 1 1.1.1.1"),
myRecord("@ A 1 1.2.3.4"),
}
desired := []Record{
desired := []*models.RecordConfig{
myRecord("@ A 32 1.2.3.4"),
myRecord("www A 1 1.1.1.1"),
}
un, _, _, mod := checkLengths(t, existing, desired, 1, 0, 0, 1)
if t.Failed() {
return
}
if un[0].Desired != desired[1] || un[0].Existing != existing[0] {
t.Error("Expected unchanged records to be correlated")
}
@@ -62,10 +56,10 @@ func TestModification(t *testing.T) {
}
func TestUnchangedWithAddition(t *testing.T) {
existing := []Record{
existing := []*models.RecordConfig{
myRecord("www A 1 1.1.1.1"),
}
desired := []Record{
desired := []*models.RecordConfig{
myRecord("www A 1 1.2.3.4"),
myRecord("www A 1 1.1.1.1"),
}
@@ -76,12 +70,12 @@ func TestUnchangedWithAddition(t *testing.T) {
}
func TestOutOfOrderRecords(t *testing.T) {
existing := []Record{
existing := []*models.RecordConfig{
myRecord("www A 1 1.1.1.1"),
myRecord("www A 1 2.2.2.2"),
myRecord("www A 1 3.3.3.3"),
}
desired := []Record{
desired := []*models.RecordConfig{
myRecord("www A 1 1.1.1.1"),
myRecord("www A 1 2.2.2.2"),
myRecord("www A 1 2.2.2.3"),
@@ -91,11 +85,55 @@ func TestOutOfOrderRecords(t *testing.T) {
if mods[0].Desired != desired[3] || mods[0].Existing != existing[2] {
t.Fatalf("Expected to match %s and %s, but matched %s and %s", existing[2], desired[3], mods[0].Existing, mods[0].Desired)
}
}
func checkLengths(t *testing.T, existing, desired []Record, unCount, createCount, delCount, modCount int) (un, cre, del, mod Changeset) {
un, cre, del, mod = IncrementalDiff(existing, desired)
func TestMxPrio(t *testing.T) {
existing := []*models.RecordConfig{
myRecord("www MX 1 1.1.1.1"),
}
desired := []*models.RecordConfig{
myRecord("www MX 1 1.1.1.1"),
}
existing[0].Priority = 10
desired[0].Priority = 20
checkLengths(t, existing, desired, 0, 0, 0, 1)
}
func TestTTLChange(t *testing.T) {
existing := []*models.RecordConfig{
myRecord("www MX 1 1.1.1.1"),
}
desired := []*models.RecordConfig{
myRecord("www MX 10 1.1.1.1"),
}
checkLengths(t, existing, desired, 0, 0, 0, 1)
}
func TestMetaChange(t *testing.T) {
existing := []*models.RecordConfig{
myRecord("www MX 1 1.1.1.1"),
}
desired := []*models.RecordConfig{
myRecord("www MX 1 1.1.1.1"),
}
existing[0].Metadata["k"] = "aa"
desired[0].Metadata["k"] = "bb"
checkLengths(t, existing, desired, 1, 0, 0, 0)
getMeta := func(r *models.RecordConfig) map[string]string {
return map[string]string{
"k": r.Metadata["k"],
}
}
checkLengths(t, existing, desired, 0, 0, 0, 1, getMeta)
}
func checkLengths(t *testing.T, existing, desired []*models.RecordConfig, unCount, createCount, delCount, modCount int, valFuncs ...func(*models.RecordConfig) map[string]string) (un, cre, del, mod Changeset) {
dc := &models.DomainConfig{
Name: "example.com",
Records: desired,
}
d := New(dc, valFuncs...)
un, cre, del, mod = d.IncrementalDiff(existing)
if len(un) != unCount {
t.Errorf("Got %d unchanged records, but expected %d", len(un), unCount)
}
@@ -108,5 +146,8 @@ func checkLengths(t *testing.T, existing, desired []Record, unCount, createCount
if len(mod) != modCount {
t.Errorf("Got %d records to modify, but expected %d", len(mod), modCount)
}
if t.Failed() {
t.FailNow()
}
return
}