diff --git a/integrationTest/integration_test.go b/integrationTest/integration_test.go index c3258f413..058fc5fda 100644 --- a/integrationTest/integration_test.go +++ b/integrationTest/integration_test.go @@ -616,7 +616,19 @@ func makeTests(t *testing.T) []*TestCase { ) } + // test r53 for very very large batch sizes + if *providerToRun == "ROUTE53"{ + tests = append(tests, + tc("600 records", manyA("rec%04d", "1.2.3.4", 600)...), + tc("Update 600 records", manyA("rec%04d", "1.2.3.5", 600)...), + tc("Empty"), + tc("1200 records", manyA("rec%04d", "1.2.3.4", 1200)...), + tc("Update 1200 records", manyA("rec%04d", "1.2.3.5", 1200)...), + tc("Empty"), + ) + } + // Empty last - tc("Empty") + tests = append(tests,tc("Empty")) return tests } diff --git a/providers/route53/route53Provider.go b/providers/route53/route53Provider.go index 1e04d9b0b..a5ceb9b2a 100644 --- a/providers/route53/route53Provider.go +++ b/providers/route53/route53Provider.go @@ -207,7 +207,8 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode } updates := map[models.RecordKey][]*models.RecordConfig{} - // for each name we need to update, collect relevant records from dc + + // for each name we need to update, collect relevant records from our desired domain state for k := range namesToUpdate { updates[k] = nil for _, rc := range dc.Records { @@ -217,13 +218,18 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode } } + // we collect all changes into one of two categories now: + // pure deletions where we delete an entire record set, + // or changes where we upsert an entire record set. dels := []*r53.Change{} changes := []*r53.Change{} changeDesc := "" delDesc := "" + for k, recs := range updates { chg := &r53.Change{} var rrset *r53.ResourceRecordSet + // if there are no records in our desired state for a key, then we just delete it from r53 if len(recs) == 0 { dels = append(dels, chg) chg.Action = sPtr("DELETE") @@ -264,13 +270,9 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode chg.ResourceRecordSet = rrset } - changeReq := &r53.ChangeResourceRecordSetsInput{ - ChangeBatch: &r53.ChangeBatch{Changes: changes}, - } + - delReq := &r53.ChangeResourceRecordSetsInput{ - ChangeBatch: &r53.ChangeBatch{Changes: dels}, - } + addCorrection := func(msg string, req *r53.ChangeResourceRecordSetsInput) { corrections = append(corrections, @@ -288,11 +290,30 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode }) } - if len(dels) > 0 { + getBatchSize := func(size, max int) int{ + if size > max{ + return max + } + return size + } + + for len(dels) > 0 { + batchSize := getBatchSize(len(dels), 1000) + batch := dels[:batchSize] + dels = dels[batchSize:] + delReq := &r53.ChangeResourceRecordSetsInput{ + ChangeBatch: &r53.ChangeBatch{Changes: batch}, + } addCorrection(delDesc, delReq) } - if len(changes) > 0 { + for len(changes) > 0 { + batchSize := getBatchSize(len(changes), 500) + batch := changes[:batchSize] + changes = changes[batchSize:] + changeReq := &r53.ChangeResourceRecordSetsInput{ + ChangeBatch: &r53.ChangeBatch{Changes: batch}, + } addCorrection(changeDesc, changeReq) }