From 900d4042e8c86fa85b76cade46a4cd31df3690a3 Mon Sep 17 00:00:00 2001 From: Jaye Doepke Date: Tue, 7 Dec 2021 15:29:29 -0600 Subject: [PATCH] ROUTE53: Adopt aws-sdk-go-v2 (#1321) * Switch to aws-sdk-go-v2 AWS has released v2 of their SDK for Go. See: https://aws.github.io/aws-sdk-go-v2/ One big advantage of this is no longer needing to export the `AWS_SDK_LOAD_CONFIG=1` env var when using named profiles. * Update integration test README * Reenable pager601 and pager1201 integration tests for AWS Route53 * Implement intelligent batching for Route53 record changes The AWS Route53 API for batch record changes limits the request size to the smaller of: - 1000 records. - 32000 characters total for record values. Also UPSERTs count as double (a DELETE and then a CREATE). This commit changes how the record ChangeBatches are created to respect these limits. * Remove old comments Co-authored-by: Tom Limoncelli --- docs/_providers/route53.md | 3 +- go.mod | 6 +- go.sum | 29 +- integrationTest/integration_test.go | 4 +- integrationTest/readme.md | 18 +- providers/route53/route53Provider.go | 322 ++++++++++++++-------- providers/route53/route53Provider_test.go | 121 +++++++- 7 files changed, 368 insertions(+), 135 deletions(-) diff --git a/docs/_providers/route53.md b/docs/_providers/route53.md index eaa8d1308..87ad4d781 100644 --- a/docs/_providers/route53.md +++ b/docs/_providers/route53.md @@ -36,10 +36,9 @@ $ export AWS_SESSION_TOKEN=ZZZZZZZZ } {% endhighlight %} -Alternatively if you want to used [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) you need to export the following variables +Alternatively if you want to used [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) you need to export the following variable ``` -$ export AWS_SDK_LOAD_CONFIG=1 $ export AWS_PROFILE=ZZZZZZZZ ``` diff --git a/go.mod b/go.mod index 964ea8154..ab36d8f80 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,11 @@ require ( github.com/TomOnTime/utfutil v0.0.0-20210710122150-437f72b26edf github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 - github.com/aws/aws-sdk-go v1.42.13 + github.com/aws/aws-sdk-go-v2 v1.11.0 + github.com/aws/aws-sdk-go-v2/config v1.10.1 + github.com/aws/aws-sdk-go-v2/credentials v1.6.1 + github.com/aws/aws-sdk-go-v2/service/route53 v1.14.0 + github.com/aws/aws-sdk-go-v2/service/route53domains v1.7.0 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e github.com/billputer/go-namecheap v0.0.0-20210108011502-994a912fb7f9 diff --git a/go.sum b/go.sum index d316c04fb..4b85b6cc6 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,32 @@ github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.42.13 h1:+Nx87T+Bjiq2XybxK6vI98cTEBPLE/hILuZyEenlyEg= -github.com/aws/aws-sdk-go v1.42.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go-v2 v1.11.0 h1:HxyD62DyNhCfiFGUHqJ/xITD6rAjJ7Dm/2nLxLmO4Ag= +github.com/aws/aws-sdk-go-v2 v1.11.0/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2/config v1.10.1 h1:z/ViqIjW6ZeuLWgTWMTSyZzaVWo/1cWeVf1Uu+RF01E= +github.com/aws/aws-sdk-go-v2/config v1.10.1/go.mod h1:auIv5pIIn3jIBHNRcVQcsczn6Pfa6Dyv80Fai0ueoJU= +github.com/aws/aws-sdk-go-v2/credentials v1.6.1 h1:A39JYth2fFCx+omN/gib/jIppx3rRnt2r7UKPq7Mh5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.6.1/go.mod h1:QyvQk1IYTqBWSi1T6UgT/W8DMxBVa5pVuLFSRLLhGf8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0 h1:OpZjuUy8Jt3CA1WgJgBC5Bz+uOjE5Ppx4NFTRaooUuA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0/go.mod h1:5E1J3/TTYy6z909QNR0QnXGBpfESYGDqd3O0zqONghU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0 h1:zY8cNmbBXt3pzjgWgdIbzpQ6qxoCwt+Nx9JbrAf2mbY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0/go.mod h1:NO3Q5ZTTQtO2xIg2+xTXYDiT7knSejfeDm7WGDaOo0U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0 h1:Z3aR/OXBnkYK9zXkNkfitHX6SmUBzSsx8VMHbH4Lvhw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0/go.mod h1:anlUzBoEWglcUxUQwZA7HQOEVEnQALVZsizAapB2hq8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.0 h1:c10Z7fWxtJCoyc8rv06jdh9xrKnu7bAJiRaKWvTb2mU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.0/go.mod h1:6oXGy4GLpypD3uCh8wcqztigGgmhLToMfjavgh+VySg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0 h1:qGZWS/WgiFY+Zgad2u0gwBHpJxz6Ne401JE7iQI1nKs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0/go.mod h1:Mq6AEc+oEjCUlBuLiK5YwW4shSOAKCQ3tXN0sQeYoBA= +github.com/aws/aws-sdk-go-v2/service/route53 v1.14.0 h1:0SJgP/L7413/m8itu30tEcsEfup9Ky5TOyhqGaefZ4c= +github.com/aws/aws-sdk-go-v2/service/route53 v1.14.0/go.mod h1:s0AHQXKd6Jo4hsu2N9R1kxJuKLsEY8pIp3GUegGMrqk= +github.com/aws/aws-sdk-go-v2/service/route53domains v1.7.0 h1:wXztJWR5n6LIep/rWo5HlMPjyyUP67xaGDWaFjCcbTU= +github.com/aws/aws-sdk-go-v2/service/route53domains v1.7.0/go.mod h1:qPnejxOymP2/tcqFuYAWJyaeCgSuEjahjXT5s/2bteI= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.0 h1:JDgKIUZOmLFu/Rv6zXLrVTWCmzA0jcTdvsT8iFIKrAI= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.0/go.mod h1:Q/l0ON1annSU+mc0JybDy1Gy6dnJxIcWjphO6qJPzvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.10.0 h1:1jh8J+JjYRp+QWKOsaZt7rGUgoyrqiiVwIm+w0ymeUw= +github.com/aws/aws-sdk-go-v2/service/sts v1.10.0/go.mod h1:jLKCFqS+1T4i7HDqCP9GM4Uk75YW1cS0o82LdxpMyOE= +github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= +github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -691,7 +715,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/integrationTest/integration_test.go b/integrationTest/integration_test.go index c71d91188..43516c92e 100644 --- a/integrationTest/integration_test.go +++ b/integrationTest/integration_test.go @@ -1041,7 +1041,7 @@ func makeTests(t *testing.T) []*TestGroup { //"AZURE_DNS", // Currently failing. "HEXONET", "GCLOUD", - //"ROUTE53", // Currently failing. See https://github.com/StackExchange/dnscontrol/issues/908 + "ROUTE53", ), tc("601 records", manyA("rec%04d", "1.2.3.4", 600)...), tc("Update 601 records", manyA("rec%04d", "1.2.3.5", 600)...), @@ -1054,7 +1054,7 @@ func makeTests(t *testing.T) []*TestGroup { //"AZURE_DNS", // Currently failing. See https://github.com/StackExchange/dnscontrol/issues/770 "HEXONET", "HOSTINGDE", - //"ROUTE53", // Currently failing. See https://github.com/StackExchange/dnscontrol/issues/908 + "ROUTE53", ), tc("1200 records", manyA("rec%04d", "1.2.3.4", 1200)...), tc("Update 1200 records", manyA("rec%04d", "1.2.3.5", 1200)...), diff --git a/integrationTest/readme.md b/integrationTest/readme.md index dd91ccec9..bc91a57e2 100644 --- a/integrationTest/readme.md +++ b/integrationTest/readme.md @@ -12,19 +12,19 @@ For each step, it will run the config once and expect changes. It will run it ag ## Running a test -1. Define all environment variables expected for the provider you wish to run. I setup a local `.env` file with the appropriate values and use [zoo](https://github.com/jsonmaur/zoo) to run my commands. -2. run `go test -v -provider $NAME` where $NAME is the name of the provider you wish to run. +1. Define all environment variables expected for the provider you wish to run. I setup a local `.env` file with the appropriate values and use [zoo](https://github.com/jsonmaur/zoo) to run my commands. +2. run `go test -v -provider $NAME` where $NAME is the name of the provider you wish to run. Example: ``` -$ egrep R53 providers.json - "KeyId": "$R53_KEY_ID", - "SecretKey": "$R53_KEY", - "domain": "$R53_DOMAIN" -$ export R53_KEY_ID="redacted" -$ export R53_KEY="also redacted" -$ export R53_DOMAIN="testdomain.tld" +$ egrep ROUTE53 providers.json + "KeyId": "$ROUTE53_KEY_ID", + "SecretKey": "$ROUTE53_KEY", + "domain": "$ROUTE53_DOMAIN" +$ export ROUTE53_KEY_ID="redacted" +$ export ROUTE53_KEY="also redacted" +$ export ROUTE53_DOMAIN="testdomain.tld" $ go test -v -verbose -provider ROUTE53 ``` diff --git a/providers/route53/route53Provider.go b/providers/route53/route53Provider.go index a40f88ba1..cebd39edf 100644 --- a/providers/route53/route53Provider.go +++ b/providers/route53/route53Provider.go @@ -1,6 +1,7 @@ package route53 import ( + "context" "encoding/json" "errors" "fmt" @@ -8,12 +9,15 @@ import ( "sort" "strings" "time" + "unicode/utf8" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - r53 "github.com/aws/aws-sdk-go/service/route53" - r53d "github.com/aws/aws-sdk-go/service/route53domains" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + r53 "github.com/aws/aws-sdk-go-v2/service/route53" + r53Types "github.com/aws/aws-sdk-go-v2/service/route53/types" + r53d "github.com/aws/aws-sdk-go-v2/service/route53domains" + r53dTypes "github.com/aws/aws-sdk-go-v2/service/route53domains/types" "github.com/StackExchange/dnscontrol/v3/models" "github.com/StackExchange/dnscontrol/v3/pkg/diff" @@ -22,12 +26,12 @@ import ( ) type route53Provider struct { - client *r53.Route53 - registrar *r53d.Route53Domains + client *r53.Client + registrar *r53d.Client delegationSet *string - zonesById map[string]*r53.HostedZone - zonesByDomain map[string]*r53.HostedZone - originalRecords []*r53.ResourceRecordSet + zonesById map[string]r53Types.HostedZone + zonesByDomain map[string]r53Types.HostedZone + originalRecords []r53Types.ResourceRecordSet } func newRoute53Reg(conf map[string]string) (providers.Registrar, error) { @@ -39,28 +43,31 @@ func newRoute53Dsp(conf map[string]string, metadata json.RawMessage) (providers. } func newRoute53(m map[string]string, metadata json.RawMessage) (*route53Provider, error) { - keyID, secretKey, tokenID := m["KeyId"], m["SecretKey"], m["Token"] - - // Route53 uses a global endpoint and route53domains - // currently only has a single regional endpoint in us-east-1 - // http://docs.aws.amazon.com/general/latest/gr/rande.html#r53_region - config := &aws.Config{ - Region: aws.String("us-east-1"), + optFns := []func(*config.LoadOptions) error{ + // Route53 uses a global endpoint and route53domains + // currently only has a single regional endpoint in us-east-1 + // http://docs.aws.amazon.com/general/latest/gr/rande.html#r53_region + config.WithRegion("us-east-1"), } + keyID, secretKey, tokenID := m["KeyId"], m["SecretKey"], m["Token"] // Token is optional and left empty unless required if keyID != "" || secretKey != "" { - config.Credentials = credentials.NewStaticCredentials(keyID, secretKey, tokenID) + optFns = append(optFns, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(keyID, secretKey, tokenID))) + } + + config, err := config.LoadDefaultConfig(context.Background(), optFns...) + if err != nil { + return nil, err } - sess := session.Must(session.NewSession(config)) var dls *string if val, ok := m["DelegationSet"]; ok { fmt.Printf("ROUTE53 DelegationSet %s configured\n", val) - dls = sPtr(val) + dls = aws.String(val) } - api := &route53Provider{client: r53.New(sess), registrar: r53d.New(sess), delegationSet: dls} - err := api.getZones() + api := &route53Provider{client: r53.NewFromConfig(config), registrar: r53d.NewFromConfig(config), delegationSet: dls} + err = api.getZones() if err != nil { return nil, err } @@ -89,10 +96,6 @@ func init() { providers.RegisterCustomRecordType("R53_ALIAS", "ROUTE53", "") } -func sPtr(s string) *string { - return &s -} - func withRetry(f func() error) { const maxRetries = 23 // TODO: exponential backoff @@ -128,14 +131,14 @@ func (r *route53Provider) ListZones() ([]string, error) { func (r *route53Provider) getZones() error { var nextMarker *string - r.zonesByDomain = make(map[string]*r53.HostedZone) - r.zonesById = make(map[string]*r53.HostedZone) + r.zonesByDomain = make(map[string]r53Types.HostedZone) + r.zonesById = make(map[string]r53Types.HostedZone) for { var out *r53.ListHostedZonesOutput var err error withRetry(func() error { inp := &r53.ListHostedZonesInput{Marker: nextMarker} - out, err = r.client.ListHostedZones(inp) + out, err = r.client.ListHostedZones(context.Background(), inp) return err }) if err != nil && strings.Contains(err.Error(), "is not authorized") { @@ -144,9 +147,9 @@ func (r *route53Provider) getZones() error { return err } for _, z := range out.HostedZones { - domain := strings.TrimSuffix(*z.Name, ".") + domain := strings.TrimSuffix(aws.ToString(z.Name), ".") r.zonesByDomain[domain] = z - r.zonesById[parseZoneId(*z.Id)] = z + r.zonesById[parseZoneId(aws.ToString(z.Id))] = z } if out.NextMarker != nil { nextMarker = out.NextMarker @@ -182,7 +185,7 @@ func (r *route53Provider) GetNameservers(domain string) ([]*models.Nameserver, e var z *r53.GetHostedZoneOutput var err error withRetry(func() error { - z, err = r.client.GetHostedZone(&r53.GetHostedZoneInput{Id: zone.Id}) + z, err = r.client.GetHostedZone(context.Background(), &r53.GetHostedZoneInput{Id: zone.Id}) return err }) if err != nil { @@ -191,9 +194,7 @@ func (r *route53Provider) GetNameservers(domain string) ([]*models.Nameserver, e var nss []string if z.DelegationSet != nil { - for _, nsPtr := range z.DelegationSet.NameServers { - nss = append(nss, *nsPtr) - } + nss = z.DelegationSet.NameServers } return models.ToNameservers(nss) } @@ -206,11 +207,11 @@ func (r *route53Provider) GetZoneRecords(domain string) (models.Records, error) return nil, errDomainNoExist{domain} } -func (r *route53Provider) getZone(dc *models.DomainConfig) (*r53.HostedZone, error) { +func (r *route53Provider) getZone(dc *models.DomainConfig) (r53Types.HostedZone, error) { if zoneId, ok := dc.Metadata["zone_id"]; ok { zone, ok := r.zonesById[zoneId] if !ok { - return nil, errZoneNoExist{zoneId} + return r53Types.HostedZone{}, errZoneNoExist{zoneId} } return zone, nil } @@ -219,10 +220,10 @@ func (r *route53Provider) getZone(dc *models.DomainConfig) (*r53.HostedZone, err return zone, nil } - return nil, errDomainNoExist{dc.Name} + return r53Types.HostedZone{}, errDomainNoExist{dc.Name} } -func (r *route53Provider) getZoneRecords(zone *r53.HostedZone) (models.Records, error) { +func (r *route53Provider) getZoneRecords(zone r53Types.HostedZone) (models.Records, error) { records, err := r.fetchRecordSets(zone.Id) if err != nil { return nil, err @@ -321,9 +322,9 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode // we collect all changes into one of two categories now: // pure deletions where we delete an entire record set, // or changes where we upsert an entire record set. - dels := []*r53.Change{} + dels := []r53Types.Change{} delDesc := []string{} - changes := []*r53.Change{} + changes := []r53Types.Change{} changeDesc := []string{} for _, k := range updateOrder { @@ -332,22 +333,26 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode // indicates we should delete all records at that key. if len(recs) == 0 { // To delete, we submit the original resource set we got from r53. - var rrset *r53.ResourceRecordSet + var ( + rrset r53Types.ResourceRecordSet + found bool + ) // Find the original resource set: for _, r := range r.originalRecords { - if unescape(r.Name) == k.NameFQDN && (*r.Type == k.Type || k.Type == "R53_ALIAS_"+*r.Type) { + if unescape(r.Name) == k.NameFQDN && (string(r.Type) == k.Type || k.Type == "R53_ALIAS_"+string(r.Type)) { rrset = r + found = true break } } - if rrset == nil { + if !found { // This should not happen. return nil, fmt.Errorf("no record set found to delete. Name: '%s'. Type: '%s'", k.NameFQDN, k.Type) } // Assemble the change and add it to the list: - chg := &r53.Change{ - Action: sPtr("DELETE"), - ResourceRecordSet: rrset, + chg := r53Types.Change{ + Action: r53Types.ChangeActionDelete, + ResourceRecordSet: &rrset, } dels = append(dels, chg) delDesc = append(delDesc, strings.Join(namesToUpdate[k], "\n")) @@ -363,10 +368,10 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode } for _, r := range recs { rrset := aliasToRRSet(zone, r) - rrset.Name = sPtr(k.NameFQDN) + rrset.Name = aws.String(k.NameFQDN) // Assemble the change and add it to the list: - chg := &r53.Change{ - Action: sPtr("UPSERT"), + chg := r53Types.Change{ + Action: r53Types.ChangeActionUpsert, ResourceRecordSet: rrset, } changes = append(changes, chg) @@ -374,22 +379,22 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode } } else { // All other keys combine their updates into one rrset: - rrset := &r53.ResourceRecordSet{ - Name: sPtr(k.NameFQDN), - Type: sPtr(k.Type), + rrset := &r53Types.ResourceRecordSet{ + Name: aws.String(k.NameFQDN), + Type: r53Types.RRType(k.Type), } for _, r := range recs { val := r.GetTargetCombined() - rr := &r53.ResourceRecord{ - Value: &val, + rr := r53Types.ResourceRecord{ + Value: aws.String(val), } rrset.ResourceRecords = append(rrset.ResourceRecords, rr) i := int64(r.TTL) rrset.TTL = &i // TODO: make sure that ttls are consistent within a set } // Assemble the change and add it to the list: - chg := &r53.Change{ - Action: sPtr("UPSERT"), + chg := r53Types.Change{ + Action: r53Types.ChangeActionUpsert, ResourceRecordSet: rrset, } changes = append(changes, chg) @@ -407,7 +412,7 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode var err error req.HostedZoneId = zone.Id withRetry(func() error { - _, err = r.client.ChangeResourceRecordSets(req) + _, err = r.client.ChangeResourceRecordSets(context.Background(), req) return err }) return err @@ -415,75 +420,67 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode }) } - getBatchSize := func(size, max int) int { - if size > max { - return max + batcher := newChangeBatcher(dels) + for batcher.Next() { + start, end := batcher.Batch() + batch := dels[start:end] + descBatchStr := "\n" + strings.Join(delDesc[start:end], "\n") + "\n" + req := &r53.ChangeResourceRecordSetsInput{ + ChangeBatch: &r53Types.ChangeBatch{Changes: batch}, } - return size + addCorrection(descBatchStr, req) + } + if err := batcher.Err(); err != nil { + return nil, err } - for len(dels) > 0 { - batchSize := getBatchSize(len(dels), 1000) - batch := dels[:batchSize] - dels = dels[batchSize:] - delDescBatch := delDesc[:batchSize] - delDesc = delDesc[batchSize:] - - delDescBatchStr := "\n" + strings.Join(delDescBatch, "\n") + "\n" - - delReq := &r53.ChangeResourceRecordSetsInput{ - ChangeBatch: &r53.ChangeBatch{Changes: batch}, + batcher = newChangeBatcher(changes) + for batcher.Next() { + start, end := batcher.Batch() + batch := changes[start:end] + descBatchStr := "\n" + strings.Join(changeDesc[start:end], "\n") + "\n" + req := &r53.ChangeResourceRecordSetsInput{ + ChangeBatch: &r53Types.ChangeBatch{Changes: batch}, } - addCorrection(delDescBatchStr, delReq) + addCorrection(descBatchStr, req) } - - for len(changes) > 0 { - batchSize := getBatchSize(len(changes), 500) - batch := changes[:batchSize] - changes = changes[batchSize:] - changeDescBatch := changeDesc[:batchSize] - changeDesc = changeDesc[batchSize:] - changeDescBatchStr := "\n" + strings.Join(changeDescBatch, "\n") + "\n" - - changeReq := &r53.ChangeResourceRecordSetsInput{ - ChangeBatch: &r53.ChangeBatch{Changes: batch}, - } - addCorrection(changeDescBatchStr, changeReq) + if err := batcher.Err(); err != nil { + return nil, err } return corrections, nil } -func nativeToRecords(set *r53.ResourceRecordSet, origin string) ([]*models.RecordConfig, error) { +func nativeToRecords(set r53Types.ResourceRecordSet, origin string) ([]*models.RecordConfig, error) { results := []*models.RecordConfig{} if set.AliasTarget != nil { rc := &models.RecordConfig{ Type: "R53_ALIAS", TTL: 300, R53Alias: map[string]string{ - "type": *set.Type, - "zone_id": *set.AliasTarget.HostedZoneId, + "type": string(set.Type), + "zone_id": aws.ToString(set.AliasTarget.HostedZoneId), }, } rc.SetLabelFromFQDN(unescape(set.Name), origin) - rc.SetTarget(aws.StringValue(set.AliasTarget.DNSName)) + rc.SetTarget(aws.ToString(set.AliasTarget.DNSName)) results = append(results, rc) } else if set.TrafficPolicyInstanceId != nil { // skip traffic policy records } else { for _, rec := range set.ResourceRecords { - switch rtype := *set.Type; rtype { - case "SOA": + switch rtype := set.Type; rtype { + case r53Types.RRTypeSoa: continue - case "SPF": + case r53Types.RRTypeSpf: // route53 uses a custom record type for SPF rtype = "TXT" fallthrough default: - rc := &models.RecordConfig{TTL: uint32(*set.TTL)} + rc := &models.RecordConfig{TTL: uint32(aws.ToInt64(set.TTL))} rc.SetLabelFromFQDN(unescape(set.Name), origin) - if err := rc.PopulateFromString(rtype, *rec.Value, origin); err != nil { + if err := rc.PopulateFromString(string(rtype), *rec.Value, origin); err != nil { return nil, fmt.Errorf("unparsable record received from R53: %w", err) } results = append(results, rc) @@ -500,25 +497,24 @@ func getAliasMap(r *models.RecordConfig) map[string]string { return r.R53Alias } -func aliasToRRSet(zone *r53.HostedZone, r *models.RecordConfig) *r53.ResourceRecordSet { +func aliasToRRSet(zone r53Types.HostedZone, r *models.RecordConfig) *r53Types.ResourceRecordSet { target := r.GetTargetField() zoneID := getZoneID(zone, r) - targetHealth := false - rrset := &r53.ResourceRecordSet{ - Type: sPtr(r.R53Alias["type"]), - AliasTarget: &r53.AliasTarget{ + rrset := &r53Types.ResourceRecordSet{ + Type: r53Types.RRType(r.R53Alias["type"]), + AliasTarget: &r53Types.AliasTarget{ DNSName: &target, HostedZoneId: aws.String(zoneID), - EvaluateTargetHealth: &targetHealth, + EvaluateTargetHealth: false, }, } return rrset } -func getZoneID(zone *r53.HostedZone, r *models.RecordConfig) string { +func getZoneID(zone r53Types.HostedZone, r *models.RecordConfig) string { zoneID := r.R53Alias["zone_id"] if zoneID == "" { - zoneID = aws.StringValue(zone.Id) + zoneID = aws.ToString(zone.Id) } return parseZoneId(zoneID) } @@ -566,7 +562,7 @@ func (r *route53Provider) getRegistrarNameservers(domainName *string) ([]string, var domainDetail *r53d.GetDomainDetailOutput var err error withRetry(func() error { - domainDetail, err = r.registrar.GetDomainDetail(&r53d.GetDomainDetailInput{DomainName: domainName}) + domainDetail, err = r.registrar.GetDomainDetail(context.Background(), &r53d.GetDomainDetailInput{DomainName: domainName}) return err }) if err != nil { @@ -575,22 +571,24 @@ func (r *route53Provider) getRegistrarNameservers(domainName *string) ([]string, nameservers := []string{} for _, ns := range domainDetail.Nameservers { - nameservers = append(nameservers, *ns.Name) + nameservers = append(nameservers, aws.ToString(ns.Name)) } return nameservers, nil } func (r *route53Provider) updateRegistrarNameservers(domainName string, nameservers []string) (*string, error) { - servers := []*r53d.Nameserver{} + servers := make([]r53dTypes.Nameserver, len(nameservers)) for i := range nameservers { - servers = append(servers, &r53d.Nameserver{Name: &nameservers[i]}) + servers[i] = r53dTypes.Nameserver{Name: aws.String(nameservers[i])} } var domainUpdate *r53d.UpdateDomainNameserversOutput var err error withRetry(func() error { - domainUpdate, err = r.registrar.UpdateDomainNameservers(&r53d.UpdateDomainNameserversInput{ - DomainName: &domainName, Nameservers: servers}) + domainUpdate, err = r.registrar.UpdateDomainNameservers(context.Background(), &r53d.UpdateDomainNameserversInput{ + DomainName: aws.String(domainName), + Nameservers: servers, + }) return err }) if err != nil { @@ -600,24 +598,24 @@ func (r *route53Provider) updateRegistrarNameservers(domainName string, nameserv return domainUpdate.OperationId, nil } -func (r *route53Provider) fetchRecordSets(zoneID *string) ([]*r53.ResourceRecordSet, error) { +func (r *route53Provider) fetchRecordSets(zoneID *string) ([]r53Types.ResourceRecordSet, error) { if zoneID == nil || *zoneID == "" { return nil, nil } var next *string - var nextType *string - var records []*r53.ResourceRecordSet + var nextType r53Types.RRType + var records []r53Types.ResourceRecordSet for { listInput := &r53.ListResourceRecordSetsInput{ HostedZoneId: zoneID, StartRecordName: next, StartRecordType: nextType, - MaxItems: sPtr("100"), + MaxItems: aws.Int32(100), } var list *r53.ListResourceRecordSetsOutput var err error withRetry(func() error { - list, err = r.client.ListResourceRecordSets(listInput) + list, err = r.client.ListResourceRecordSets(context.Background(), listInput) return err }) if err != nil { @@ -657,12 +655,102 @@ func (r *route53Provider) EnsureDomainExists(domain string) error { in := &r53.CreateHostedZoneInput{ Name: &domain, DelegationSetId: r.delegationSet, - CallerReference: sPtr(fmt.Sprint(time.Now().UnixNano())), + CallerReference: aws.String(fmt.Sprint(time.Now().UnixNano())), } var err error withRetry(func() error { - _, err := r.client.CreateHostedZone(in) + _, err := r.client.CreateHostedZone(context.Background(), in) return err }) return err } + +// changeBatcher takes a set of r53Types.Changes and turns them into a series of +// batches that meet the limits of the ChangeResourceRecordSets API. +// +// See also: https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-changeresourcerecordsets +type changeBatcher struct { + changes []r53Types.Change + + maxSize int // Max records per request. + maxChars int // Max record value characters per request. + + start, end int // Cursors into changes. + err error // Populated by Next. +} + +// newChangeBatcher returns a new changeBatcher. +func newChangeBatcher(changes []r53Types.Change) *changeBatcher { + return &changeBatcher{ + changes: changes, + maxSize: 1000, // "A request cannot contain more than 1,000 ResourceRecord elements." + maxChars: 32000, // "The sum of the number of characters (including spaces) in all Value elements in a request cannot exceed 32,000 characters." + } +} + +// Next returns true if there is another batch of Changes. +// It returns false if there are no more batches or an error occurred. +func (b *changeBatcher) Next() bool { + if b.end >= len(b.changes) || b.err != nil { + return false + } + + start, end := b.end, b.end + var ( + reqSize int + reqChars int + ) + for end < len(b.changes) { + c := &b.changes[end] + + // Check that we won't exceed 1000 ResourceRecords in the request. + rrsetSize := len(c.ResourceRecordSet.ResourceRecords) + if c.Action == r53Types.ChangeActionUpsert { + // "When the value of the Action element is UPSERT, each ResourceRecord element is counted twice." + rrsetSize *= 2 + } + if newReqSize := reqSize + rrsetSize; newReqSize > b.maxSize { + break + } else { + reqSize = newReqSize + } + + // Check that we won't exceed 32000 Value characters in the request. + var rrsetChars int + for _, rr := range c.ResourceRecordSet.ResourceRecords { + rrsetChars += utf8.RuneCountInString(aws.ToString(rr.Value)) + } + if c.Action == r53Types.ChangeActionUpsert { + // "When the value of the Action element is UPSERT, each character in a Value element is counted twice." + rrsetChars *= 2 + } + if newReqChars := reqChars + rrsetChars; newReqChars > b.maxChars { + break + } else { + reqChars = newReqChars + } + + end++ + } + + if start == end { + b.err = errors.New("could not create ChangeResourceRecordSets request within AWS API limits") + return false + } + + b.start = start + b.end = end + + return true +} + +// Batch returns the current batch. It should only be called +// after Next returns true. +func (b *changeBatcher) Batch() (start, end int) { + return b.start, b.end +} + +// Err returns the error encountered during the previous call to Next. +func (b *changeBatcher) Err() error { + return b.err +} diff --git a/providers/route53/route53Provider_test.go b/providers/route53/route53Provider_test.go index bab443ea9..e4171896d 100644 --- a/providers/route53/route53Provider_test.go +++ b/providers/route53/route53Provider_test.go @@ -1,6 +1,13 @@ package route53 -import "testing" +import ( + "fmt" + "reflect" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + r53Types "github.com/aws/aws-sdk-go-v2/service/route53/types" +) func TestUnescape(t *testing.T) { var tests = []struct { @@ -22,3 +29,115 @@ func TestUnescape(t *testing.T) { } } } + +type batch struct { + start int + end int +} + +func (b batch) String() string { + return fmt.Sprintf("%d:%d", b.start, b.end) +} + +func Test_changeBatcher(t *testing.T) { + genChanges := func(action r53Types.ChangeAction, typ r53Types.RRType, namePattern string, n int, targets ...string) []r53Types.Change { + changes := make([]r53Types.Change, n) + for i := 0; i < n; i++ { + changes[i].Action = action + changes[i].ResourceRecordSet = &r53Types.ResourceRecordSet{ + Name: aws.String(fmt.Sprintf(namePattern, i)), + Type: typ, + } + for j := 0; j < len(targets); j++ { + changes[i].ResourceRecordSet.ResourceRecords = append(changes[i].ResourceRecordSet.ResourceRecords, r53Types.ResourceRecord{ + Value: aws.String(targets[j]), + }) + } + } + return changes + } + + type fields struct { + changes []r53Types.Change + maxSize int + maxChars int + } + tests := []struct { + name string + fields fields + want []batch + wantErr bool + }{ + { + name: "one_batch", + fields: fields{ + changes: genChanges(r53Types.ChangeActionUpsert, r53Types.RRTypeA, "rec%04d", 99, "1.2.3.4"), + maxSize: 1000, + maxChars: 32000, + }, + want: []batch{ + {start: 0, end: 99}, + }, + wantErr: false, + }, + { + name: "multi_batch_size", + fields: fields{ + changes: genChanges(r53Types.ChangeActionUpsert, r53Types.RRTypeA, "rec%04d", 2000, "1.2.3.4"), + maxSize: 1000, + maxChars: 32000, + }, + want: []batch{ + {start: 0, end: 500}, + {start: 500, end: 1000}, + {start: 1000, end: 1500}, + {start: 1500, end: 2000}, + }, + wantErr: false, + }, + { + name: "multi_batch_chars", + fields: fields{ + changes: genChanges(r53Types.ChangeActionCreate, r53Types.RRTypeTxt, "rec%04d", 1000, "1.2.3.4", "1.2.3.5", "1.2.3.6", "1.2.3.7", "1.2.3.8", "1.2.3.9"), + maxSize: 1000, + maxChars: 32000, + }, + want: []batch{ + {start: 0, end: 166}, + {start: 166, end: 332}, + {start: 332, end: 498}, + {start: 498, end: 664}, + {start: 664, end: 830}, + {start: 830, end: 996}, + {start: 996, end: 1000}, + }, + wantErr: false, + }, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &changeBatcher{ + changes: tt.fields.changes, + maxSize: tt.fields.maxSize, + maxChars: tt.fields.maxChars, + } + got := make([]batch, 0) + for b.Next() { + start, end := b.Batch() + got = append(got, batch{ + start: start, + end: end, + }) + } + err := b.Err() + if tt.wantErr && err == nil { + t.Errorf("%d: Expected an error, got nil", i) + } else if !tt.wantErr && err != nil { + t.Errorf("%d: Expected no error, got '%s'", i, err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("%d: Expected %s, got %s", i, tt.want, got) + } + }) + } +}