ROUTE53/GCLOUD: Add Delegation/nameserver Sets (#448)

- Support DelegationSet for Route53 (create-domains only)
- Retry Route53 operations which fail for rate limits under large numbers of domains
- Support for name_server_set for GCloud (create-domains only)
- Docs for both

docs/_providers/gcloud.md

@@ -23,12 +23,13 @@ For Google cloud authentication, DNSControl requires a JSON 'Service Account Key
         "auth_uri": "https://accounts.google.com/o/oauth2/auth",
         "token_uri": "https://accounts.google.com/o/oauth2/token",
         "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-        "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/dnscontrolsdfsdfsdf%40craigdnstest.iam.gserviceaccount.com"
+        "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/dnscontrolsdfsdfsdf%40craigdnstest.iam.gserviceaccount.com",
+        "name_server_set" : "optional_name_server_set_name (contact your TAM)"
     }
 }
 {% endhighlight %}
 
-**Note**: The `project_id`, `private_key`, and `client_email`, are the only fields that are strictly required, but it is sometimes easier to just paste the entire json object in. Either way is fine.
+**Note**: The `project_id`, `private_key`, and `client_email`, are the only fields that are strictly required, but it is sometimes easier to just paste the entire json object in. Either way is fine.  `name_server_set` is optional and requires special permission from your TAM at Google in order to setup (See [Name server sets](#name_server_sets) below)
 
 See [the Activation section](#activation) for some tips on obtaining these credentials.
 
@@ -58,5 +59,13 @@ D("example.tld", REG_NAMECOM, DnsProvider(GCLOUD),
 
 ## New domains
 If a domain does not exist in your Google Cloud DNS account, DNSControl
-will *not* automatically add it with the `create-domains` account. You'll need to do that via the
-control panel manually.
+will *not* automatically add it with the `push` command. You'll need to do that via the
+control panel manually or via the `create-domains` command.
+
+## Name server sets
+
+This optional feature lets you pin domains to a set of GCLOUD name servers.  The `nameServerSet` field is exposed in their API but there is
+currently no facility for creating a name server set.  You need special permission from your technical account manager at Google and they 
+will enable it on your account, responding with a list of names to use in the `name_server_set` field above.
+
+> `name_server_set` only applies on `create-domains` at the moment. Additional work needs to be done to support it during `push`

docs/_providers/route53.md

@@ -13,7 +13,8 @@ You can specify the API credentials in the credentials json file:
  "r53_main":{
       "KeyId": "your-aws-key",
       "SecretKey": "your-aws-secret-key",
-      "Token": "optional-sts-token"
+      "Token": "optional-sts-token",
+      "DelegationSet" : "optional-delegation-set-id"
  }
 }
 {% endhighlight %}
@@ -56,7 +57,32 @@ D('example.tld', REG_NONE, DnsProvider(R53),
 DNSControl depends on a standard [AWS access key](https://aws.amazon.com/developers/access-keys/) with permission to list, create and update hosted zones.
 
 ## New domains
-If a domain does not exist in your Route53 account, DNSControl will *not* automatically add it with the `create-domains` command. You can do that either manually via the control panel, or via the command `dnscontrol create-domains` command.
+If a domain does not exist in your Route53 account, DNSControl will *not* automatically add it with the `push` command. You can do that either manually via the control panel, or via the command `dnscontrol create-domains` command.
+
+## Delegation Sets
+Creation of new delegation sets are not supported by this code.  However, if you have a delegation set already created, ala:
+
+```
+$ aws route53 create-reusable-delegation-set --caller-reference "foo"
+{
+    "Location": "https://route53.amazonaws.com/2013-04-01/delegationset/12312312123",
+    "DelegationSet": {
+        "Id": "/delegationset/12312312123",
+        "CallerReference": "foo",
+        "NameServers": [
+            "ns-1056.awsdns-04.org",
+            "ns-215.awsdns-26.com",
+            "ns-1686.awsdns-18.co.uk",
+            "ns-970.awsdns-57.net"
+        ]
+    }
+}
+```
+
+You can then reference the DelegationSet.Id in your `r53_main` block (with your other credentials) to have all created domains placed in that
+delegation set.  Note that you you only want the portion of the `Id` after the `/delegationset/` (the `12312312123` in the example above).
+
+> Delegation sets only apply during `create-domains` at the moment.  Further work needs to be done to have them apply during `push`.
 
 ## Caveats
 This code may not function properly if a domain has R53 as a Registrar

providers/gcloud/gcloudProvider.go

@@ -25,6 +25,10 @@ var features = providers.DocumentationNotes{
 	providers.CanUseTXTMulti:         providers.Can(),
 }
 
+func sPtr(s string) *string {
+	return &s
+}
+
 func init() {
 	providers.RegisterDomainServiceProviderType("GCLOUD", New, features)
 }
@@ -32,11 +36,12 @@ func init() {
 type gcloud struct {
 	client        *gdns.Service
 	project       string
+	nameServerSet *string
 	zones         map[string]*gdns.ManagedZone
 }
 
 // New creates a new gcloud provider
-func New(cfg map[string]string, _ json.RawMessage) (providers.DNSServiceProvider, error) {
+func New(cfg map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
 	raw, err := json.Marshal(cfg)
 	if err != nil {
 		return nil, err
@@ -51,8 +56,14 @@ func New(cfg map[string]string, _ json.RawMessage) (providers.DNSServiceProvider
 	if err != nil {
 		return nil, err
 	}
+	var nss *string = nil
+	if val, ok := cfg["name_server_set"]; ok {
+		fmt.Printf("GCLOUD :name_server_set %s configured\n", val)
+		nss = sPtr(val)
+	}
 	return &gcloud{
 		client:        dcli,
+		nameServerSet: nss,
 		project:       cfg["project_id"],
 	}, nil
 }
@@ -231,12 +242,23 @@ func (g *gcloud) EnsureDomainExists(domain string) error {
 	if z != nil {
 		return nil
 	}
+	var mz *gdns.ManagedZone
+	if g.nameServerSet != nil {
+		fmt.Printf("Adding zone for %s to gcloud account with name_server_set %s\n", domain, *g.nameServerSet)
+		mz = &gdns.ManagedZone{
+			DnsName:       domain + ".",
+			NameServerSet: *g.nameServerSet,
+			Name:          "zone-" + strings.Replace(domain, ".", "-", -1),
+			Description:   "zone added by dnscontrol",
+		}
+	} else {
 		fmt.Printf("Adding zone for %s to gcloud account \n", domain)
-	mz := &gdns.ManagedZone{
+		mz = &gdns.ManagedZone{
 			DnsName:     domain + ".",
 			Name:        "zone-" + strings.Replace(domain, ".", "-", -1),
 			Description: "zone added by dnscontrol",
 		}
+	}
 	g.zones = nil // reset cache
 	_, err = g.client.ManagedZones.Create(g.project, mz).Do()
 	return err

providers/route53/route53Provider.go

@@ -21,6 +21,7 @@ import (
 type route53Provider struct {
 	client        *r53.Route53
 	registrar     *r53d.Route53Domains
+	delegationSet *string
 	zones         map[string]*r53.HostedZone
 }
 
@@ -48,7 +49,12 @@ func newRoute53(m map[string]string, metadata json.RawMessage) (*route53Provider
 	}
 	sess := session.New(config)
 
-	api := &route53Provider{client: r53.New(sess), registrar: r53d.New(sess)}
+	var dls *string = nil
+	if val, ok := m["DelegationSet"]; ok {
+		fmt.Printf("ROUTE53 DelegationSet %s configured\n", val)
+		dls = sPtr(val)
+	}
+	api := &route53Provider{client: r53.New(sess), registrar: r53d.New(sess), delegationSet: dls}
 	err := api.getZones()
 	if err != nil {
 		return nil, err
@@ -78,13 +84,41 @@ func sPtr(s string) *string {
 	return &s
 }
 
+func withRetry(f func() error) {
+	const maxRetries = 23
+	// TODO: exponential backoff
+	const sleepTime = 5 * time.Second
+	var currentRetry int = 0
+	for {
+		err := f()
+		if err == nil {
+			return
+		}
+		if strings.Contains(err.Error(), "Rate exceeded") {
+			currentRetry++
+			if currentRetry >= maxRetries {
+				return
+			}
+			fmt.Printf("============ Route53 rate limit exceeded. Waiting %s to retry.\n", sleepTime)
+			time.Sleep(sleepTime)
+		} else {
+			return
+		}
+	}
+	return
+}
+
 func (r *route53Provider) getZones() error {
 	var nextMarker *string
 	r.zones = make(map[string]*r53.HostedZone)
 	for {
-
+		var out *r53.ListHostedZonesOutput
+		var err error
+		withRetry(func() error {
 			inp := &r53.ListHostedZonesInput{Marker: nextMarker}
-		out, err := r.client.ListHostedZones(inp)
+			out, err = r.client.ListHostedZones(inp)
+			return err
+		})
 		if err != nil && strings.Contains(err.Error(), "is not authorized") {
 			return errors.New("Check your credentials, your not authorized to perform actions on Route 53 AWS Service")
 		} else if err != nil {
@@ -117,7 +151,12 @@ func (r *route53Provider) GetNameservers(domain string) ([]*models.Nameserver, e
 	if !ok {
 		return nil, errNoExist{domain}
 	}
-	z, err := r.client.GetHostedZone(&r53.GetHostedZoneInput{Id: zone.Id})
+	var z *r53.GetHostedZoneOutput
+	var err error
+	withRetry(func() error {
+		z, err = r.client.GetHostedZone(&r53.GetHostedZoneInput{Id: zone.Id})
+		return err
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -238,8 +277,12 @@ func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*mode
 			&models.Correction{
 				Msg: msg,
 				F: func() error {
+					var err error
 					req.HostedZoneId = zone.Id
-					_, err := r.client.ChangeResourceRecordSets(req)
+					withRetry(func() error {
+						_, err = r.client.ChangeResourceRecordSets(req)
+						return err
+					})
 					return err
 				},
 			})
@@ -357,7 +400,12 @@ func (r *route53Provider) GetRegistrarCorrections(dc *models.DomainConfig) ([]*m
 }
 
 func (r *route53Provider) getRegistrarNameservers(domainName *string) ([]string, error) {
-	domainDetail, err := r.registrar.GetDomainDetail(&r53d.GetDomainDetailInput{DomainName: domainName})
+	var domainDetail *r53d.GetDomainDetailOutput
+	var err error
+	withRetry(func() error {
+		domainDetail, err = r.registrar.GetDomainDetail(&r53d.GetDomainDetailInput{DomainName: domainName})
+		return err
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -375,8 +423,13 @@ func (r *route53Provider) updateRegistrarNameservers(domainName string, nameserv
 	for i := range nameservers {
 		servers = append(servers, &r53d.Nameserver{Name: &nameservers[i]})
 	}
-
-	domainUpdate, err := r.registrar.UpdateDomainNameservers(&r53d.UpdateDomainNameserversInput{DomainName: &domainName, Nameservers: servers})
+	var domainUpdate *r53d.UpdateDomainNameserversOutput
+	var err error
+	withRetry(func() error {
+		domainUpdate, err = r.registrar.UpdateDomainNameservers(&r53d.UpdateDomainNameserversInput{
+			DomainName: &domainName, Nameservers: servers})
+		return err
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -398,10 +451,16 @@ func (r *route53Provider) fetchRecordSets(zoneID *string) ([]*r53.ResourceRecord
 			StartRecordType: nextType,
 			MaxItems:        sPtr("100"),
 		}
-		list, err := r.client.ListResourceRecordSets(listInput)
+		var list *r53.ListResourceRecordSetsOutput
+		var err error
+		withRetry(func() error {
+			list, err = r.client.ListResourceRecordSets(listInput)
+			return err
+		})
 		if err != nil {
 			return nil, err
 		}
+
 		records = append(records, list.ResourceRecordSets...)
 		if list.NextRecordName != nil {
 			next = list.NextRecordName
@@ -427,12 +486,20 @@ func (r *route53Provider) EnsureDomainExists(domain string) error {
 	if _, ok := r.zones[domain]; ok {
 		return nil
 	}
+	if r.delegationSet != nil {
+		fmt.Printf("Adding zone for %s to route 53 account with delegationSet %s\n", domain, *r.delegationSet)
+	} else {
 		fmt.Printf("Adding zone for %s to route 53 account\n", domain)
+	}
 	in := &r53.CreateHostedZoneInput{
 		Name:            &domain,
+		DelegationSetId: r.delegationSet,
 		CallerReference: sPtr(fmt.Sprint(time.Now().UnixNano())),
 	}
+	var err error
+	withRetry(func() error {
 		_, err := r.client.CreateHostedZone(in)
 		return err
-
+	})
+	return err
 }