HEXONET: Implement get-zones, fix module problem (#898)

* VULTR: Update govultr to v1.0.0 (fixes #892) (#897)

* go get -u github.com/hexonet/go-sdk

* Fix HEXONET providers.json entry

* providers.json: json commma

* providers.json: fmtjson

* HEXONET: Implement get-zones. Fix tests and docs.

* fixup!

* Update azure test failures

* Move version info into its own package

* Use new version system

docs/_providers/hexonet.md

@@ -13,6 +13,8 @@ up-time, and most importantly for DNS expertise. DNSControl with HEXONET's DNS
 marries DNS automation with an industry-leading DNS platform that supports DNSSEC,
 PremiumDNS via Anycast Network, and nearly all of DNSControl's listed provider features.
 
+This is based on API documents found at [https://wiki.hexonet.net/wiki/DNS_API](https://wiki.hexonet.net/wiki/DNS_API)
+
 ## Configuration
 
 Please provide your HEXONET login data in your credentials file `creds.json` as follows:
@@ -42,6 +44,20 @@ Here a working example for our OT&E System:
 }
 {% endhighlight %}
 
+NOTE: The above credentials are known to the public.
+
+With the above hexonet entry in `creds.json`, you can run the
+integration tests as follows:
+
+    dnscontrol get-zones --format=nameonly hexonet HEXONET  all
+    # Review the output.  Pick one domain and set HEXONET_DOMAIN.
+    cd $GIT/dnscontrol/integrationTest
+    export HEXONET_DOMAIN=a-b-c-movies.com       # Pick a domain name.
+    export HEXONET_ENTITY=OTE
+    export HEXONET_UID=test.user
+    export HEXONET_PW=test.passw0rd
+    go test -v -verbose -provider HEXONET
+
 ## Usage
 
 Here's an example DNS Configuration `dnsconfig.js` using our provider module.
@@ -77,6 +93,11 @@ D('abhoster.com', REG_HX, DnsProvider(DNS_HX),
 
 This provider does not recognize any special metadata fields unique to HEXONET.
 
+## get-zones
+
+`dnscontrol get-zones` is implemented for this provider. The list
+includes both basic and premier zones.
+
 ## New domains
 
 If a dnszone does not exist in your HEXONET account, DNSControl will *not* automatically add it with the `dnscontrol push` or `dnscontrol preview` command. You'll need to do that via the control panel manually or using the command `dnscontrol create-domains`.
@@ -89,4 +110,4 @@ In general this is thought for our purpose to have an easy way to dive into issu
 
 ## IP Filter
 
-In case you have ip filter settings made for you HEXONET account, please provide your outgoing ip address as shown in the configuration examples above.
+In case you have ip filter settings made for your HEXONET account, please provide your outgoing ip address as shown in the configuration examples above.

docs/release-engineering.md

@@ -44,7 +44,7 @@ how it tests that gofmt was run.
 
 ## Step 3. Bump the version number
 
-Edit the "Version" variable in `main.go` and commit.
+Edit the "Version" variable in `pkg/version/version.go` and commit.
 
 ```
 export PREVVERSION=3.0.0       <<< Change to the previous version

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/google/go-querystring v1.0.1-0.20190318165438-c8c88dbee036 // indirect
 	github.com/gopherjs/jquery v0.0.0-20191017083323-73f4c7416038
 	github.com/hashicorp/vault/api v1.0.4
-	github.com/hexonet/go-sdk v2.2.3+incompatible
+	github.com/hexonet/go-sdk v3.5.0+incompatible
 	github.com/jarcoal/httpmock v1.0.4 // indirect
 	github.com/miekg/dns v1.1.31
 	github.com/mittwald/go-powerdns v0.5.2

go.sum

@@ -234,8 +234,8 @@ github.com/hashicorp/vault/sdk v0.1.13 h1:mOEPeOhT7jl0J4AMl1E705+BcmeRs1VmKNb9F0
 github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
-github.com/hexonet/go-sdk v2.2.3+incompatible h1:V4FVWC11TXdUtxakyhnr6+ttf1e9ah9AQzZsP4u4R24=
+github.com/hexonet/go-sdk v3.5.0+incompatible h1:p64FYQjx4HdhVDNd/qa8QBVSTnD3HP33uJYQsyfArzs=
-github.com/hexonet/go-sdk v2.2.3+incompatible/go.mod h1:B0oC4YZT3P2o0DHTm5SH0WCItW3N+r16nCTOykJZF1c=
+github.com/hexonet/go-sdk v3.5.0+incompatible/go.mod h1:B0oC4YZT3P2o0DHTm5SH0WCItW3N+r16nCTOykJZF1c=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
 github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=

integrationTest/integration_test.go

@@ -686,14 +686,14 @@ func makeTests(t *testing.T) []*TestGroup {
 		),
 
 		testgroup("ws TXT",
-			not("CLOUDFLAREAPI", "INWX", "NAMEDOTCOM"),
+			not("CLOUDFLAREAPI", "HEXONET", "INWX", "NAMEDOTCOM"),
 			// These providers strip whitespace at the end of TXT records.
 			// TODO(tal): Add a check for this in normalize/validate.go
 			tc("Change a TXT with ws at end", txt("foo", "with space at end  ")),
 		),
 
 		testgroup("empty TXT",
-			not("CLOUDFLAREAPI", "INWX", "NETCUP"),
+			not("CLOUDFLAREAPI", "HEXONET", "INWX", "NETCUP"),
 			tc("TXT with empty str", txt("foo1", "")),
 			// https://github.com/StackExchange/dnscontrol/issues/598
 			// We decided that permitting the TXT target to be an empty
@@ -752,7 +752,8 @@ func makeTests(t *testing.T) []*TestGroup {
 
 		testgroup("pager601",
 			// AWS:  https://github.com/StackExchange/dnscontrol/issues/493
-			only("ROUTE53"),
+			//only("AZURE_DNS", "HEXONET", "ROUTE53"),
+			only("HEXONET", "ROUTE53"), // AZURE_DNS is failing.
 			tc("601 records", manyA("rec%04d", "1.2.3.4", 600)...),
 			tc("Update 601 records", manyA("rec%04d", "1.2.3.5", 600)...),
 		),
@@ -760,8 +761,8 @@ func makeTests(t *testing.T) []*TestGroup {
 		testgroup("pager1201",
 			// AWS:  https://github.com/StackExchange/dnscontrol/issues/493
 			// Azure: https://github.com/StackExchange/dnscontrol/issues/770
-			//only("ROUTE53", "AZURE_DNS"),
+			//only("AZURE_DNS", "HEXONET", "ROUTE53"),
-			only("ROUTE53"), // Azure is failing ATM.
+			only("HEXONET", "ROUTE53"), // AZURE_DNS is failing.
 			tc("1200 records", manyA("rec%04d", "1.2.3.4", 1200)...),
 			tc("Update 1200 records", manyA("rec%04d", "1.2.3.5", 1200)...),
 		),

integrationTest/providers.json

@@ -4,11 +4,11 @@
     "domain": "$AD_DOMAIN"
   },
   "AXFRDDNS": {
+    "domain": "$AXFRDDNS_DOMAIN",
     "master": "$AXFRDDNS_MASTER",
     "nameservers": "ns.example.com",
     "transfer-key": "$AXFRDDNS_TRANSFER_KEY",
-    "update-key": "$AXFRDDNS_UPDATE_KEY",
+    "update-key": "$AXFRDDNS_UPDATE_KEY"
-    "domain": "$AXFRDDNS_DOMAIN"
   },
   "AZURE_DNS": {
     "ClientID": "$AZURE_CLIENT_ID",
@@ -63,20 +63,26 @@
     "type": "$GCLOUD_TYPE"
   },
   "HEDNS": {
-    "username": "$HEDNS_USERNAME",
+    "domain": "$HEDNS_DOMAIN",
     "password": "$HEDNS_PASSWORD",
-    "totp-key": "$HEDNS_TOTP_SECRET",
     "session-file-path": ".",
-    "domain": "$HEDNS_DOMAIN"
+    "totp-key": "$HEDNS_TOTP_SECRET",
+    "username": "$HEDNS_USERNAME"
   },
   "HEXONET": {
     "apientity": "$HEXONET_ENTITY",
     "apilogin": "$HEXONET_UID",
     "apipassword": "$HEXONET_PW",
     "debugmode": "$HEXONET_DEBUGMODE",
-    "domain": "dnscontrol.com",
+    "domain": "$HEXONET_DOMAIN",
     "ipaddress": "$HEXONET_IP"
   },
+  "INWX": {
+    "domain": "$INWX_DOMAIN",
+    "password": "$INWX_PASSWORD",
+    "sandbox": "1",
+    "username": "$INWX_USER"
+  },
   "LINODE": {
     "domain": "$LINODE_DOMAIN",
     "token": "$LINODE_TOKEN"
@@ -130,11 +136,5 @@
   "VULTR": {
     "domain": "$VULTR_DOMAIN",
     "token": "$VULTR_TOKEN"
-  },
-  "INWX": {
-    "username":  "$INWX_USER",
-    "password": "$INWX_PASSWORD",
-    "domain": "$INWX_DOMAIN",
-    "sandbox": "1",
   }
 }

main.go

@@ -5,10 +5,9 @@ import (
 	"log"
 	"os"
 	"runtime/debug"
-	"strconv"
-	"time"
 
 	"github.com/StackExchange/dnscontrol/v3/commands"
+	"github.com/StackExchange/dnscontrol/v3/pkg/version"
 	_ "github.com/StackExchange/dnscontrol/v3/providers/_all"
 )
 
@@ -19,36 +18,5 @@ func main() {
 	if info, ok := debug.ReadBuildInfo(); !ok && info == nil {
 		fmt.Fprint(os.Stderr, "Warning: dnscontrol was built without Go modules. See https://github.com/StackExchange/dnscontrol#from-source for more information on how to build dnscontrol correctly.\n\n")
 	}
-	os.Exit(commands.Run(versionString()))
+	os.Exit(commands.Run("dnscontrol " + version.VersionString()))
-}
-
-// Version management. Goals:
-// 1. Someone who just does "go get" has at least some information.
-// 2. If built with build/build.go, more specific build information gets put in.
-// Update the number here manually each release, so at least we have a range for go-get people.
-var (
-	SHA       = ""
-	Version   = "3.3.0"
-	BuildTime = ""
-)
-
-// printVersion prints the version banner.
-func versionString() string {
-	var version string
-	if SHA != "" {
-		version = fmt.Sprintf("%s (%s)", Version, SHA)
-	} else {
-		version = fmt.Sprintf("%s-dev", Version) // no SHA. '0.x.y-dev' indicates it is run from source without build script.
-	}
-	if info, ok := debug.ReadBuildInfo(); !ok && info == nil {
-		version += " (non-modules)"
-	}
-	if BuildTime != "" {
-		i, err := strconv.ParseInt(BuildTime, 10, 64)
-		if err == nil {
-			tm := time.Unix(i, 0)
-			version += fmt.Sprintf(" built %s", tm.Format(time.RFC822))
-		}
-	}
-	return fmt.Sprintf("dnscontrol %s", version)
 }

pkg/version/version.go

@@ -0,0 +1,47 @@
+package version
+
+import (
+	"fmt"
+	"runtime/debug"
+	"strconv"
+	"time"
+)
+
+// Version management. Goals:
+// 1. Someone who just does "go get" has at least some information.
+// 2. If built with build/build.go, more specific build information gets put in.
+// Update the number here manually each release, so at least we have a range for go-get people.
+var (
+	SHA       = ""
+	Version   = "3.3.0"
+	BuildTime = ""
+)
+
+var versionCache string
+
+// VersionString returns the version banner.
+func VersionString() string {
+	if versionCache != "" {
+		return versionCache
+	}
+
+	var version string
+	if SHA != "" {
+		version = fmt.Sprintf("%s (%s)", Version, SHA)
+	} else {
+		version = fmt.Sprintf("%s-dev", Version) // no SHA. '0.x.y-dev' indicates it is run from source without build script.
+	}
+	if info, ok := debug.ReadBuildInfo(); !ok && info == nil {
+		version += " (non-modules)"
+	}
+	if BuildTime != "" {
+		i, err := strconv.ParseInt(BuildTime, 10, 64)
+		if err == nil {
+			tm := time.Unix(i, 0)
+			version += fmt.Sprintf(" built %s", tm.Format(time.RFC822))
+		}
+	}
+
+	versionCache = version
+	return version
+}

providers/hexonet/domains.go

@@ -1,16 +1,18 @@
 package hexonet
 
+import "fmt"
+
 //EnsureDomainExists returns an error
 // * if access to dnszone is not allowed (not authorized) or
 // * if it doesn't exist and creating it fails
 func (n *HXClient) EnsureDomainExists(domain string) error {
-	r := n.client.Request(map[string]string{
+	r := n.client.Request(map[string]interface{}{
 		"COMMAND": "StatusDNSZone",
 		"DNSZONE": domain + ".",
 	})
 	code := r.GetCode()
 	if code == 545 {
-		r = n.client.Request(map[string]string{
+		r = n.client.Request(map[string]interface{}{
 			"COMMAND": "CreateDNSZone",
 			"DNSZONE": domain + ".",
 		})
@@ -24,3 +26,44 @@ func (n *HXClient) EnsureDomainExists(domain string) error {
 	}
 	return nil
 }
+
+// ListZones lists all the
+func (n *HXClient) ListZones() ([]string, error) {
+	var zones []string
+
+	// Basic
+
+	rs := n.client.RequestAllResponsePages(map[string]string{
+		"COMMAND": "QueryDNSZoneList",
+	})
+	for _, r := range rs {
+		if r.IsError() {
+			return nil, n.GetHXApiError("Error while QueryDNSZoneList", "Basic", &r)
+		}
+		zoneColumn := r.GetColumn("DNSZONE")
+		if zoneColumn == nil {
+			return nil, fmt.Errorf("failed getting DNSZONE BASIC column")
+		}
+		zones = append(zones, zoneColumn.GetData()...)
+	}
+
+	// Premium
+
+	rs = n.client.RequestAllResponsePages(map[string]string{
+		"COMMAND":         "QueryDNSZoneList",
+		"PROPERTIES":      "PREMIUMDNS",
+		"PREMIUMDNSCLASS": "*",
+	})
+	for _, r := range rs {
+		if r.IsError() {
+			return nil, n.GetHXApiError("Error while QueryDNSZoneList", "Basic", &r)
+		}
+		zoneColumn := r.GetColumn("DNSZONE")
+		if zoneColumn == nil {
+			return nil, fmt.Errorf("failed getting DNSZONE PREMIUM column")
+		}
+		zones = append(zones, zoneColumn.GetData()...)
+	}
+
+	return zones, nil
+}

providers/hexonet/hexonetProvider.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/StackExchange/dnscontrol/v3/pkg/version"
 	"github.com/StackExchange/dnscontrol/v3/providers"
 	hxcl "github.com/hexonet/go-sdk/apiclient"
 )
@@ -36,6 +37,7 @@ func newProvider(conf map[string]string) (*HXClient, error) {
 	api := &HXClient{
 		client: hxcl.NewAPIClient(),
 	}
+	api.client.SetUserAgent("DNSControl", version.VersionString())
 	api.APILogin, api.APIPassword, api.APIEntity = conf["apilogin"], conf["apipassword"], conf["apientity"]
 	if conf["debugmode"] == "1" {
 		api.client.EnableDebugMode()

providers/hexonet/nameservers.go

@@ -41,7 +41,7 @@ func (n *HXClient) GetNameservers(domain string) ([]*models.Nameserver, error) {
 }
 
 func (n *HXClient) getNameserversRaw(domain string) ([]string, error) {
-	r := n.client.Request(map[string]string{
+	r := n.client.Request(map[string]interface{}{
 		"COMMAND": "StatusDomain",
 		"DOMAIN":  domain,
 	})
@@ -87,7 +87,7 @@ func (n *HXClient) GetRegistrarCorrections(dc *models.DomainConfig) ([]*models.C
 
 func (n *HXClient) updateNameservers(ns []string, domain string) func() error {
 	return func() error {
-		cmd := map[string]string{
+		cmd := map[string]interface{}{
 			"COMMAND": "ModifyDomain",
 			"DOMAIN":  domain,
 		}

providers/hexonet/records.go

@@ -37,30 +37,34 @@ type HXRecord struct {
 
 // GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
 func (n *HXClient) GetZoneRecords(domain string) (models.Records, error) {
-	return nil, fmt.Errorf("not implemented")
+	records, err := n.getRecords(domain)
-	// This enables the get-zones subcommand.
-	// Implement this by extracting the code from GetDomainCorrections into
-	// a single function.  For most providers this should be relatively easy.
-}
-
-// GetDomainCorrections gathers correctios that would bring n to match dc.
-func (n *HXClient) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
-	dc.Punycode()
-	records, err := n.getRecords(dc.Name)
 	if err != nil {
 		return nil, err
 	}
 	actual := make([]*models.RecordConfig, len(records))
 	for i, r := range records {
-		actual[i] = toRecord(r, dc.Name)
+		actual[i] = toRecord(r, domain)
 	}
 
-	for _, rec := range dc.Records {
+	for _, rec := range actual {
 		if rec.Type == "ALIAS" {
 			return nil, fmt.Errorf("we support realtime ALIAS RR over our X-DNS service, please get in touch with us")
 		}
 	}
 
+	return actual, nil
+
+}
+
+// GetDomainCorrections gathers correctios that would bring n to match dc.
+func (n *HXClient) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
+	dc.Punycode()
+
+	actual, err := n.GetZoneRecords(dc.Name)
+	if err != nil {
+		return nil, err
+	}
+
 	//checkNSModifications(dc)
 
 	// Normalize
@@ -77,7 +81,7 @@ func (n *HXClient) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Corr
 	buf := &bytes.Buffer{}
 	// Print a list of changes. Generate an actual change that is the zone
 	changes := false
-	params := map[string]string{}
+	params := map[string]interface{}{}
 	delrridx := 0
 	addrridx := 0
 	for _, cre := range create {
@@ -161,9 +165,9 @@ func (n *HXClient) showCommand(cmd map[string]string) {
 	fmt.Print(string(b))
 }
 
-func (n *HXClient) updateZoneBy(params map[string]string, domain string) error {
+func (n *HXClient) updateZoneBy(params map[string]interface{}, domain string) error {
 	zone := domain + "."
-	cmd := map[string]string{
+	cmd := map[string]interface{}{
 		"COMMAND":   "UpdateDNSZone",
 		"DNSZONE":   zone,
 		"INCSERIAL": "1",
@@ -182,7 +186,7 @@ func (n *HXClient) updateZoneBy(params map[string]string, domain string) error {
 func (n *HXClient) getRecords(domain string) ([]*HXRecord, error) {
 	var records []*HXRecord
 	zone := domain + "."
-	cmd := map[string]string{
+	cmd := map[string]interface{}{
 		"COMMAND":  "QueryDNSZoneRRList",
 		"DNSZONE":  zone,
 		"SHORT":    "1",