moving a bunch of packages under pkg (#124)

* moving a bunch of packages under pkg

* fix gen

* fix tst

* oops

* fix test fo real

* parse mx/a

build/generate/generate.go

@@ -5,14 +5,13 @@ import (
 )
 
 func main() {
-	//go:generate esc -modtime 0 -o js/static.go -pkg js -include helpers\.js -ignore go -prefix js js
 	conf := &embed.Config{
 		ModTime:    "0",
-		OutputFile: "js/static.go",
+		OutputFile: "pkg/js/static.go",
 		Package:    "js",
-		Prefix:     "js",
+		Prefix:     "pkg/js",
 		Private:    true,
-		Files:      []string{`js/helpers.js`},
+		Files:      []string{`pkg/js/helpers.js`},
 	}
 	embed.Run(conf)
 }

cmd/spftest/main.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/StackExchange/dnscontrol/dnsresolver"
+	"github.com/StackExchange/dnscontrol/pkg/dnsresolver"
-	"github.com/StackExchange/dnscontrol/spflib"
+	"github.com/StackExchange/dnscontrol/pkg/spflib"
 )
 
 func main() {
@@ -27,9 +27,11 @@ func main() {
 	fmt.Println("---------------------")
 	fmt.Println()
 
-	res := dnsresolver.NewResolverLive("preload-dns.json")
+	//res := dnsresolver.NewResolverLive("spf-store2.json")
-	//res := dnsresolver.NewResolverPreloaded("preload-dns.json")
+	res, err := dnsresolver.NewResolverPreloaded("spf-store2.json")
-
+	if err != nil {
+		panic(err)
+	}
 	rec, err := spflib.Parse(strings.Join([]string{"v=spf1",
 		"ip4:198.252.206.0/24",
 		"ip4:192.111.0.0/24",
@@ -50,15 +52,15 @@ func main() {
 	fmt.Println("---------------------")
 	fmt.Println()
 
-	var spfs []string
+	var spf string
-	spfs, err = spflib.Lookup("stackex.com", res)
+	spf, err = spflib.Lookup("whatexit.org", res)
 	if err != nil {
 		panic(err)
 	}
-	rec, err = spflib.Parse(strings.Join(spfs, " "), res)
+	rec, err = spflib.Parse(spf, res)
 	if err != nil {
 		panic(err)
 	}
 	spflib.DumpSPF(rec, "")
-
+	//res.Close()
 }

cmd/spftest/spf-store.json

@@ -0,0 +1,12 @@
+{
+  "_spf.google.com": {
+    "txt": [
+      "v=spf1 include:_netblocks.google.com include:_netblocks2.google.com include:_netblocks3.google.com ~all"
+    ]
+  },
+  "spf-basic.fogcreek.com": {
+    "txt": [
+      "v=spf1 ip4:64.34.80.172 -all"
+    ]
+  }
+}

cmd/spftest/spf-store2.json

@@ -0,0 +1,69 @@
+{
+  "_netblocks.google.com": {
+    "txt": [
+      "v=spf1 ip4:64.18.0.0/20 ip4:64.233.160.0/19 ip4:66.102.0.0/20 ip4:66.249.80.0/20 ip4:72.14.192.0/18 ip4:74.125.0.0/16 ip4:108.177.8.0/21 ip4:173.194.0.0/16 ip4:207.126.144.0/20 ip4:209.85.128.0/17 ip4:216.58.192.0/19 ip4:216.239.32.0/19 ~all"
+    ]
+  },
+  "_netblocks2.google.com": {
+    "txt": [
+      "v=spf1 ip6:2001:4860:4000::/36 ip6:2404:6800:4000::/36 ip6:2607:f8b0:4000::/36 ip6:2800:3f0:4000::/36 ip6:2a00:1450:4000::/36 ip6:2c0f:fb50:4000::/36 ~all"
+    ]
+  },
+  "_netblocks3.google.com": {
+    "txt": [
+      "v=spf1 ip4:172.217.0.0/19 ip4:108.177.96.0/19 ~all"
+    ]
+  },
+  "_spf.google.com": {
+    "txt": [
+      "v=spf1 include:_netblocks.google.com include:_netblocks2.google.com include:_netblocks3.google.com ~all"
+    ]
+  },
+  "mail.zendesk.com": {
+    "txt": [
+      "v=spf1 ip4:192.161.144.0/20 ip4:185.12.80.0/22 ip4:96.46.150.192/27 ip4:174.137.46.0/24 ip4:188.172.128.0/20 ip4:216.198.0.0/18 ~all"
+    ]
+  },
+  "mailgun.org": {
+    "txt": [
+      "google-site-verification=FIGVOKZm6lQFDBJaiC2DdwvBy8TInunoGCt-1gnL4PA",
+      "v=spf1 include:spf1.mailgun.org include:spf2.mailgun.org ~all"
+    ]
+  },
+  "sendgrid.net": {
+    "txt": [
+      "v=spf1 ip4:167.89.0.0/17 ip4:208.117.48.0/20 ip4:50.31.32.0/19 ip4:198.37.144.0/20 ip4:198.21.0.0/21 ip4:192.254.112.0/20 ip4:168.245.0.0/17 ~all",
+      "google-site-verification=NxyooVvVaIgddVa23KTlOEuVPuhffcDqJFV8RzWrAys"
+    ]
+  },
+  "servers.mcsv.net": {
+    "txt": [
+      "v=spf1 ip4:205.201.128.0/20 ip4:198.2.128.0/18 ip4:148.105.8.0/21 ?all"
+    ]
+  },
+  "spf-basic.fogcreek.com": {
+    "txt": [
+      "v=spf1 ip4:64.34.80.172 -all"
+    ]
+  },
+  "spf.mtasv.net": {
+    "txt": [
+      "v=spf1 ip4:50.31.156.96/27 ip4:104.245.209.192/26 ~all"
+    ]
+  },
+  "spf1.mailgun.org": {
+    "txt": [
+      "v=spf1 ip4:173.193.210.32/27 ip4:50.23.218.192/27 ip4:174.37.226.64/27 ip4:208.43.239.136/30 ip4:184.173.105.0/24 ip4:184.173.153.0/24 ip4:104.130.122.0/23 ip4:146.20.112.0/26 ~all"
+    ]
+  },
+  "spf2.mailgun.org": {
+    "txt": [
+      "v=spf1 ip4:209.61.151.0/24 ip4:166.78.68.0/22 ip4:198.61.254.0/23 ip4:192.237.158.0/23 ip4:23.253.182.0/23 ip4:104.130.96.0/28 ip4:146.20.113.0/24 ip4:146.20.191.0/24 ~all"
+    ]
+  },
+  "whatexit.org": {
+    "txt": [
+      "v=spf1 ip6:2607:f2f8:a9c0::3 ip4:174.136.107.195 include:servers.mcsv.net include:_spf.google.com mx:evite.com -all"
+    ]
+  }
+}

integrationTest/integration_test.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 
 	"github.com/StackExchange/dnscontrol/models"
-	"github.com/StackExchange/dnscontrol/nameservers"
+	"github.com/StackExchange/dnscontrol/pkg/nameservers"
 	"github.com/StackExchange/dnscontrol/providers"
 	_ "github.com/StackExchange/dnscontrol/providers/_all"
 	"github.com/StackExchange/dnscontrol/providers/config"

js/parse_tests/008-import.js

@@ -1,2 +0,0 @@
-
-require("js/parse_tests/import.js")

main.go

@@ -12,10 +12,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/StackExchange/dnscontrol/js"
 	"github.com/StackExchange/dnscontrol/models"
-	"github.com/StackExchange/dnscontrol/nameservers"
+	"github.com/StackExchange/dnscontrol/pkg/js"
-	"github.com/StackExchange/dnscontrol/normalize"
+	"github.com/StackExchange/dnscontrol/pkg/nameservers"
+	"github.com/StackExchange/dnscontrol/pkg/normalize"
 	"github.com/StackExchange/dnscontrol/providers"
 	_ "github.com/StackExchange/dnscontrol/providers/_all"
 	"github.com/StackExchange/dnscontrol/providers/config"

models/dns.go

@@ -10,7 +10,7 @@ import (
 	"reflect"
 	"strconv"
 
-	"github.com/StackExchange/dnscontrol/transform"
+	"github.com/StackExchange/dnscontrol/pkg/transform"
 	"github.com/miekg/dns"
 	"golang.org/x/net/idna"
 )

pkg/js/js_test.go

@@ -12,12 +12,12 @@ import (
 )
 
 const (
-	testDir  = "js/parse_tests"
+	testDir  = "pkg/js/parse_tests"
-	errorDir = "js/error_tests"
+	errorDir = "pkg/js/error_tests"
 )
 
 func init() {
-	os.Chdir("..") // go up a directory so we helpers.js is in a consistent place.
+	os.Chdir("../..") // go up a directory so we helpers.js is in a consistent place.
 }
 
 func TestParsedFiles(t *testing.T) {

pkg/js/parse_tests/008-import.js

@@ -0,0 +1,2 @@
+
+require("pkg/js/parse_tests/import.js")

pkg/js/static.go

@@ -189,7 +189,7 @@ func _escFSMustString(useLocal bool, name string) string {
 var _escData = map[string]*_escFile{
 
 	"/helpers.js": {
-		local:   "js/helpers.js",
+		local:   "pkg/js/helpers.js",
 		size:    8320,
 		modtime: 0,
 		compressed: `
@@ -240,6 +240,6 @@ juGPPwSZXUc736wWxSGRz1JGA/lBJGQc4pypseoabTAEaRwj1nqyQmvwU+ljuaLdYjToWa7VE3qVnY+p
 
 	"/": {
 		isDir: true,
-		local: "js",
+		local: "pkg/js",
 	},
 }

pkg/normalize/validate.go

@@ -6,8 +6,8 @@ import (
 	"strings"
 
 	"github.com/StackExchange/dnscontrol/models"
+	"github.com/StackExchange/dnscontrol/pkg/transform"
 	"github.com/StackExchange/dnscontrol/providers"
-	"github.com/StackExchange/dnscontrol/transform"
 	"github.com/miekg/dns"
 	"github.com/miekg/dns/dnsutil"
 )

pkg/spflib/parse.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/StackExchange/dnscontrol/dnsresolver"
+	"github.com/StackExchange/dnscontrol/pkg/dnsresolver"
 )
 
 type SPFRecord struct {
@@ -18,10 +18,10 @@ type SPFPart struct {
 	IncludeRecord *SPFRecord
 }
 
-func Lookup(target string, dnsres dnsresolver.DnsResolver) ([]string, error) {
+func Lookup(target string, dnsres dnsresolver.DnsResolver) (string, error) {
 	txts, err := dnsres.GetTxt(target)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	var result []string
 	for _, txt := range txts {
@@ -29,7 +29,20 @@ func Lookup(target string, dnsres dnsresolver.DnsResolver) ([]string, error) {
 			result = append(result, txt)
 		}
 	}
-	return result, nil
+	if len(result) == 0 {
+		return "", fmt.Errorf("%s has no spf TXT records", target)
+	}
+	if len(result) != 1 {
+		return "", fmt.Errorf("%s has multiple spf TXT records", target)
+	}
+	return result[0], nil
+}
+
+var qualifiers = map[byte]bool{
+	'?': true,
+	'~': true,
+	'-': true,
+	'+': true,
 }
 
 func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) {
@@ -40,17 +53,23 @@ func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) {
 	rec := &SPFRecord{}
 	for _, part := range parts[1:] {
 		p := &SPFPart{Text: part}
+		if qualifiers[part[0]] {
+			part = part[1:]
+		}
 		rec.Parts = append(rec.Parts, p)
-		if part == "~all" || part == "-all" || part == "?all" {
+		if part == "all" {
 			//all. nothing else matters.
 			break
+		} else if strings.HasPrefix(part, "a") || strings.HasPrefix(part, "mx") {
+			rec.Lookups++
+			p.Lookups = 1
 		} else if strings.HasPrefix(part, "ip4:") || strings.HasPrefix(part, "ip6:") {
 			//ip address, 0 lookups
 			continue
 		} else if strings.HasPrefix(part, "include:") {
 			rec.Lookups++
 			includeTarget := strings.TrimPrefix(part, "include:")
-			subRecord, err := resolveSPF(includeTarget, dnsres)
+			subRecord, err := Lookup(includeTarget, dnsres)
 			if err != nil {
 				return nil, err
 			}
@@ -59,6 +78,7 @@ func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) {
 				return nil, fmt.Errorf("In included spf: %s", err)
 			}
 			rec.Lookups += p.IncludeRecord.Lookups
+			p.Lookups = p.IncludeRecord.Lookups + 1
 		} else {
 			return nil, fmt.Errorf("Unsupported spf part %s", part)
 		}
@@ -77,22 +97,11 @@ func DumpSPF(rec *SPFRecord, indent string) {
 	fmt.Println()
 	indent += "\t"
 	for _, p := range rec.Parts {
-		if p.IncludeRecord != nil {
+		if p.Lookups > 0 {
 			fmt.Println(indent + p.Text)
+		}
+		if p.IncludeRecord != nil {
 			DumpSPF(p.IncludeRecord, indent+"\t")
 		}
 	}
 }
-
-func resolveSPF(target string, dnsres dnsresolver.DnsResolver) (string, error) {
-	recs, err := dnsres.GetTxt(target)
-	if err != nil {
-		return "", err
-	}
-	for _, r := range recs {
-		if strings.HasPrefix(r, "v=spf1 ") {
-			return r, nil
-		}
-	}
-	return "", fmt.Errorf("No SPF records found for %s", target)
-}

pkg/spflib/parse_test.go

@@ -4,7 +4,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/StackExchange/dnscontrol/dnsresolver"
+	"github.com/StackExchange/dnscontrol/pkg/dnsresolver"
 )
 
 func TestParse(t *testing.T) {

providers/cloudflare/cloudflareProvider.go

@@ -9,9 +9,9 @@ import (
 	"time"
 
 	"github.com/StackExchange/dnscontrol/models"
+	"github.com/StackExchange/dnscontrol/pkg/transform"
 	"github.com/StackExchange/dnscontrol/providers"
 	"github.com/StackExchange/dnscontrol/providers/diff"
-	"github.com/StackExchange/dnscontrol/transform"
 	"github.com/miekg/dns/dnsutil"
 )
 

providers/cloudflare/preprocess_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/StackExchange/dnscontrol/models"
-	"github.com/StackExchange/dnscontrol/transform"
+	"github.com/StackExchange/dnscontrol/pkg/transform"
 )
 
 func newDomainConfig() *models.DomainConfig {