From f1a0d6519842adf7ef5fd34a1eba771168cd407a Mon Sep 17 00:00:00 2001 From: Craig Peterson Date: Fri, 19 May 2017 14:15:57 -0400 Subject: [PATCH 1/4] Cloudflare Redirects (#119) * function sig * sig * some custom record infrastructure * CLOUDFLARE REDIRECTS! * comments out * guarding redirects behind provider metadata to manage * catch commas in js to ensure proper encoding. * gen * small fix * revendor otto * docs --- docs/_providers/cloudflare.md | 35 ++++- js/error_tests/01-oldDSP.js | 1 - js/helpers.js | 18 +++ js/js_test.go | 2 + js/parse_tests/011-cfRedirect.js | 4 + js/parse_tests/011-cfRedirect.json | 24 +++ js/static.go | 83 +++++----- normalize/validate.go | 42 ++++- providers/cloudflare/cloudflareProvider.go | 112 +++++++++++--- providers/cloudflare/rest.go | 146 +++++++++++++++++- providers/providers.go | 21 +++ .../robertkrimen/otto/README.markdown | 5 +- .../github.com/robertkrimen/otto/builtin.go | 5 +- vendor/github.com/robertkrimen/otto/otto.go | 1 + .../github.com/robertkrimen/otto/runtime.go | 60 +++++-- .../robertkrimen/otto/value_boolean.go | 3 + .../robertkrimen/otto/value_number.go | 2 +- vendor/vendor.json | 6 +- 18 files changed, 471 insertions(+), 99 deletions(-) delete mode 100644 js/error_tests/01-oldDSP.js create mode 100644 js/parse_tests/011-cfRedirect.js create mode 100644 js/parse_tests/011-cfRedirect.json diff --git a/docs/_providers/cloudflare.md b/docs/_providers/cloudflare.md index c36f99709..9c48021d5 100644 --- a/docs/_providers/cloudflare.md +++ b/docs/_providers/cloudflare.md @@ -22,13 +22,14 @@ username and access token: ## Metadata Record level metadata availible: - * cloudflare_proxy ("on", "off", or "full") + * `cloudflare_proxy` ("on", "off", or "full") Domain level metadata availible: - * cloudflare_proxy_default ("on", "off", or "full") + * `cloudflare_proxy_default` ("on", "off", or "full") Provider level metadata availible: - * ip_conversions + * `ip_conversions` + * `manage_redirects`: set to `true` to manage page-rule based redirects ## Usage @@ -55,3 +56,31 @@ If a domain does not exist in your CloudFlare account, DNSControl will *not* automatically add it. You'll need to do that via the control panel manually or via the command `dnscontrol create-domains` -command. + +## Redirects + +The cloudflare provider can manage Page-Rule based redirects for your domains. Simply use the `CF_REDIRECT` and `CF_TEMP_REDIRECT` functions to make redirects: + +{% highlight js %} + +// chiphacker.com is an alias for electronics.stackexchange.com + +D("chiphacker.com", REG_NAMECOM, DnsProvider(CFLARE), + // must have A records with orange cloud on. Otherwise page rule will never run. + A("@","1.2.3.4", CF_PROXY_ON), + A("www", "1.2.3.4", CF_PROXY_ON) + A("meta", "1.2.3.4", CF_PROXY_ON), + + // 302 for meta subdomain + CF_TEMP_REDIRECT("meta.chiphacker.com/*", "https://electronics.meta.stackexchange.com/$1), + + // 301 all subdomains and preserve path + CF_REDIRECT("*chiphacker.com/*", "https://electronics.stackexchange.com/$2), +); +{%endhighlight%} + +Notice a few details: + +1. We need an A record with cloudflare proxy on, or the page rule will never run. +2. The IP address in those A records may be mostly irrelevant, as cloudflare should handle all requests (assuming some page rule matches). +3. Ordering matters for priority. CF_REDIRECT records will be added in the order they appear in your js. So put catch-alls at the bottom. \ No newline at end of file diff --git a/js/error_tests/01-oldDSP.js b/js/error_tests/01-oldDSP.js deleted file mode 100644 index 4d47e66c4..000000000 --- a/js/error_tests/01-oldDSP.js +++ /dev/null @@ -1 +0,0 @@ -D("foo.com","reg","dsp") \ No newline at end of file diff --git a/js/helpers.js b/js/helpers.js index a87711fbb..8e7720678 100644 --- a/js/helpers.js +++ b/js/helpers.js @@ -303,3 +303,21 @@ function num2dot(num) } return d; } + +// CUSTOM, PROVIDER SPECIFIC RECORD TYPES +function CF_REDIRECT(src, dst) { + return function(d) { + if (src.indexOf(",") !== -1 || dst.indexOf(",") !== -1){ + throw("redirect src and dst must not have commas") + } + addRecord(d,"CF_REDIRECT","@",src+","+dst) + } +} +function CF_TEMP_REDIRECT(src, dst) { + return function(d) { + if (src.indexOf(",") !== -1 || dst.indexOf(",") !== -1){ + throw("redirect src and dst must not have commas") + } + addRecord(d,"CF_TEMP_REDIRECT","@",src+","+dst) + } +} \ No newline at end of file diff --git a/js/js_test.go b/js/js_test.go index f6241520d..17f67a678 100644 --- a/js/js_test.go +++ b/js/js_test.go @@ -72,6 +72,8 @@ func TestErrors(t *testing.T) { {"old dsp style", `D("foo.com","reg","dsp")`}, {"MX no priority", `D("foo.com","reg",MX("@","test."))`}, {"MX reversed", `D("foo.com","reg",MX("@","test.", 5))`}, + {"CF_REDIRECT With comma", `D("foo.com","reg",CF_REDIRECT("foo.com,","baaa"))`}, + {"CF_TEMP_REDIRECT With comma", `D("foo.com","reg",CF_TEMP_REDIRECT("foo.com","baa,a"))`}, } for _, tst := range tests { t.Run(tst.desc, func(t *testing.T) { diff --git a/js/parse_tests/011-cfRedirect.js b/js/parse_tests/011-cfRedirect.js new file mode 100644 index 000000000..9c9dad939 --- /dev/null +++ b/js/parse_tests/011-cfRedirect.js @@ -0,0 +1,4 @@ +D("foo.com","none", + CF_REDIRECT("test.foo.com","https://goo.com/$1"), + CF_TEMP_REDIRECT("test.foo.com","https://goo.com/$1") +); \ No newline at end of file diff --git a/js/parse_tests/011-cfRedirect.json b/js/parse_tests/011-cfRedirect.json new file mode 100644 index 000000000..729e2a0e7 --- /dev/null +++ b/js/parse_tests/011-cfRedirect.json @@ -0,0 +1,24 @@ +{ + "registrars": [], + "dns_providers": [], + "domains": [ + { + "name": "foo.com", + "registrar": "none", + "dnsProviders": {}, + "records": [ + { + "type": "CF_REDIRECT", + "name": "@", + "target": "test.foo.com,https://goo.com/$1" + }, + { + "type": "CF_TEMP_REDIRECT", + "name": "@", + "target": "test.foo.com,https://goo.com/$1" + } + ], + "keepunknown": false + } + ] + } \ No newline at end of file diff --git a/js/static.go b/js/static.go index e3ca35c6d..4e15971dd 100644 --- a/js/static.go +++ b/js/static.go @@ -190,48 +190,51 @@ var _escData = map[string]*_escFile{ "/helpers.js": { local: "js/helpers.js", - size: 7758, + size: 8320, modtime: 0, compressed: ` -H4sIAAAAAAAA/7wZbW/bvPG7f8U9AlZLi6q8tM0GuR7mNemDYokbJO4WwDACRqJttnoDSTlPVji/fTiS -kijJblJgXT+kJnnvd7w7npxSUBCSs0g6o8FgQzhEebaEMXwfAABwumJCcsJFCPOFr/biTNwVPN+wmLa2 -85SwTG0MtoZWTJekTOSErwSMYb4YDQbLMoskyzNgGZOMJOw/1PU0sxbnfdx/IEFXClxvR1q4niBbS5Qp -fbiuWLkZSakvHwvqp1QSz4jDluDipleLhysYj8G5nEy/TC4czWir/qLunK5QGSQXgiKqUEL11wckHqq/ -RkTUPmg0DopSrF1OV97IeEKWPFOEesKfZeLKmMNtOGkelgLgKhXypTqA8XgMw/z+K43k0INXr8AdsuIu -yrMN5YLlmRgCyzQNz3IKbgRtQBjDMucpkXdSujvOvY5pYlH8vGlaTtfWiUXxnHUy+nCmQkIbpravVwe4 -QmzJUgOFzU8j1fctHkc5j0U4X/gYiVdNIOKpibTZ7CKEI19RFJSjJcL5YtsWruB5RIU4I3wl3NQ3wWsb -+/AQLQuURGtI85gtGeU++pJJYAJIEAQtWEM5hIgkCQI9MLk2dG1Awjl5DCsBUKWSC7ahyaMNpYMDXcFX -VLHMZK4MERNJaki8G3cBEx8NdzdtBUwVN65Rb1SfbIEmgtb4ExRqBzJawMW4+aoCsk+7bcf510Vtyhbg -dh/jz0rPHZzvAvqHpFlsRA9QdT/ta2BjyTXPH8D59+R6+mn6e2gkqb2n80aZibIoci5pHIJzANW9hANw -QAes2jd8dVw3emwHg8NDOOvGdAgfOCWSAoGz6Y2hE8AXQUGuKRSEk5RKygUQUYUxkCxG4UTQxGWPsFFQ -3V2tznj/zdKC1k5jMIajEbD3dhIOEpqt5HoE7ODAq63X8qMFPWcL33Lots/gBBkQvipTmsk2dcs5CJ3C -GGrAOVs0Zt1zG5vcpdOQLjAmARkQ44/zj5MvF7MbMGlKAAFBJeTLSvWGM8gcSFEkj+pHksCylCWnVf0K -kN453np1kWXeEH9gSQJRQgkHkj1CwemG5aWADUlKKpCh7UmDVZXYfh3c7atnTWn7UpnCtqlX1UJtl9ns -wt14IdxQqeJwNrtQLHWU6ji0ZNbg7fxcHbrcFoIHUiYwhk2b31mdgltsKx9U7NWeviKWwWzcPTLELUME -TcbviKKFsWqzU9WvKUmp48ORBwiSiQ95mak4OYKUkkxAnGdDCdic5dwUIar9bRWUwEbOclnFHTdEEJ0k -ia1dr1Ew6F7VJFQdQkVWNQllFtMly2g8bO5qAwGvj+3e5zlrWRVzjjIsMJdoWm03TrSIrKhK7qVJoSII -Aq9RysABK+w8hSkNxrCiskZrYtQ/8Z6XlcTxteLrxr4zcfxKGqTstSWdTF4sbA36i+WdTH4s8sWnyY3p -dQlfUfmc3A08aIRfKTwyM9Ib6ToaoAofppPL859QwYL/9SooZj9UARPj7ewn5K+hf730s9vZc7Jf3mph -Cs5yzuTjy3SosKBG6ygTrWn0DauKO8fO7EZylq18wN/TMr3H7rfZX/hNQfXBubwF+kdBIylgHxfHe6HJ -3rzAZKprUsWv4mN1hrY9UTTHB9t5PnRMWpuosYD6JZSOAh8WIvKaxyhpuih4r5GqtZWkVTPqKlQrRe/o -zVoEOm2Z4vebhpizhWKNVd5rN8sNrwMHXteeAeeAHTj4WsESFeWc00iqhtfxrJbWjq3pz2Sm6f8tLU1/ -nJNQ8Mnl+c359b/Or20FbGE7AB2hn6mddu1Xcdd+QitSofl/uyu2mle65CQTuLyT5D4xYw1MSch/Pk/y -hxCOfViz1TqEEx+7/X8QQUN4s/BBH7+tjt+p409XIZwuFpqMeig6x/AEJ/AEb+BpBG/hCd7BE8ATnDoD -7aCEZVQ3ogM7KscYk/AeOkLu6kUVfAHjLmzd2SOAkg7GwIpA/RzVt0gtW5FuvUT1YSfKK1p3QUoKDeLX -/mLe92oSUaYncS5d5m294GvOMtfx7XjHZ+NuwhWm5j7qXRFLKfRIrRYuWorhxg9UU8d95QzNWj1c/88U -NMQtFZUU+5XEp/QY5ua85lkESf7g+f1tDMhm30g/sAysfuvRoAo+M2bLH4wO8ASOh2qgDEZVDWjOR+BU -771Pl1efr2d3s+vJ9Obj5+tLfakSgpbSUdg8Iusr+HIkX8rkRYlBTxsjfNi2ik6XleOD83enJl+bVf/7 -PuxcoWHYzRe2lN524bUKBErbdjinkXmgSZn0fayNePXl+vdz1zKQ3jAKxsE/KS2+ZN+y/CGDMSxJImiV -bD/f9ZDrvT34kpe0lRG7tUH4QhK+q4rsfCwr4JF6L+99KjdtQlU4+68lhGnPBm1XqrFor/IYFphtlybp -qypr2iQiRJlSTI4kjjkVIgA9kpXAZFAniqazck0tsmU3ZJsra2D6w24Mv+/2FHd/afIxHkL74dx0ampo -akatZvq7ewYa04jFFO6JoDHkmR4gV/Cv4WNnEir0JBTf/LqbACLUquoHGtTPO6eeCNuafCpYbbkQPn2E -y9uGsra8ckelWG1w23e9eNLNmIqYPdEE1hwL4eZs0Tp72TAWUpfTyEq88BNTUdDqV9FUpw011BKqMxd9 -BKV7UAPDq1dgDX2bg25NqiW2cFvfGyzUPuK2t1XPdDE99Qa6L4fqWMvcoVR9SWm+Dd06O6yHNKu4QDfu -JNy3QpRnIsc2KF+5zXz5cu9g2fHrubIPjnvzjRUFy1a/eU5XlZ31Nw7MiLj6FBW1P7ZwGo10KmYFNF97 -6iIlYMnzFNZSFuHhoZAk+pZvKF8m+UMQ5ekhOfzr8dG7v7w9Ojw+OT49PcKcvmGkQvhKNkREnBUyIPd5 -KRVOwu454Y+H9wkrTPwFa5la5fXKjXPpDayBNYwhzmUgioRJdxgM21q46t9BPD9aeH8+eXfqHeDieOFZ -q5PW6s3C63xjqtqZMq0YsyWu1PSsHp559odNxdtpfTSsIkm/bRW1PkpWpp3UG+vs/KeTd6c7CtQb7KT/ -pvLK69f6flgjPBQRLolcB8skzznyPEQ9m/CwqMMBDIMhHEC8Y9wXo0n+GwAA//9DCRFRTh4AAA== +H4sIAAAAAAAA/9w5W2/bytHv+hUTAl9EfqLpSxK3oKKiqi0fGLVkQ5ZPXQiCsSZX0ia8YXcpx82Rf3ux +F5JLUoodoOlD8+Bod+c+szOzQytnGBinJOBWv9PZIApBmixhAN87AAAUrwjjFFHmw3zhyr0wYQ8ZTTck +xLXtNEYkkRudraYV4iXKIz6kKwYDmC/6nc4yTwJO0gRIQjhBEfkXth3FrMZ5H/cfSNCUQqy3fSVcS5Ct +IcoEP00LVnaCYuzy5wy7MebI0eKQJdhi0ynFEysYDMAaDyd3wytLMdrKv0J3ildCGUHOB0lUovjyrwuC +uC//ahGF9l6lsZflbG1TvHL62hM8p4kk1BL+PGE32hx2xUnxMBQAW6qQLuUBDAYD6KaPX3DAuw68fw92 +l2QPQZpsMGUkTVgXSKJoOIZTxIZXB4QBLFMaI/7Aub3j3GmYJmTZz5um5nRlnZBlr1knwU/nMiSUYUr7 +OmWAS8SaLCWQX/3UUn3fiuMgpSHz5wtXROJNFYjiVEfabHblw5ErKTJMhSX8+WJbFy6jaYAZO0d0xezY +1cFrGvvwUFgWMArWEKchWRJMXeFLwoEwQJ7n1WA1ZR8CFEUC6InwtaZrAiJK0bNfCCBUyikjGxw9m1Aq +OIQr6ApLlglPpSFCxFEJKe7Gg0fYheZux7WAKeLG1ur1y5Mt4IjhEn8ohNqBLCxgi7j5IgOyTbtux/mX +RWnKGuB2H+NrqecOzg8e/sZxEmrRPaG6G7c1MLH4mqZPYP1jOJ1cTn7ztSSl91TeyBOWZ1lKOQ59sHpQ +3EvogQUqYOW+5qviutJj2+kcHsJ5M6Z9OKMYcQwIzie3mo4HdwwDX2PIEEUx5pgyQKwIY0BJKIRjXhWX +LcJaQXl3lTqD/TdLCVo6jcAAjvpAPptJ2ItwsuLrPpBezymtV/OjAT0nC9dw6LbN4EQwQHSVxzjhdeqG +cwR0DAMoAedkUZl1z22scpdKQ6rA6ASkQbQ/RhfDu6vZLeg0xQABwxzSZaF6xRl4CijLomf5I4pgmfOc +4qJ+eYLeSNx6eZF5WhF/IlEEQYQRBZQ8Q0bxhqQ5gw2KcswEQ9OTGqsose06uNtXr5rS9KU0hWlTp6iF +yi6z2ZW9cXy4xVzG4Wx2JVmqKFVxaMiswOv5uTi0qSkE9TiPYACbOr/zMgXX2BY+KNjLPXVFDIOZuHtk +CGuG8KqM3xBFCWPUZquoXxMUY8uFIwcESMLO0jyRcXIEMUYJgzBNuhxEc5ZSXYSw8rdRUDwTOUl5EXdU +ExHoKIpM7VqNgkZ3iiah6BAKsrJJyJMQL0mCw251VysIODg2e5/XrGVUzLmQYSFyiaJVd+NQiUiyouSO +dQplnuc5lVIaDkhm5imR0mAAK8xLtCpG3RPndVlRGE4lXzt0raHlFtIIyk5d0uHwzcKWoL9Y3uHwxyJf +XQ5vda+L6Arz1+Su4EEh/ErhBTMtvZauoYFQ4WwyHI9+QgUD/terIJn9UAWRGO9nPyF/Cf3rpZ/dz16T +fXyvhMkoSSnhz2/TocCCEq2hTLDGwVdRVey56MxuOSXJygXxe5LHj6L7rfYXblVQXbDG94C/ZTjgDPZx +sZw3muzDG0wmuyZZ/Ao+Rmdo2lOIZrlgOs+FhklLE1UWkL+Y1JGJhwULnOoxiqouCj4rpGJtJGnZjNoS +1UjRO3qzGoFGWyb5vVMQc7KQrEWVd+rNcsWrZ8FB6RmweqRnideKKFFBSikOuGx4Lcdoac3YmvxMZpr8 +19LS5Mc5SQg+HI9uR9PfR1NTAVPYBkBD6Fdqp1n7ZdzVn9CSlK//3+6KreqVzilKmFg+cPQY6bGGSEmC +/3wepU8+HLuwJqu1Dyeu6Pb/hhj24cPCBXX8sTj+JI8vb3w4XSwUGflQtI7hBU7gBT7ASx8+wgt8gheA +Fzi1OspBEUmwakQ7ZlQOREzCZ2gIuasXlfAZDJqwZWcvAKR0MACSefJnv7xFclmLdOMlqg4bUV7QevBi +lCkQt/QXcb4Xk4g8PglTbhNn63hfUpLYlmvGu3g27iZcYCru/dYVMZQSHinVEouaYmLjB6rJ47Zymmap +nlj/xxTUxA0VpRT7lRRP6QHM9XnJM/Oi9Mlx29siIKt9LX3HMLD8rUaDMvj0mC190jrAC1iOUEPIoFVV +gPq8D1bx3rsc31xPZw+z6XBye3E9HatLFSFhKRWF1SOyvIJvR3I5j96UGNS0MRAP21rRabKyXLD+apXk +S7Oqf9+7jSvU9Zv5wpTS2S6cWoEQ0tYdTnGgH2icR20fKyPe3E1/G9mGgdSGVjD0/o5xdpd8TdKnBAaw +RBHDRbK9fmghl3t78DnNcS0jNmsDcxlHdFcV2flYlsB9+V7e+1Su2oSicLZfSwKmPhs0XSnHoq3Ko1mI +bLvUSV9WWd0mIcbyGIvkiMKQYsY8UCNZDoR7ZaKoOitb1yJTdk22urIapj3sFuH33Zzi7i9NrogH33w4 +V52aHJrqUaue/u6egYY4ICGGR8RwCGmiBsgF/AFcNCahTE1CxZtfdROAmFwV/UCFer1z6ilga5NPCass +58PlBYzvK8rK8tIdhWKlwU3fteJJNWMyYvZEExhzLAE3J4va2duGsRDbFAdG4oWfmIqCUr+IpjJtyKEW +k505ayNI3b0SGN6/B2PoWx00a1IpsYFb+95goLYRt62tcqYr0lNroPt2qIa19B2K5ZeU6tvQvbXDeoJm +ERfCjTsJt60QpAlLRRuUruxqvjzeO1i23HKu7IJl334lWUaS1TvHaqqys/6Gnh4RF5+igvrHFoqDvkrF +JIPqa09ZpBgsaRrDmvPMPzxkHAVf0w2myyh98oI0PkSHfz4++vSnj0eHxyfHp6dHIqdvCCoQvqANYgEl +GffQY5pziRORR4ro8+FjRDIdf96ax0Z5vbHDlDsdY2ANAwhT7rEsItzuet26Frb81wvnRwvn/08+nTo9 +sTheOMbqpLb6sHAa35iKdiaPC8ZkKVZyelYOzxzzw6bkbdU+GhaRpN62klobJcnjRuoNVXb+v5NPpzsK +1AfRSf9F5pWDA3U/jBGeEBHGiK+9ZZSmVPA8FHpW4WFQhx50vS70INwx7gt1KMDZ3e3seuzCzfT698vz +0RRub0ZnlxeXZzAdnV1Pz2H2z5vRrTGVuXiYjs4vp6Ozmc1o4ELI3vYcEuZiNPBIEuJv10vZfsK7wQAO +juGPPwSZXUc736wWxSGRz1JGA/lBJGQc4pypseoabTAEaRwj1nqyQmvwU+ljuaLdYjToWa7VE3qVnY+p +/mw0vvmfs0FNqf2G+HcAAAD//xD4Q9mAIAAA `, }, diff --git a/normalize/validate.go b/normalize/validate.go index 2bec1ea9f..a9b4a06b8 100644 --- a/normalize/validate.go +++ b/normalize/validate.go @@ -44,7 +44,7 @@ func checkTarget(target string) error { } // validateRecordTypes list of valid rec.Type values. Returns true if this is a real DNS record type, false means it is a pseudo-type used internally. -func validateRecordTypes(rec *models.RecordConfig, domain string) error { +func validateRecordTypes(rec *models.RecordConfig, domain string, pTypes []string) error { var validTypes = map[string]bool{ "A": true, "AAAA": true, @@ -55,9 +55,22 @@ func validateRecordTypes(rec *models.RecordConfig, domain string) error { "NS": true, "ALIAS": false, } - - if _, ok := validTypes[rec.Type]; !ok { - return fmt.Errorf("Unsupported record type (%v) domain=%v name=%v", rec.Type, domain, rec.Name) + _, ok := validTypes[rec.Type] + if !ok { + cType := providers.GetCustomRecordType(rec.Type) + if cType == nil { + return fmt.Errorf("Unsupported record type (%v) domain=%v name=%v", rec.Type, domain, rec.Name) + } + for _, providerType := range pTypes { + if providerType != cType.Provider { + return fmt.Errorf("Custom record type %s is not compatible with provider type %s", rec.Type, providerType) + } + } + //it is ok. Lets replace the type with real type and add metadata to say we checked it + rec.Metadata["orig_custom_type"] = rec.Type + if cType.RealType != "" { + rec.Type = cType.RealType + } } return nil } @@ -128,6 +141,10 @@ func checkTargets(rec *models.RecordConfig, domain string) (errs []error) { check(checkTarget(target)) case "TXT", "IMPORT_TRANSFORM": default: + if rec.Metadata["orig_custom_type"] != "" { + //it is a valid custom type. We perform no validation on target + return + } errs = append(errs, fmt.Errorf("Unimplemented record type (%v) domain=%v name=%v", rec.Type, domain, rec.Name)) } @@ -207,21 +224,34 @@ type Warning struct { } func NormalizeAndValidateConfig(config *models.DNSConfig) (errs []error) { + ptypeMap := map[string]string{} + for _, p := range config.DNSProviders { + ptypeMap[p.Name] = p.Type + } for _, domain := range config.Domains { + pTypes := []string{} + for p := range domain.DNSProviders { + pType, ok := ptypeMap[p] + if !ok { + errs = append(errs, fmt.Errorf("%s uses undefined DNS provider %s", domain.Name, p)) + } else { + pTypes = append(pTypes, pType) + } + } + // Normalize Nameservers. for _, ns := range domain.Nameservers { ns.Name = dnsutil.AddOrigin(ns.Name, domain.Name) ns.Name = strings.TrimRight(ns.Name, ".") } - // Normalize Records. for _, rec := range domain.Records { if rec.TTL == 0 { rec.TTL = models.DefaultTTL } // Validate the unmodified inputs: - if err := validateRecordTypes(rec, domain.Name); err != nil { + if err := validateRecordTypes(rec, domain.Name, pTypes); err != nil { errs = append(errs, err) } if err := checkLabel(rec.Name, rec.Type, domain.Name); err != nil { diff --git a/providers/cloudflare/cloudflareProvider.go b/providers/cloudflare/cloudflareProvider.go index 895eff201..5794f46e1 100644 --- a/providers/cloudflare/cloudflareProvider.go +++ b/providers/cloudflare/cloudflareProvider.go @@ -33,13 +33,20 @@ Domain level metadata available: - ip_conversions */ +func init() { + providers.RegisterDomainServiceProviderType("CLOUDFLAREAPI", newCloudflare, providers.CanUseAlias) + providers.RegisterCustomRecordType("CF_REDIRECT", "CLOUDFLAREAPI", "") + providers.RegisterCustomRecordType("CF_TEMP_REDIRECT", "CLOUDFLAREAPI", "") +} + type CloudflareApi struct { - ApiKey string `json:"apikey"` - ApiUser string `json:"apiuser"` - domainIndex map[string]string - nameservers map[string][]string - ipConversions []transform.IpConversion - ignoredLabels []string + ApiKey string `json:"apikey"` + ApiUser string `json:"apiuser"` + domainIndex map[string]string + nameservers map[string][]string + ipConversions []transform.IpConversion + ignoredLabels []string + manageRedirects bool } func labelMatches(label string, matches []string) bool { @@ -51,6 +58,7 @@ func labelMatches(label string, matches []string) bool { } return false } + func (c *CloudflareApi) GetNameservers(domain string) ([]*models.Nameserver, error) { if c.domainIndex == nil { if err := c.fetchDomainList(); err != nil { @@ -89,6 +97,13 @@ func (c *CloudflareApi) GetDomainCorrections(dc *models.DomainConfig) ([]*models records = append(records[:i], records[i+1:]...) } } + if c.manageRedirects { + prs, err := c.getPageRules(id, dc.Name) + if err != nil { + return nil, err + } + records = append(records, prs...) + } for _, rec := range dc.Records { if rec.Type == "ALIAS" { rec.Type = "CNAME" @@ -103,19 +118,45 @@ func (c *CloudflareApi) GetDomainCorrections(dc *models.DomainConfig) ([]*models corrections := []*models.Correction{} for _, d := range del { - corrections = append(corrections, c.deleteRec(d.Existing.Original.(*cfRecord), id)) + ex := d.Existing + if ex.Type == "PAGE_RULE" { + corrections = append(corrections, &models.Correction{ + Msg: d.String(), + F: func() error { return c.deletePageRule(ex.Original.(*pageRule).ID, id) }, + }) + + } else { + corrections = append(corrections, c.deleteRec(ex.Original.(*cfRecord), id)) + } } for _, d := range create { - corrections = append(corrections, c.createRec(d.Desired, id)...) + des := d.Desired + if des.Type == "PAGE_RULE" { + corrections = append(corrections, &models.Correction{ + Msg: d.String(), + F: func() error { return c.createPageRule(id, des.Target) }, + }) + } else { + corrections = append(corrections, c.createRec(des, id)...) + } } for _, d := range mod { - e, rec := d.Existing.Original.(*cfRecord), d.Desired - proxy := e.Proxiable && rec.Metadata[metaProxy] != "off" - corrections = append(corrections, &models.Correction{ - Msg: d.String(), - F: func() error { return c.modifyRecord(id, e.ID, proxy, rec) }, - }) + rec := d.Desired + ex := d.Existing + if rec.Type == "PAGE_RULE" { + corrections = append(corrections, &models.Correction{ + Msg: d.String(), + F: func() error { return c.updatePageRule(ex.Original.(*pageRule).ID, id, rec.Target) }, + }) + } else { + e := ex.Original.(*cfRecord) + proxy := e.Proxiable && rec.Metadata[metaProxy] != "off" + corrections = append(corrections, &models.Correction{ + Msg: d.String(), + F: func() error { return c.modifyRecord(id, e.ID, proxy, rec) }, + }) + } } return corrections, nil } @@ -163,10 +204,14 @@ func (c *CloudflareApi) preprocessConfig(dc *models.DomainConfig) error { } } + currentPrPrio := 1 + // Normalize the proxy setting for each record. // A and CNAMEs: Validate. If null, set to default. // else: Make sure it wasn't set. Set to default. - for _, rec := range dc.Records { + // iterate backwards so first defined page rules have highest priority + for i := len(dc.Records) - 1; i >= 0; i-- { + rec := dc.Records[i] if rec.Metadata == nil { rec.Metadata = map[string]string{} } @@ -193,6 +238,23 @@ func (c *CloudflareApi) preprocessConfig(dc *models.DomainConfig) error { rec.Metadata[metaProxy] = val } } + // CF_REDIRECT record types. Encode target as $FROM,$TO,$PRIO,$CODE + if rec.Type == "CF_REDIRECT" || rec.Type == "CF_TEMP_REDIRECT" { + if !c.manageRedirects { + return fmt.Errorf("you must add 'manage_redirects: true' metadata to cloudflare provider to use CF_REDIRECT records") + } + parts := strings.Split(rec.Target, ",") + if len(parts) != 2 { + return fmt.Errorf("Invalid data specified for cloudflare redirect record") + } + code := 301 + if rec.Type == "CF_TEMP_REDIRECT" { + code = 302 + } + rec.Target = fmt.Sprintf("%s,%d,%d", rec.Target, currentPrPrio, code) + currentPrPrio++ + rec.Type = "PAGE_RULE" + } } // look for ip conversions and transform records @@ -224,7 +286,7 @@ func newCloudflare(m map[string]string, metadata json.RawMessage) (providers.DNS api.ApiUser, api.ApiKey = m["apiuser"], m["apikey"] // check api keys from creds json file if api.ApiKey == "" || api.ApiUser == "" { - return nil, fmt.Errorf("Cloudflare apikey and apiuser must be provided.") + return nil, fmt.Errorf("cloudflare apikey and apiuser must be provided") } err := api.fetchDomainList() @@ -234,30 +296,30 @@ func newCloudflare(m map[string]string, metadata json.RawMessage) (providers.DNS if len(metadata) > 0 { parsedMeta := &struct { - IPConversions string `json:"ip_conversions"` - IgnoredLabels []string `json:"ignored_labels"` + IPConversions string `json:"ip_conversions"` + IgnoredLabels []string `json:"ignored_labels"` + ManageRedirects bool `json:"manage_redirects"` }{} err := json.Unmarshal([]byte(metadata), parsedMeta) if err != nil { return nil, err } + api.manageRedirects = parsedMeta.ManageRedirects // ignored_labels: for _, l := range parsedMeta.IgnoredLabels { api.ignoredLabels = append(api.ignoredLabels, l) } // parse provider level metadata - api.ipConversions, err = transform.DecodeTransformTable(parsedMeta.IPConversions) - if err != nil { - return nil, err + if len(parsedMeta.IPConversions) > 0 { + api.ipConversions, err = transform.DecodeTransformTable(parsedMeta.IPConversions) + if err != nil { + return nil, err + } } } return api, nil } -func init() { - providers.RegisterDomainServiceProviderType("CLOUDFLAREAPI", newCloudflare, providers.CanUseAlias) -} - // Used on the "existing" records. type cfRecord struct { ID string `json:"id"` diff --git a/providers/cloudflare/rest.go b/providers/cloudflare/rest.go index 3a9e58ceb..eed8a92cf 100644 --- a/providers/cloudflare/rest.go +++ b/providers/cloudflare/rest.go @@ -5,15 +5,22 @@ import ( "encoding/json" "fmt" "net/http" + "time" + + "strings" + + "strconv" "github.com/StackExchange/dnscontrol/models" ) const ( - baseURL = "https://api.cloudflare.com/client/v4/" - zonesURL = baseURL + "zones/" - recordsURL = zonesURL + "%s/dns_records/" - singleRecordURL = recordsURL + "%s" + baseURL = "https://api.cloudflare.com/client/v4/" + zonesURL = baseURL + "zones/" + recordsURL = zonesURL + "%s/dns_records/" + pageRulesURL = zonesURL + "%s/pagerules/" + singlePageRuleURL = pageRulesURL + "%s" + singleRecordURL = recordsURL + "%s" ) // get list of domains for account. Cache so the ids can be looked up from domain name @@ -231,6 +238,99 @@ func (c *CloudflareApi) get(endpoint string, target interface{}) error { return decoder.Decode(target) } +func (c *CloudflareApi) getPageRules(id string, domain string) ([]*models.RecordConfig, error) { + url := fmt.Sprintf(pageRulesURL, id) + data := pageRuleResponse{} + if err := c.get(url, &data); err != nil { + return nil, fmt.Errorf("Error fetching page rule list from cloudflare: %s", err) + } + if !data.Success { + return nil, fmt.Errorf("Error fetching page rule list cloudflare: %s", stringifyErrors(data.Errors)) + } + recs := []*models.RecordConfig{} + for _, pr := range data.Result { + // only interested in forwarding rules. Lets be very specific, and skip anything else + if len(pr.Actions) != 1 || len(pr.Targets) != 1 { + continue + } + if pr.Actions[0].ID != "forwarding_url" { + continue + } + err := json.Unmarshal([]byte(pr.Actions[0].Value), &pr.ForwardingInfo) + if err != nil { + return nil, err + } + var thisPr = pr + recs = append(recs, &models.RecordConfig{ + Name: "@", + NameFQDN: domain, + Type: "PAGE_RULE", + //$FROM,$TO,$PRIO,$CODE + Target: fmt.Sprintf("%s,%s,%d,%d", pr.Targets[0].Constraint.Value, pr.ForwardingInfo.URL, pr.Priority, pr.ForwardingInfo.StatusCode), + Original: thisPr, + TTL: 1, + }) + } + return recs, nil +} + +func (c *CloudflareApi) deletePageRule(recordID, domainID string) error { + endpoint := fmt.Sprintf(singlePageRuleURL, domainID, recordID) + req, err := http.NewRequest("DELETE", endpoint, nil) + if err != nil { + return err + } + c.setHeaders(req) + _, err = handleActionResponse(http.DefaultClient.Do(req)) + return err +} + +func (c *CloudflareApi) updatePageRule(recordID, domainID string, target string) error { + if err := c.deletePageRule(recordID, domainID); err != nil { + return err + } + return c.createPageRule(domainID, target) +} + +func (c *CloudflareApi) createPageRule(domainID string, target string) error { + endpoint := fmt.Sprintf(pageRulesURL, domainID) + return c.sendPageRule(endpoint, "POST", target) +} + +func (c *CloudflareApi) sendPageRule(endpoint, method string, data string) error { + //from to priority code + parts := strings.Split(data, ",") + priority, _ := strconv.Atoi(parts[2]) + code, _ := strconv.Atoi(parts[3]) + fwdInfo := &pageRuleFwdInfo{ + StatusCode: code, + URL: parts[1], + } + dat, _ := json.Marshal(fwdInfo) + pr := &pageRule{ + Status: "active", + Priority: priority, + Targets: []pageRuleTarget{ + {Target: "url", Constraint: pageRuleConstraint{Operator: "matches", Value: parts[0]}}, + }, + Actions: []pageRuleAction{ + {ID: "forwarding_url", Value: json.RawMessage(dat)}, + }, + } + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + if err := enc.Encode(pr); err != nil { + return err + } + req, err := http.NewRequest(method, endpoint, buf) + if err != nil { + return err + } + c.setHeaders(req) + _, err = handleActionResponse(http.DefaultClient.Do(req)) + return err +} + func stringifyErrors(errors []interface{}) string { dat, err := json.Marshal(errors) if err != nil { @@ -244,6 +344,7 @@ type recordsResponse struct { Result []*cfRecord `json:"result"` ResultInfo pagingInfo `json:"result_info"` } + type basicResponse struct { Success bool `json:"success"` Errors []interface{} `json:"errors"` @@ -253,6 +354,43 @@ type basicResponse struct { } `json:"result"` } +type pageRuleResponse struct { + basicResponse + Result []*pageRule `json:"result"` + ResultInfo pagingInfo `json:"result_info"` +} + +type pageRule struct { + ID string `json:"id,omitempty"` + Targets []pageRuleTarget `json:"targets"` + Actions []pageRuleAction `json:"actions"` + Priority int `json:"priority"` + Status string `json:"status"` + ModifiedOn time.Time `json:"modified_on,omitempty"` + CreatedOn time.Time `json:"created_on,omitempty"` + ForwardingInfo *pageRuleFwdInfo `json:"-"` +} + +type pageRuleTarget struct { + Target string `json:"target"` + Constraint pageRuleConstraint `json:"constraint"` +} + +type pageRuleConstraint struct { + Operator string `json:"operator"` + Value string `json:"value"` +} + +type pageRuleAction struct { + ID string `json:"id"` + Value json.RawMessage `json:"value"` +} + +type pageRuleFwdInfo struct { + URL string `json:"url"` + StatusCode int `json:"status_code"` +} + type zoneResponse struct { basicResponse Result []struct { diff --git a/providers/providers.go b/providers/providers.go index cf9b3d9cc..6941422f4 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -138,3 +138,24 @@ func init() { return None{}, nil }) } + +type CustomRType struct { + Name string + Provider string + RealType string +} + +// RegisterCustomRecordType registers a record type that is only valid for one provider. +// provider is the registered type of provider this is valid with +// name is the record type as it will appear in the js. (should be something like $PROVIDER_FOO) +// realType is the record type it will be replaced with after validation +func RegisterCustomRecordType(name, provider, realType string) { + customRecordTypes[name] = &CustomRType{Name: name, Provider: provider, RealType: realType} +} + +// GetCustomRecordType returns a registered custom record type, or nil if none +func GetCustomRecordType(rType string) *CustomRType { + return customRecordTypes[rType] +} + +var customRecordTypes = map[string]*CustomRType{} diff --git a/vendor/github.com/robertkrimen/otto/README.markdown b/vendor/github.com/robertkrimen/otto/README.markdown index 30f734ea3..a1ae7d1ae 100644 --- a/vendor/github.com/robertkrimen/otto/README.markdown +++ b/vendor/github.com/robertkrimen/otto/README.markdown @@ -88,7 +88,7 @@ Set a Go function that returns something useful ```go vm.Set("twoPlus", func(call otto.FunctionCall) otto.Value { right, _ := call.Argument(0).ToInteger() - return, _ := vm.ToValue(2 + right) + result, _ := vm.ToValue(2 + right) return result }) ``` @@ -114,7 +114,7 @@ http://godoc.org/github.com/robertkrimen/otto/parser Parse and return an AST ```go -filenamee := "" // A filename is optional +filename := "" // A filename is optional src := ` // Sample xyzzy example (function(){ @@ -167,6 +167,7 @@ The following are some limitations with otto: * "use strict" will parse, but does nothing. * The regular expression engine (re2/regexp) is not fully compatible with the ECMA5 specification. + * Otto targets ES5. ES6 features (eg: Typed Arrays) are not supported. ### Regular Expression Incompatibility diff --git a/vendor/github.com/robertkrimen/otto/builtin.go b/vendor/github.com/robertkrimen/otto/builtin.go index 83f715083..256ee3c55 100644 --- a/vendor/github.com/robertkrimen/otto/builtin.go +++ b/vendor/github.com/robertkrimen/otto/builtin.go @@ -70,7 +70,7 @@ func digitValue(chr rune) int { } func builtinGlobal_parseInt(call FunctionCall) Value { - input := strings.TrimSpace(call.Argument(0).string()) + input := strings.Trim(call.Argument(0).string(), builtinString_trim_whitespace) if len(input) == 0 { return NaNValue() } @@ -153,7 +153,8 @@ var parseFloat_matchValid = regexp.MustCompile(`[0-9eE\+\-\.]|Infinity`) func builtinGlobal_parseFloat(call FunctionCall) Value { // Caveat emptor: This implementation does NOT match the specification - input := strings.TrimSpace(call.Argument(0).string()) + input := strings.Trim(call.Argument(0).string(), builtinString_trim_whitespace) + if parseFloat_matchBadSpecial.MatchString(input) { return NaNValue() } diff --git a/vendor/github.com/robertkrimen/otto/otto.go b/vendor/github.com/robertkrimen/otto/otto.go index 7d9065851..b5b528d53 100644 --- a/vendor/github.com/robertkrimen/otto/otto.go +++ b/vendor/github.com/robertkrimen/otto/otto.go @@ -132,6 +132,7 @@ The following are some limitations with otto: * "use strict" will parse, but does nothing. * The regular expression engine (re2/regexp) is not fully compatible with the ECMA5 specification. + * Otto targets ES5. ES6 features (eg: Typed Arrays) are not supported. Regular Expression Incompatibility diff --git a/vendor/github.com/robertkrimen/otto/runtime.go b/vendor/github.com/robertkrimen/otto/runtime.go index 6ebb1f982..7d29ecca0 100644 --- a/vendor/github.com/robertkrimen/otto/runtime.go +++ b/vendor/github.com/robertkrimen/otto/runtime.go @@ -302,7 +302,11 @@ func (self *_runtime) convertCallParameter(v Value, t reflect.Type) reflect.Valu } if t.Kind() == reflect.Interface { - iv := reflect.ValueOf(v.export()) + e := v.export() + if e == nil { + return reflect.Zero(t) + } + iv := reflect.ValueOf(e) if iv.Type().AssignableTo(t) { return iv } @@ -352,20 +356,52 @@ func (self *_runtime) convertCallParameter(v Value, t reflect.Type) reflect.Valu tt := t.Elem() - for i := int64(0); i < l; i++ { - p, ok := o.property[strconv.FormatInt(i, 10)] - if !ok { - continue + if o.class == "Array" { + for i := int64(0); i < l; i++ { + p, ok := o.property[strconv.FormatInt(i, 10)] + if !ok { + continue + } + + e, ok := p.value.(Value) + if !ok { + continue + } + + ev := self.convertCallParameter(e, tt) + + s.Index(int(i)).Set(ev) + } + } else if o.class == "GoArray" { + + var gslice bool + switch o.value.(type) { + case *_goSliceObject: + gslice = true + case *_goArrayObject: + gslice = false } - e, ok := p.value.(Value) - if !ok { - continue + for i := int64(0); i < l; i++ { + var p *_property + if gslice { + p = goSliceGetOwnProperty(o, strconv.FormatInt(i, 10)) + } else { + p = goArrayGetOwnProperty(o, strconv.FormatInt(i, 10)) + } + if p == nil { + continue + } + + e, ok := p.value.(Value) + if !ok { + continue + } + + ev := self.convertCallParameter(e, tt) + + s.Index(int(i)).Set(ev) } - - ev := self.convertCallParameter(e, tt) - - s.Index(int(i)).Set(ev) } return s diff --git a/vendor/github.com/robertkrimen/otto/value_boolean.go b/vendor/github.com/robertkrimen/otto/value_boolean.go index 3040f4163..b631507b0 100644 --- a/vendor/github.com/robertkrimen/otto/value_boolean.go +++ b/vendor/github.com/robertkrimen/otto/value_boolean.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "reflect" + "unicode/utf16" ) func (value Value) bool() bool { @@ -32,6 +33,8 @@ func (value Value) bool() bool { return true case string: return 0 != len(value) + case []uint16: + return 0 != len(utf16.Decode(value)) } if value.IsObject() { return true diff --git a/vendor/github.com/robertkrimen/otto/value_number.go b/vendor/github.com/robertkrimen/otto/value_number.go index 870bf115b..8cbf136d2 100644 --- a/vendor/github.com/robertkrimen/otto/value_number.go +++ b/vendor/github.com/robertkrimen/otto/value_number.go @@ -11,7 +11,7 @@ import ( var stringToNumberParseInteger = regexp.MustCompile(`^(?:0[xX])`) func parseNumber(value string) float64 { - value = strings.TrimSpace(value) + value = strings.Trim(value, builtinString_trim_whitespace) if value == "" { return 0 diff --git a/vendor/vendor.json b/vendor/vendor.json index 78b8eb23e..599669bfb 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -303,10 +303,10 @@ "revisionTime": "2016-04-18T18:49:04Z" }, { - "checksumSHA1": "UH75lsKCrVFdCZvJchkAPo2QXjw=", + "checksumSHA1": "EqyHXBcg5cWi4ERsMXN6g1opi1o=", "path": "github.com/robertkrimen/otto", - "revision": "7d9cbc2befca39869eb0e5bcb0f44c0692c2f8ff", - "revisionTime": "2016-07-28T22:04:12Z" + "revision": "21ec96599b1279b5673e4df0097dd56bb8360068", + "revisionTime": "2017-04-24T10:46:44Z" }, { "checksumSHA1": "qgziiO3/QDVJMKw2nGrUbC8QldY=", From 20253130cf7fef824d2f4109e2715ddf5be2993f Mon Sep 17 00:00:00 2001 From: Craig Peterson Date: Tue, 23 May 2017 13:38:14 -0400 Subject: [PATCH 2/4] allowing metadata to force remove all nameservers --- main.go | 4 +-- providers/namedotcom/namedotcom.md | 48 ------------------------------ 2 files changed, 2 insertions(+), 50 deletions(-) delete mode 100644 providers/namedotcom/namedotcom.md diff --git a/main.go b/main.go index 8665bdbbd..d530cce1f 100644 --- a/main.go +++ b/main.go @@ -207,8 +207,8 @@ func main() { if !ok { log.Fatalf("Registrar %s not declared.", reg) } - if len(domain.Nameservers) == 0 { - //fmt.Printf("No nameservers declared; skipping registrar.\n") + if len(domain.Nameservers) == 0 && domain.Metadata["no_ns"] != "true" { + fmt.Printf("No nameservers declared; skipping registrar. Add {no_ns:'true'} to force.\n") continue } dc, err := domain.Copy() diff --git a/providers/namedotcom/namedotcom.md b/providers/namedotcom/namedotcom.md deleted file mode 100644 index 9988c73eb..000000000 --- a/providers/namedotcom/namedotcom.md +++ /dev/null @@ -1,48 +0,0 @@ -## name.com Provider - -### required config - -In your providers config json file you must provide your name.com api username and access token: - -``` - "yourNameDotComProviderName":{ - "apikey": "yourApiKeyFromName.com-klasjdkljasdlk235235235235", - "apiuser": "yourUsername" - } -``` - -In order to get api access you need to [apply for access](https://www.name.com/reseller/apply) - -### example dns config js (registrar only): - -``` -var NAMECOM = NewRegistrar("myNameCom","NAMEDOTCOM"); - -var mynameServers = [ - NAMESERVER("bill.ns.cloudflare.com"), - NAMESERVER("fred.ns.cloudflare.com") -]; - -D("example.tld",NAMECOM,myNameServers - //records handled by another provider... -); -``` - -### example config (registrar and records managed by namedotcom) - -``` -var NAMECOM = NewRegistrar("myNameCom","NAMEDOTCOM"); -var NAMECOMDSP = NewDSP("myNameCom","NAMEDOTCOM") - -D("exammple.tld", NAMECOM, NAMECOMDSP, - //ns[1-4].name.com used by default as nameservers - - //override default ttl of 300s - DefaultTTL(3600), - - A("test","1.2.3.4"), - - //override ttl for one record only - CNAME("foo","some.otherdomain.tld.",TTL(100)) -) -``` \ No newline at end of file From 01a242480c7b39db1926abb67a2a9847c3d2195d Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Thu, 25 May 2017 12:27:36 -0400 Subject: [PATCH 3/4] Initial DNS Resolvers and SPF scaffolding (#123) * Implemented Live and Preloaded resolvers * Integrated Craig's parser. --- cmd/spftest/main.go | 64 +++++++++++++++++++++++ dnsresolver/dnscache.go | 28 +++++++++++ dnsresolver/dnscache_test.go | 31 ++++++++++++ dnsresolver/resolver.go | 83 ++++++++++++++++++++++++++++++ spflib/parse.go | 98 ++++++++++++++++++++++++++++++++++++ spflib/parse_test.go | 30 +++++++++++ spflib/testdata-dns1.json | 64 +++++++++++++++++++++++ 7 files changed, 398 insertions(+) create mode 100644 cmd/spftest/main.go create mode 100644 dnsresolver/dnscache.go create mode 100644 dnsresolver/dnscache_test.go create mode 100644 dnsresolver/resolver.go create mode 100644 spflib/parse.go create mode 100644 spflib/parse_test.go create mode 100644 spflib/testdata-dns1.json diff --git a/cmd/spftest/main.go b/cmd/spftest/main.go new file mode 100644 index 000000000..df3a86c96 --- /dev/null +++ b/cmd/spftest/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/StackExchange/dnscontrol/dnsresolver" + "github.com/StackExchange/dnscontrol/spflib" +) + +func main() { + + h := dnsresolver.NewResolverLive("spf-store.json") + fmt.Println(h.GetTxt("_spf.google.com")) + fmt.Println(h.GetTxt("spf-basic.fogcreek.com")) + h.Close() + + i, err := dnsresolver.NewResolverPreloaded("spf-store.json") + if err != nil { + panic(err) + } + fmt.Println(i.GetTxt("_spf.google.com")) + fmt.Println(i.GetTxt("spf-basic.fogcreek.com")) + fmt.Println(i.GetTxt("wontbefound")) + + fmt.Println() + fmt.Println("---------------------") + fmt.Println() + + res := dnsresolver.NewResolverLive("preload-dns.json") + //res := dnsresolver.NewResolverPreloaded("preload-dns.json") + + rec, err := spflib.Parse(strings.Join([]string{"v=spf1", + "ip4:198.252.206.0/24", + "ip4:192.111.0.0/24", + "include:_spf.google.com", + "include:mailgun.org", + "include:spf-basic.fogcreek.com", + "include:mail.zendesk.com", + "include:servers.mcsv.net", + "include:sendgrid.net", + "include:spf.mtasv.net", + "~all"}, " "), res) + if err != nil { + panic(err) + } + spflib.DumpSPF(rec, "") + + fmt.Println() + fmt.Println("---------------------") + fmt.Println() + + var spfs []string + spfs, err = spflib.Lookup("stackex.com", res) + if err != nil { + panic(err) + } + rec, err = spflib.Parse(strings.Join(spfs, " "), res) + if err != nil { + panic(err) + } + spflib.DumpSPF(rec, "") + +} diff --git a/dnsresolver/dnscache.go b/dnsresolver/dnscache.go new file mode 100644 index 000000000..edb102d04 --- /dev/null +++ b/dnsresolver/dnscache.go @@ -0,0 +1,28 @@ +package dnsresolver + +// dnsCache implements a very simple DNS cache. +// It caches the entire answer (i.e. all TXT records), filtering +// out the non-SPF answers is done at a higher layer. +// At this time the only rtype is "TXT". Eventually we'll need +// to cache A/AAAA/CNAME records to to CNAME flattening. +type dnsCache map[string]map[string][]string // map[fqdn]map[rtype] -> answers + +func (c dnsCache) get(label, rtype string) ([]string, bool) { + v1, ok := c[label] + if !ok { + return nil, false + } + v2, ok := v1[rtype] + if !ok { + return nil, false + } + return v2, true +} + +func (c dnsCache) put(label, rtype string, answers []string) { + _, ok := c[label] + if !ok { + c[label] = make(map[string][]string) + } + c[label][rtype] = answers +} diff --git a/dnsresolver/dnscache_test.go b/dnsresolver/dnscache_test.go new file mode 100644 index 000000000..e4f32845c --- /dev/null +++ b/dnsresolver/dnscache_test.go @@ -0,0 +1,31 @@ +package dnsresolver + +import "testing" + +func TestDnsCache(t *testing.T) { + + cache := &dnsCache{} + cache.put("one", "txt", []string{"a", "b", "c"}) + cache.put("two", "txt", []string{"d", "e", "f"}) + + a, b := cache.get("one", "txt") + if !(b == true && len(a) == 3 && a[0] == "a" && a[1] == "b" && a[2] == "c") { + t.Errorf("one-txt didn't work") + } + + a, b = cache.get("two", "txt") + if !(b == true && len(a) == 3 && a[0] == "d" && a[1] == "e" && a[2] == "f") { + t.Errorf("one-txt didn't work") + } + + a, b = cache.get("three", "txt") + if !(b == false) { + t.Errorf("three-txt didn't work") + } + + a, b = cache.get("two", "not") + if !(b == false) { + t.Errorf("two-not didn't work") + } + +} diff --git a/dnsresolver/resolver.go b/dnsresolver/resolver.go new file mode 100644 index 000000000..706dda89d --- /dev/null +++ b/dnsresolver/resolver.go @@ -0,0 +1,83 @@ +package dnsresolver + +import ( + "encoding/json" + "io/ioutil" + "net" + + "github.com/pkg/errors" +) + +// This file includes all the DNS Resolvers used by package spf. + +// DnsResolver looks up txt strings associated with a FQDN. +type DnsResolver interface { + GetTxt(string) ([]string, error) // Given a DNS label, return the TXT values records. +} + +// The "Live DNS" Resolver: + +type dnsLive struct { + filename string + cache dnsCache +} + +func NewResolverLive(filename string) *dnsLive { + // Does live DNS lookups. Records them. Writes file on Close. + c := &dnsLive{filename: filename} + c.cache = dnsCache{} + return c +} + +func (c *dnsLive) GetTxt(label string) ([]string, error) { + // Try the cache. + txts, ok := c.cache.get(label, "txt") + if ok { + return txts, nil + } + + // Populate the cache: + t, err := net.LookupTXT(label) + if err == nil { + c.cache.put(label, "txt", t) + } + + return t, err +} + +func (c *dnsLive) Close() { + // Write out and close the file. + m, _ := json.MarshalIndent(c.cache, "", " ") + m = append(m, "\n"...) + ioutil.WriteFile(c.filename, m, 0666) +} + +// The "Pre-Cached DNS" Resolver: + +type dnsPreloaded struct { + cache dnsCache +} + +func NewResolverPreloaded(filename string) (*dnsPreloaded, error) { + c := &dnsPreloaded{} + c.cache = dnsCache{} + j, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + err = json.Unmarshal(j, &(*c).cache) + return c, err +} + +func (c *dnsPreloaded) DumpCache() dnsCache { + return c.cache +} + +func (c *dnsPreloaded) GetTxt(label string) ([]string, error) { + // Try the cache. + txts, ok := c.cache.get(label, "txt") + if ok { + return txts, nil + } + return nil, errors.Errorf("No preloaded DNS entry for: %#v", label) +} diff --git a/spflib/parse.go b/spflib/parse.go new file mode 100644 index 000000000..37cb61a4c --- /dev/null +++ b/spflib/parse.go @@ -0,0 +1,98 @@ +package spflib + +import ( + "fmt" + "strings" + + "github.com/StackExchange/dnscontrol/dnsresolver" +) + +type SPFRecord struct { + Lookups int + Parts []*SPFPart +} + +type SPFPart struct { + Text string + Lookups int + IncludeRecord *SPFRecord +} + +func Lookup(target string, dnsres dnsresolver.DnsResolver) ([]string, error) { + txts, err := dnsres.GetTxt(target) + if err != nil { + return nil, err + } + var result []string + for _, txt := range txts { + if strings.HasPrefix(txt, "v=spf1 ") { + result = append(result, txt) + } + } + return result, nil +} + +func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) { + if !strings.HasPrefix(text, "v=spf1 ") { + return nil, fmt.Errorf("Not an spf record") + } + parts := strings.Split(text, " ") + rec := &SPFRecord{} + for _, part := range parts[1:] { + p := &SPFPart{Text: part} + rec.Parts = append(rec.Parts, p) + if part == "~all" || part == "-all" || part == "?all" { + //all. nothing else matters. + break + } else if strings.HasPrefix(part, "ip4:") || strings.HasPrefix(part, "ip6:") { + //ip address, 0 lookups + continue + } else if strings.HasPrefix(part, "include:") { + rec.Lookups++ + includeTarget := strings.TrimPrefix(part, "include:") + subRecord, err := resolveSPF(includeTarget, dnsres) + if err != nil { + return nil, err + } + p.IncludeRecord, err = Parse(subRecord, dnsres) + if err != nil { + return nil, fmt.Errorf("In included spf: %s", err) + } + rec.Lookups += p.IncludeRecord.Lookups + } else { + return nil, fmt.Errorf("Unsupported spf part %s", part) + } + + } + return rec, nil +} + +// DumpSPF outputs an SPFRecord and related data for debugging purposes. +func DumpSPF(rec *SPFRecord, indent string) { + fmt.Printf("%sTotal Lookups: %d\n", indent, rec.Lookups) + fmt.Print(indent + "v=spf1") + for _, p := range rec.Parts { + fmt.Print(" " + p.Text) + } + fmt.Println() + indent += "\t" + for _, p := range rec.Parts { + if p.IncludeRecord != nil { + fmt.Println(indent + p.Text) + DumpSPF(p.IncludeRecord, indent+"\t") + } + } +} + +func resolveSPF(target string, dnsres dnsresolver.DnsResolver) (string, error) { + recs, err := dnsres.GetTxt(target) + if err != nil { + return "", err + } + for _, r := range recs { + if strings.HasPrefix(r, "v=spf1 ") { + return r, nil + } + } + return "", fmt.Errorf("No SPF records found for %s", target) +} diff --git a/spflib/parse_test.go b/spflib/parse_test.go new file mode 100644 index 000000000..aee030509 --- /dev/null +++ b/spflib/parse_test.go @@ -0,0 +1,30 @@ +package spflib + +import ( + "strings" + "testing" + + "github.com/StackExchange/dnscontrol/dnsresolver" +) + +func TestParse(t *testing.T) { + dnsres, err := dnsresolver.NewResolverPreloaded("testdata-dns1.json") + if err != nil { + t.Fatal(err) + } + rec, err := Parse(strings.Join([]string{"v=spf1", + "ip4:198.252.206.0/24", + "ip4:192.111.0.0/24", + "include:_spf.google.com", + "include:mailgun.org", + "include:spf-basic.fogcreek.com", + "include:mail.zendesk.com", + "include:servers.mcsv.net", + "include:sendgrid.net", + "include:spf.mtasv.net", + "~all"}, " "), dnsres) + if err != nil { + t.Fatal(err) + } + DumpSPF(rec, "") +} diff --git a/spflib/testdata-dns1.json b/spflib/testdata-dns1.json new file mode 100644 index 000000000..15f8266fe --- /dev/null +++ b/spflib/testdata-dns1.json @@ -0,0 +1,64 @@ +{ + "_netblocks.google.com": { + "txt": [ + "v=spf1 ip4:64.18.0.0/20 ip4:64.233.160.0/19 ip4:66.102.0.0/20 ip4:66.249.80.0/20 ip4:72.14.192.0/18 ip4:74.125.0.0/16 ip4:108.177.8.0/21 ip4:173.194.0.0/16 ip4:207.126.144.0/20 ip4:209.85.128.0/17 ip4:216.58.192.0/19 ip4:216.239.32.0/19 ~all" + ] + }, + "_netblocks2.google.com": { + "txt": [ + "v=spf1 ip6:2001:4860:4000::/36 ip6:2404:6800:4000::/36 ip6:2607:f8b0:4000::/36 ip6:2800:3f0:4000::/36 ip6:2a00:1450:4000::/36 ip6:2c0f:fb50:4000::/36 ~all" + ] + }, + "_netblocks3.google.com": { + "txt": [ + "v=spf1 ip4:172.217.0.0/19 ip4:108.177.96.0/19 ~all" + ] + }, + "_spf.google.com": { + "txt": [ + "v=spf1 include:_netblocks.google.com include:_netblocks2.google.com include:_netblocks3.google.com ~all" + ] + }, + "mail.zendesk.com": { + "txt": [ + "v=spf1 ip4:192.161.144.0/20 ip4:185.12.80.0/22 ip4:96.46.150.192/27 ip4:174.137.46.0/24 ip4:188.172.128.0/20 ip4:216.198.0.0/18 ~all" + ] + }, + "mailgun.org": { + "txt": [ + "google-site-verification=FIGVOKZm6lQFDBJaiC2DdwvBy8TInunoGCt-1gnL4PA", + "v=spf1 include:spf1.mailgun.org include:spf2.mailgun.org ~all" + ] + }, + "sendgrid.net": { + "txt": [ + "google-site-verification=NxyooVvVaIgddVa23KTlOEuVPuhffcDqJFV8RzWrAys", + "v=spf1 ip4:167.89.0.0/17 ip4:208.117.48.0/20 ip4:50.31.32.0/19 ip4:198.37.144.0/20 ip4:198.21.0.0/21 ip4:192.254.112.0/20 ip4:168.245.0.0/17 ~all" + ] + }, + "servers.mcsv.net": { + "txt": [ + "v=spf1 ip4:205.201.128.0/20 ip4:198.2.128.0/18 ip4:148.105.8.0/21 ?all" + ] + }, + "spf-basic.fogcreek.com": { + "txt": [ + "v=spf1 ip4:64.34.80.172 -all" + ] + }, + "spf.mtasv.net": { + "txt": [ + "v=spf1 ip4:50.31.156.96/27 ip4:104.245.209.192/26 ~all" + ] + }, + "spf1.mailgun.org": { + "txt": [ + "v=spf1 ip4:173.193.210.32/27 ip4:50.23.218.192/27 ip4:174.37.226.64/27 ip4:208.43.239.136/30 ip4:184.173.105.0/24 ip4:184.173.153.0/24 ip4:104.130.122.0/23 ip4:146.20.112.0/26 ~all" + ] + }, + "spf2.mailgun.org": { + "txt": [ + "v=spf1 ip4:209.61.151.0/24 ip4:166.78.68.0/22 ip4:198.61.254.0/23 ip4:192.237.158.0/23 ip4:23.253.182.0/23 ip4:104.130.96.0/28 ip4:146.20.113.0/24 ip4:146.20.191.0/24 ~all" + ] + } +} From 42ef9f4b9e9d3b9813dcde7091a5521a053a597a Mon Sep 17 00:00:00 2001 From: Craig Peterson Date: Thu, 25 May 2017 14:25:39 -0400 Subject: [PATCH 4/4] moving a bunch of packages under pkg (#124) * moving a bunch of packages under pkg * fix gen * fix tst * oops * fix test fo real * parse mx/a --- build/generate/generate.go | 7 +- {misc => cmd}/convertzone/README.md | 0 {misc => cmd}/convertzone/main.go | 0 cmd/spftest/main.go | 20 +++--- cmd/spftest/spf-store.json | 12 ++++ cmd/spftest/spf-store2.json | 69 +++++++++++++++++++ integrationTest/integration_test.go | 2 +- js/parse_tests/008-import.js | 2 - main.go | 6 +- models/dns.go | 2 +- {dnsresolver => pkg/dnsresolver}/dnscache.go | 0 .../dnsresolver}/dnscache_test.go | 0 {dnsresolver => pkg/dnsresolver}/resolver.go | 0 {js => pkg/js}/helpers.js | 0 {js => pkg/js}/js.go | 0 {js => pkg/js}/js_test.go | 6 +- {js => pkg/js}/parse_tests/001-basic.js | 0 {js => pkg/js}/parse_tests/001-basic.json | 0 {js => pkg/js}/parse_tests/002-ttl.js | 0 {js => pkg/js}/parse_tests/002-ttl.json | 0 {js => pkg/js}/parse_tests/003-meta.js | 0 {js => pkg/js}/parse_tests/003-meta.json | 0 {js => pkg/js}/parse_tests/004-ips.js | 0 {js => pkg/js}/parse_tests/004-ips.json | 0 .../js}/parse_tests/005-multipleDomains.js | 0 .../js}/parse_tests/005-multipleDomains.json | 0 {js => pkg/js}/parse_tests/006-transforms.js | 0 .../js}/parse_tests/006-transforms.json | 0 .../js}/parse_tests/007-importTransformTTL.js | 0 .../parse_tests/007-importTransformTTL.json | 0 pkg/js/parse_tests/008-import.js | 2 + {js => pkg/js}/parse_tests/008-import.json | 0 {js => pkg/js}/parse_tests/010-alias.js | 0 {js => pkg/js}/parse_tests/010-alias.json | 0 {js => pkg/js}/parse_tests/011-cfRedirect.js | 0 .../js}/parse_tests/011-cfRedirect.json | 0 {js => pkg/js}/parse_tests/import.js | 0 {js => pkg/js}/static.go | 4 +- .../nameservers}/nameservers.go | 0 .../normalize}/importTransform_test.go | 0 {normalize => pkg/normalize}/validate.go | 2 +- {normalize => pkg/normalize}/validate_test.go | 0 {spflib => pkg/spflib}/parse.go | 49 +++++++------ {spflib => pkg/spflib}/parse_test.go | 2 +- {spflib => pkg/spflib}/testdata-dns1.json | 0 {transform => pkg/transform}/transform.go | 0 .../transform}/transform_test.go | 0 providers/cloudflare/cloudflareProvider.go | 2 +- providers/cloudflare/preprocess_test.go | 2 +- 49 files changed, 140 insertions(+), 49 deletions(-) rename {misc => cmd}/convertzone/README.md (100%) rename {misc => cmd}/convertzone/main.go (100%) create mode 100644 cmd/spftest/spf-store.json create mode 100644 cmd/spftest/spf-store2.json delete mode 100644 js/parse_tests/008-import.js rename {dnsresolver => pkg/dnsresolver}/dnscache.go (100%) rename {dnsresolver => pkg/dnsresolver}/dnscache_test.go (100%) rename {dnsresolver => pkg/dnsresolver}/resolver.go (100%) rename {js => pkg/js}/helpers.js (100%) rename {js => pkg/js}/js.go (100%) rename {js => pkg/js}/js_test.go (93%) rename {js => pkg/js}/parse_tests/001-basic.js (100%) rename {js => pkg/js}/parse_tests/001-basic.json (100%) rename {js => pkg/js}/parse_tests/002-ttl.js (100%) rename {js => pkg/js}/parse_tests/002-ttl.json (100%) rename {js => pkg/js}/parse_tests/003-meta.js (100%) rename {js => pkg/js}/parse_tests/003-meta.json (100%) rename {js => pkg/js}/parse_tests/004-ips.js (100%) rename {js => pkg/js}/parse_tests/004-ips.json (100%) rename {js => pkg/js}/parse_tests/005-multipleDomains.js (100%) rename {js => pkg/js}/parse_tests/005-multipleDomains.json (100%) rename {js => pkg/js}/parse_tests/006-transforms.js (100%) rename {js => pkg/js}/parse_tests/006-transforms.json (100%) rename {js => pkg/js}/parse_tests/007-importTransformTTL.js (100%) rename {js => pkg/js}/parse_tests/007-importTransformTTL.json (100%) create mode 100644 pkg/js/parse_tests/008-import.js rename {js => pkg/js}/parse_tests/008-import.json (100%) rename {js => pkg/js}/parse_tests/010-alias.js (100%) rename {js => pkg/js}/parse_tests/010-alias.json (100%) rename {js => pkg/js}/parse_tests/011-cfRedirect.js (100%) rename {js => pkg/js}/parse_tests/011-cfRedirect.json (100%) rename {js => pkg/js}/parse_tests/import.js (100%) rename {js => pkg/js}/static.go (99%) rename {nameservers => pkg/nameservers}/nameservers.go (100%) rename {normalize => pkg/normalize}/importTransform_test.go (100%) rename {normalize => pkg/normalize}/validate.go (99%) rename {normalize => pkg/normalize}/validate_test.go (100%) rename {spflib => pkg/spflib}/parse.go (71%) rename {spflib => pkg/spflib}/parse_test.go (91%) rename {spflib => pkg/spflib}/testdata-dns1.json (100%) rename {transform => pkg/transform}/transform.go (100%) rename {transform => pkg/transform}/transform_test.go (100%) diff --git a/build/generate/generate.go b/build/generate/generate.go index c2743851a..764e7d7c0 100644 --- a/build/generate/generate.go +++ b/build/generate/generate.go @@ -5,14 +5,13 @@ import ( ) func main() { - //go:generate esc -modtime 0 -o js/static.go -pkg js -include helpers\.js -ignore go -prefix js js conf := &embed.Config{ ModTime: "0", - OutputFile: "js/static.go", + OutputFile: "pkg/js/static.go", Package: "js", - Prefix: "js", + Prefix: "pkg/js", Private: true, - Files: []string{`js/helpers.js`}, + Files: []string{`pkg/js/helpers.js`}, } embed.Run(conf) } diff --git a/misc/convertzone/README.md b/cmd/convertzone/README.md similarity index 100% rename from misc/convertzone/README.md rename to cmd/convertzone/README.md diff --git a/misc/convertzone/main.go b/cmd/convertzone/main.go similarity index 100% rename from misc/convertzone/main.go rename to cmd/convertzone/main.go diff --git a/cmd/spftest/main.go b/cmd/spftest/main.go index df3a86c96..2e698621c 100644 --- a/cmd/spftest/main.go +++ b/cmd/spftest/main.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/StackExchange/dnscontrol/dnsresolver" - "github.com/StackExchange/dnscontrol/spflib" + "github.com/StackExchange/dnscontrol/pkg/dnsresolver" + "github.com/StackExchange/dnscontrol/pkg/spflib" ) func main() { @@ -27,9 +27,11 @@ func main() { fmt.Println("---------------------") fmt.Println() - res := dnsresolver.NewResolverLive("preload-dns.json") - //res := dnsresolver.NewResolverPreloaded("preload-dns.json") - + //res := dnsresolver.NewResolverLive("spf-store2.json") + res, err := dnsresolver.NewResolverPreloaded("spf-store2.json") + if err != nil { + panic(err) + } rec, err := spflib.Parse(strings.Join([]string{"v=spf1", "ip4:198.252.206.0/24", "ip4:192.111.0.0/24", @@ -50,15 +52,15 @@ func main() { fmt.Println("---------------------") fmt.Println() - var spfs []string - spfs, err = spflib.Lookup("stackex.com", res) + var spf string + spf, err = spflib.Lookup("whatexit.org", res) if err != nil { panic(err) } - rec, err = spflib.Parse(strings.Join(spfs, " "), res) + rec, err = spflib.Parse(spf, res) if err != nil { panic(err) } spflib.DumpSPF(rec, "") - + //res.Close() } diff --git a/cmd/spftest/spf-store.json b/cmd/spftest/spf-store.json new file mode 100644 index 000000000..d1e1eaa81 --- /dev/null +++ b/cmd/spftest/spf-store.json @@ -0,0 +1,12 @@ +{ + "_spf.google.com": { + "txt": [ + "v=spf1 include:_netblocks.google.com include:_netblocks2.google.com include:_netblocks3.google.com ~all" + ] + }, + "spf-basic.fogcreek.com": { + "txt": [ + "v=spf1 ip4:64.34.80.172 -all" + ] + } +} diff --git a/cmd/spftest/spf-store2.json b/cmd/spftest/spf-store2.json new file mode 100644 index 000000000..8cddaa5a3 --- /dev/null +++ b/cmd/spftest/spf-store2.json @@ -0,0 +1,69 @@ +{ + "_netblocks.google.com": { + "txt": [ + "v=spf1 ip4:64.18.0.0/20 ip4:64.233.160.0/19 ip4:66.102.0.0/20 ip4:66.249.80.0/20 ip4:72.14.192.0/18 ip4:74.125.0.0/16 ip4:108.177.8.0/21 ip4:173.194.0.0/16 ip4:207.126.144.0/20 ip4:209.85.128.0/17 ip4:216.58.192.0/19 ip4:216.239.32.0/19 ~all" + ] + }, + "_netblocks2.google.com": { + "txt": [ + "v=spf1 ip6:2001:4860:4000::/36 ip6:2404:6800:4000::/36 ip6:2607:f8b0:4000::/36 ip6:2800:3f0:4000::/36 ip6:2a00:1450:4000::/36 ip6:2c0f:fb50:4000::/36 ~all" + ] + }, + "_netblocks3.google.com": { + "txt": [ + "v=spf1 ip4:172.217.0.0/19 ip4:108.177.96.0/19 ~all" + ] + }, + "_spf.google.com": { + "txt": [ + "v=spf1 include:_netblocks.google.com include:_netblocks2.google.com include:_netblocks3.google.com ~all" + ] + }, + "mail.zendesk.com": { + "txt": [ + "v=spf1 ip4:192.161.144.0/20 ip4:185.12.80.0/22 ip4:96.46.150.192/27 ip4:174.137.46.0/24 ip4:188.172.128.0/20 ip4:216.198.0.0/18 ~all" + ] + }, + "mailgun.org": { + "txt": [ + "google-site-verification=FIGVOKZm6lQFDBJaiC2DdwvBy8TInunoGCt-1gnL4PA", + "v=spf1 include:spf1.mailgun.org include:spf2.mailgun.org ~all" + ] + }, + "sendgrid.net": { + "txt": [ + "v=spf1 ip4:167.89.0.0/17 ip4:208.117.48.0/20 ip4:50.31.32.0/19 ip4:198.37.144.0/20 ip4:198.21.0.0/21 ip4:192.254.112.0/20 ip4:168.245.0.0/17 ~all", + "google-site-verification=NxyooVvVaIgddVa23KTlOEuVPuhffcDqJFV8RzWrAys" + ] + }, + "servers.mcsv.net": { + "txt": [ + "v=spf1 ip4:205.201.128.0/20 ip4:198.2.128.0/18 ip4:148.105.8.0/21 ?all" + ] + }, + "spf-basic.fogcreek.com": { + "txt": [ + "v=spf1 ip4:64.34.80.172 -all" + ] + }, + "spf.mtasv.net": { + "txt": [ + "v=spf1 ip4:50.31.156.96/27 ip4:104.245.209.192/26 ~all" + ] + }, + "spf1.mailgun.org": { + "txt": [ + "v=spf1 ip4:173.193.210.32/27 ip4:50.23.218.192/27 ip4:174.37.226.64/27 ip4:208.43.239.136/30 ip4:184.173.105.0/24 ip4:184.173.153.0/24 ip4:104.130.122.0/23 ip4:146.20.112.0/26 ~all" + ] + }, + "spf2.mailgun.org": { + "txt": [ + "v=spf1 ip4:209.61.151.0/24 ip4:166.78.68.0/22 ip4:198.61.254.0/23 ip4:192.237.158.0/23 ip4:23.253.182.0/23 ip4:104.130.96.0/28 ip4:146.20.113.0/24 ip4:146.20.191.0/24 ~all" + ] + }, + "whatexit.org": { + "txt": [ + "v=spf1 ip6:2607:f2f8:a9c0::3 ip4:174.136.107.195 include:servers.mcsv.net include:_spf.google.com mx:evite.com -all" + ] + } +} diff --git a/integrationTest/integration_test.go b/integrationTest/integration_test.go index 6428abf94..ad6ba9f14 100644 --- a/integrationTest/integration_test.go +++ b/integrationTest/integration_test.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/StackExchange/dnscontrol/models" - "github.com/StackExchange/dnscontrol/nameservers" + "github.com/StackExchange/dnscontrol/pkg/nameservers" "github.com/StackExchange/dnscontrol/providers" _ "github.com/StackExchange/dnscontrol/providers/_all" "github.com/StackExchange/dnscontrol/providers/config" diff --git a/js/parse_tests/008-import.js b/js/parse_tests/008-import.js deleted file mode 100644 index 99e27afef..000000000 --- a/js/parse_tests/008-import.js +++ /dev/null @@ -1,2 +0,0 @@ - -require("js/parse_tests/import.js") diff --git a/main.go b/main.go index d530cce1f..78f28783f 100644 --- a/main.go +++ b/main.go @@ -12,10 +12,10 @@ import ( "strings" "time" - "github.com/StackExchange/dnscontrol/js" "github.com/StackExchange/dnscontrol/models" - "github.com/StackExchange/dnscontrol/nameservers" - "github.com/StackExchange/dnscontrol/normalize" + "github.com/StackExchange/dnscontrol/pkg/js" + "github.com/StackExchange/dnscontrol/pkg/nameservers" + "github.com/StackExchange/dnscontrol/pkg/normalize" "github.com/StackExchange/dnscontrol/providers" _ "github.com/StackExchange/dnscontrol/providers/_all" "github.com/StackExchange/dnscontrol/providers/config" diff --git a/models/dns.go b/models/dns.go index 47c0bbb7b..7d95cb740 100644 --- a/models/dns.go +++ b/models/dns.go @@ -10,7 +10,7 @@ import ( "reflect" "strconv" - "github.com/StackExchange/dnscontrol/transform" + "github.com/StackExchange/dnscontrol/pkg/transform" "github.com/miekg/dns" "golang.org/x/net/idna" ) diff --git a/dnsresolver/dnscache.go b/pkg/dnsresolver/dnscache.go similarity index 100% rename from dnsresolver/dnscache.go rename to pkg/dnsresolver/dnscache.go diff --git a/dnsresolver/dnscache_test.go b/pkg/dnsresolver/dnscache_test.go similarity index 100% rename from dnsresolver/dnscache_test.go rename to pkg/dnsresolver/dnscache_test.go diff --git a/dnsresolver/resolver.go b/pkg/dnsresolver/resolver.go similarity index 100% rename from dnsresolver/resolver.go rename to pkg/dnsresolver/resolver.go diff --git a/js/helpers.js b/pkg/js/helpers.js similarity index 100% rename from js/helpers.js rename to pkg/js/helpers.js diff --git a/js/js.go b/pkg/js/js.go similarity index 100% rename from js/js.go rename to pkg/js/js.go diff --git a/js/js_test.go b/pkg/js/js_test.go similarity index 93% rename from js/js_test.go rename to pkg/js/js_test.go index 17f67a678..a2432011b 100644 --- a/js/js_test.go +++ b/pkg/js/js_test.go @@ -12,12 +12,12 @@ import ( ) const ( - testDir = "js/parse_tests" - errorDir = "js/error_tests" + testDir = "pkg/js/parse_tests" + errorDir = "pkg/js/error_tests" ) func init() { - os.Chdir("..") // go up a directory so we helpers.js is in a consistent place. + os.Chdir("../..") // go up a directory so we helpers.js is in a consistent place. } func TestParsedFiles(t *testing.T) { diff --git a/js/parse_tests/001-basic.js b/pkg/js/parse_tests/001-basic.js similarity index 100% rename from js/parse_tests/001-basic.js rename to pkg/js/parse_tests/001-basic.js diff --git a/js/parse_tests/001-basic.json b/pkg/js/parse_tests/001-basic.json similarity index 100% rename from js/parse_tests/001-basic.json rename to pkg/js/parse_tests/001-basic.json diff --git a/js/parse_tests/002-ttl.js b/pkg/js/parse_tests/002-ttl.js similarity index 100% rename from js/parse_tests/002-ttl.js rename to pkg/js/parse_tests/002-ttl.js diff --git a/js/parse_tests/002-ttl.json b/pkg/js/parse_tests/002-ttl.json similarity index 100% rename from js/parse_tests/002-ttl.json rename to pkg/js/parse_tests/002-ttl.json diff --git a/js/parse_tests/003-meta.js b/pkg/js/parse_tests/003-meta.js similarity index 100% rename from js/parse_tests/003-meta.js rename to pkg/js/parse_tests/003-meta.js diff --git a/js/parse_tests/003-meta.json b/pkg/js/parse_tests/003-meta.json similarity index 100% rename from js/parse_tests/003-meta.json rename to pkg/js/parse_tests/003-meta.json diff --git a/js/parse_tests/004-ips.js b/pkg/js/parse_tests/004-ips.js similarity index 100% rename from js/parse_tests/004-ips.js rename to pkg/js/parse_tests/004-ips.js diff --git a/js/parse_tests/004-ips.json b/pkg/js/parse_tests/004-ips.json similarity index 100% rename from js/parse_tests/004-ips.json rename to pkg/js/parse_tests/004-ips.json diff --git a/js/parse_tests/005-multipleDomains.js b/pkg/js/parse_tests/005-multipleDomains.js similarity index 100% rename from js/parse_tests/005-multipleDomains.js rename to pkg/js/parse_tests/005-multipleDomains.js diff --git a/js/parse_tests/005-multipleDomains.json b/pkg/js/parse_tests/005-multipleDomains.json similarity index 100% rename from js/parse_tests/005-multipleDomains.json rename to pkg/js/parse_tests/005-multipleDomains.json diff --git a/js/parse_tests/006-transforms.js b/pkg/js/parse_tests/006-transforms.js similarity index 100% rename from js/parse_tests/006-transforms.js rename to pkg/js/parse_tests/006-transforms.js diff --git a/js/parse_tests/006-transforms.json b/pkg/js/parse_tests/006-transforms.json similarity index 100% rename from js/parse_tests/006-transforms.json rename to pkg/js/parse_tests/006-transforms.json diff --git a/js/parse_tests/007-importTransformTTL.js b/pkg/js/parse_tests/007-importTransformTTL.js similarity index 100% rename from js/parse_tests/007-importTransformTTL.js rename to pkg/js/parse_tests/007-importTransformTTL.js diff --git a/js/parse_tests/007-importTransformTTL.json b/pkg/js/parse_tests/007-importTransformTTL.json similarity index 100% rename from js/parse_tests/007-importTransformTTL.json rename to pkg/js/parse_tests/007-importTransformTTL.json diff --git a/pkg/js/parse_tests/008-import.js b/pkg/js/parse_tests/008-import.js new file mode 100644 index 000000000..d1c689a5f --- /dev/null +++ b/pkg/js/parse_tests/008-import.js @@ -0,0 +1,2 @@ + +require("pkg/js/parse_tests/import.js") diff --git a/js/parse_tests/008-import.json b/pkg/js/parse_tests/008-import.json similarity index 100% rename from js/parse_tests/008-import.json rename to pkg/js/parse_tests/008-import.json diff --git a/js/parse_tests/010-alias.js b/pkg/js/parse_tests/010-alias.js similarity index 100% rename from js/parse_tests/010-alias.js rename to pkg/js/parse_tests/010-alias.js diff --git a/js/parse_tests/010-alias.json b/pkg/js/parse_tests/010-alias.json similarity index 100% rename from js/parse_tests/010-alias.json rename to pkg/js/parse_tests/010-alias.json diff --git a/js/parse_tests/011-cfRedirect.js b/pkg/js/parse_tests/011-cfRedirect.js similarity index 100% rename from js/parse_tests/011-cfRedirect.js rename to pkg/js/parse_tests/011-cfRedirect.js diff --git a/js/parse_tests/011-cfRedirect.json b/pkg/js/parse_tests/011-cfRedirect.json similarity index 100% rename from js/parse_tests/011-cfRedirect.json rename to pkg/js/parse_tests/011-cfRedirect.json diff --git a/js/parse_tests/import.js b/pkg/js/parse_tests/import.js similarity index 100% rename from js/parse_tests/import.js rename to pkg/js/parse_tests/import.js diff --git a/js/static.go b/pkg/js/static.go similarity index 99% rename from js/static.go rename to pkg/js/static.go index 4e15971dd..3391ec786 100644 --- a/js/static.go +++ b/pkg/js/static.go @@ -189,7 +189,7 @@ func _escFSMustString(useLocal bool, name string) string { var _escData = map[string]*_escFile{ "/helpers.js": { - local: "js/helpers.js", + local: "pkg/js/helpers.js", size: 8320, modtime: 0, compressed: ` @@ -240,6 +240,6 @@ juGPPwSZXUc736wWxSGRz1JGA/lBJGQc4pypseoabTAEaRwj1nqyQmvwU+ljuaLdYjToWa7VE3qVnY+p "/": { isDir: true, - local: "js", + local: "pkg/js", }, } diff --git a/nameservers/nameservers.go b/pkg/nameservers/nameservers.go similarity index 100% rename from nameservers/nameservers.go rename to pkg/nameservers/nameservers.go diff --git a/normalize/importTransform_test.go b/pkg/normalize/importTransform_test.go similarity index 100% rename from normalize/importTransform_test.go rename to pkg/normalize/importTransform_test.go diff --git a/normalize/validate.go b/pkg/normalize/validate.go similarity index 99% rename from normalize/validate.go rename to pkg/normalize/validate.go index a9b4a06b8..6d917cdd6 100644 --- a/normalize/validate.go +++ b/pkg/normalize/validate.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/StackExchange/dnscontrol/models" + "github.com/StackExchange/dnscontrol/pkg/transform" "github.com/StackExchange/dnscontrol/providers" - "github.com/StackExchange/dnscontrol/transform" "github.com/miekg/dns" "github.com/miekg/dns/dnsutil" ) diff --git a/normalize/validate_test.go b/pkg/normalize/validate_test.go similarity index 100% rename from normalize/validate_test.go rename to pkg/normalize/validate_test.go diff --git a/spflib/parse.go b/pkg/spflib/parse.go similarity index 71% rename from spflib/parse.go rename to pkg/spflib/parse.go index 37cb61a4c..52a3c5ee4 100644 --- a/spflib/parse.go +++ b/pkg/spflib/parse.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/StackExchange/dnscontrol/dnsresolver" + "github.com/StackExchange/dnscontrol/pkg/dnsresolver" ) type SPFRecord struct { @@ -18,10 +18,10 @@ type SPFPart struct { IncludeRecord *SPFRecord } -func Lookup(target string, dnsres dnsresolver.DnsResolver) ([]string, error) { +func Lookup(target string, dnsres dnsresolver.DnsResolver) (string, error) { txts, err := dnsres.GetTxt(target) if err != nil { - return nil, err + return "", err } var result []string for _, txt := range txts { @@ -29,7 +29,20 @@ func Lookup(target string, dnsres dnsresolver.DnsResolver) ([]string, error) { result = append(result, txt) } } - return result, nil + if len(result) == 0 { + return "", fmt.Errorf("%s has no spf TXT records", target) + } + if len(result) != 1 { + return "", fmt.Errorf("%s has multiple spf TXT records", target) + } + return result[0], nil +} + +var qualifiers = map[byte]bool{ + '?': true, + '~': true, + '-': true, + '+': true, } func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) { @@ -40,17 +53,23 @@ func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) { rec := &SPFRecord{} for _, part := range parts[1:] { p := &SPFPart{Text: part} + if qualifiers[part[0]] { + part = part[1:] + } rec.Parts = append(rec.Parts, p) - if part == "~all" || part == "-all" || part == "?all" { + if part == "all" { //all. nothing else matters. break + } else if strings.HasPrefix(part, "a") || strings.HasPrefix(part, "mx") { + rec.Lookups++ + p.Lookups = 1 } else if strings.HasPrefix(part, "ip4:") || strings.HasPrefix(part, "ip6:") { //ip address, 0 lookups continue } else if strings.HasPrefix(part, "include:") { rec.Lookups++ includeTarget := strings.TrimPrefix(part, "include:") - subRecord, err := resolveSPF(includeTarget, dnsres) + subRecord, err := Lookup(includeTarget, dnsres) if err != nil { return nil, err } @@ -59,6 +78,7 @@ func Parse(text string, dnsres dnsresolver.DnsResolver) (*SPFRecord, error) { return nil, fmt.Errorf("In included spf: %s", err) } rec.Lookups += p.IncludeRecord.Lookups + p.Lookups = p.IncludeRecord.Lookups + 1 } else { return nil, fmt.Errorf("Unsupported spf part %s", part) } @@ -77,22 +97,11 @@ func DumpSPF(rec *SPFRecord, indent string) { fmt.Println() indent += "\t" for _, p := range rec.Parts { - if p.IncludeRecord != nil { + if p.Lookups > 0 { fmt.Println(indent + p.Text) + } + if p.IncludeRecord != nil { DumpSPF(p.IncludeRecord, indent+"\t") } } } - -func resolveSPF(target string, dnsres dnsresolver.DnsResolver) (string, error) { - recs, err := dnsres.GetTxt(target) - if err != nil { - return "", err - } - for _, r := range recs { - if strings.HasPrefix(r, "v=spf1 ") { - return r, nil - } - } - return "", fmt.Errorf("No SPF records found for %s", target) -} diff --git a/spflib/parse_test.go b/pkg/spflib/parse_test.go similarity index 91% rename from spflib/parse_test.go rename to pkg/spflib/parse_test.go index aee030509..b6c1513ee 100644 --- a/spflib/parse_test.go +++ b/pkg/spflib/parse_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/StackExchange/dnscontrol/dnsresolver" + "github.com/StackExchange/dnscontrol/pkg/dnsresolver" ) func TestParse(t *testing.T) { diff --git a/spflib/testdata-dns1.json b/pkg/spflib/testdata-dns1.json similarity index 100% rename from spflib/testdata-dns1.json rename to pkg/spflib/testdata-dns1.json diff --git a/transform/transform.go b/pkg/transform/transform.go similarity index 100% rename from transform/transform.go rename to pkg/transform/transform.go diff --git a/transform/transform_test.go b/pkg/transform/transform_test.go similarity index 100% rename from transform/transform_test.go rename to pkg/transform/transform_test.go diff --git a/providers/cloudflare/cloudflareProvider.go b/providers/cloudflare/cloudflareProvider.go index 5794f46e1..a847cf592 100644 --- a/providers/cloudflare/cloudflareProvider.go +++ b/providers/cloudflare/cloudflareProvider.go @@ -9,9 +9,9 @@ import ( "time" "github.com/StackExchange/dnscontrol/models" + "github.com/StackExchange/dnscontrol/pkg/transform" "github.com/StackExchange/dnscontrol/providers" "github.com/StackExchange/dnscontrol/providers/diff" - "github.com/StackExchange/dnscontrol/transform" "github.com/miekg/dns/dnsutil" ) diff --git a/providers/cloudflare/preprocess_test.go b/providers/cloudflare/preprocess_test.go index b30396ef4..f23e458c0 100644 --- a/providers/cloudflare/preprocess_test.go +++ b/providers/cloudflare/preprocess_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/StackExchange/dnscontrol/models" - "github.com/StackExchange/dnscontrol/transform" + "github.com/StackExchange/dnscontrol/pkg/transform" ) func newDomainConfig() *models.DomainConfig {