Merge pull request #253 from yzguy/axfr_source

add AXFR source to OctoDNS

README.md

@@ -162,7 +162,8 @@ The above command pulled the existing data out of Route53 and placed the results
 | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | |
 | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No |  |
 | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
-| [TinyDNSSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only |
+| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
+| [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only |
 | [YamlProvider](/octodns/provider/yaml.py) | | All | Yes | config |
 
 #### Notes
@@ -174,7 +175,7 @@ The above command pulled the existing data out of Route53 and placed the results
 
 ## Custom Sources and Providers
 
-You can check out the [source](/octodns/source/) and [provider](/octodns/provider/) directory to see what's currently supported. Sources act as a source of record information. TinyDnsProvider is currently the only OSS source, though we have several others internally that are specific to our environment. These include something to pull host data from  [gPanel](https://githubengineering.com/githubs-metal-cloud/) and a similar provider that sources information about our network gear to create both `A` & `PTR` records for their interfaces. Things that might make good OSS sources might include an `ElbSource` that pulls information about [AWS Elastic Load Balancers](https://aws.amazon.com/elasticloadbalancing/) and dynamically creates `CNAME`s for them, or `Ec2Source` that pulls instance information so that records can be created for hosts similar to how our `GPanelProvider` works. An `AxfrSource` could be really interesting as well. Another case where a source may make sense is if you'd like to export data from a legacy service that you have no plans to push changes back into.
+You can check out the [source](/octodns/source/) and [provider](/octodns/provider/) directory to see what's currently supported. Sources act as a source of record information. AxfrSource and TinyDnsFileSource are currently the only OSS sources, though we have several others internally that are specific to our environment. These include something to pull host data from  [gPanel](https://githubengineering.com/githubs-metal-cloud/) and a similar provider that sources information about our network gear to create both `A` & `PTR` records for their interfaces. Things that might make good OSS sources might include an `ElbSource` that pulls information about [AWS Elastic Load Balancers](https://aws.amazon.com/elasticloadbalancing/) and dynamically creates `CNAME`s for them, or `Ec2Source` that pulls instance information so that records can be created for hosts similar to how our `GPanelProvider` works.
 
 Most of the things included in OctoDNS are providers, the obvious difference being that they can serve as both sources and targets of data. We'd really like to see this list grow over time so if you use an unsupported provider then PRs are welcome. The existing providers should serve as reasonable examples. Those that have no GeoDNS support are relatively straightforward. Unfortunately most of the APIs involved to do GeoDNS style traffic management are complex and somewhat inconsistent so adding support for that function would be nice, but is optional and best done in a separate pass.
 

octodns/source/axfr.py

@@ -0,0 +1,162 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+import dns.name
+import dns.query
+import dns.zone
+import dns.rdatatype
+
+from dns.exception import DNSException
+
+from collections import defaultdict
+import logging
+
+from ..record import Record
+from .base import BaseSource
+
+
+class AxfrBaseSource(BaseSource):
+
+    SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF',
+                    'SRV', 'TXT'))
+
+    def __init__(self, id):
+        super(AxfrBaseSource, self).__init__(id)
+
+    def _data_for_multiple(self, _type, records):
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': [r['value'] for r in records]
+        }
+
+    _data_for_A = _data_for_multiple
+    _data_for_AAAA = _data_for_multiple
+    _data_for_NS = _data_for_multiple
+
+    def _data_for_MX(self, _type, records):
+        values = []
+        for record in records:
+            preference, exchange = record['value'].split(' ', 1)
+            values.append({
+                'preference': preference,
+                'exchange': exchange,
+            })
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
+    def _data_for_TXT(self, _type, records):
+        values = [value['value'].replace(';', '\\;') for value in records]
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
+    _data_for_SPF = _data_for_TXT
+
+    def _data_for_single(self, _type, records):
+        record = records[0]
+        return {
+            'ttl': record['ttl'],
+            'type': _type,
+            'value': record['value']
+        }
+
+    _data_for_CNAME = _data_for_single
+    _data_for_PTR = _data_for_single
+
+    def _data_for_SRV(self, _type, records):
+        values = []
+        for record in records:
+            priority, weight, port, target = record['value'].split(' ', 3)
+            values.append({
+                'priority': priority,
+                'weight': weight,
+                'port': port,
+                'target': target,
+            })
+        return {
+            'type': _type,
+            'ttl': records[0]['ttl'],
+            'values': values
+        }
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
+        values = defaultdict(lambda: defaultdict(list))
+        for record in self.zone_records(zone):
+            _type = record['type']
+            if _type not in self.SUPPORTS:
+                continue
+            name = zone.hostname_from_fqdn(record['name'])
+            values[name][record['type']].append(record)
+
+        before = len(zone.records)
+        for name, types in values.items():
+            for _type, records in types.items():
+                data_for = getattr(self, '_data_for_{}'.format(_type))
+                record = Record.new(zone, name, data_for(_type, records),
+                                    source=self, lenient=lenient)
+                zone.add_record(record, lenient=lenient)
+
+        self.log.info('populate:   found %s records',
+                      len(zone.records) - before)
+
+
+class AxfrSourceException(Exception):
+    pass
+
+
+class AxfrSourceZoneTransferFailed(AxfrSourceException):
+
+    def __init__(self):
+        super(AxfrSourceZoneTransferFailed, self).__init__(
+            'Unable to Perform Zone Transfer')
+
+
+class AxfrSource(AxfrBaseSource):
+    '''
+    Axfr zonefile importer to import data
+
+    axfr:
+        class: octodns.source.axfr.AxfrSource
+        # The address of nameserver to perform zone transfer against
+        master: ns1.example.com
+    '''
+    def __init__(self, id, master):
+        self.log = logging.getLogger('AxfrSource[{}]'.format(id))
+        self.log.debug('__init__: id=%s, master=%s', id, master)
+        super(AxfrSource, self).__init__(id)
+        self.master = master
+
+    def zone_records(self, zone):
+        try:
+            z = dns.zone.from_xfr(dns.query.xfr(self.master, zone.name,
+                                                relativize=False),
+                                  relativize=False)
+        except DNSException:
+            raise AxfrSourceZoneTransferFailed()
+
+        records = []
+
+        for (name, ttl, rdata) in z.iterate_rdatas():
+            rdtype = dns.rdatatype.to_text(rdata.rdtype)
+            records.append({
+                "name": name.to_text(),
+                "ttl": ttl,
+                "type": rdtype,
+                "value": rdata.to_text()
+            })
+
+        return records

tests/test_octodns_source_axfr.py

@@ -0,0 +1,40 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+import dns.zone
+from dns.exception import DNSException
+
+from mock import patch
+from unittest import TestCase
+
+from octodns.source.axfr import AxfrSource, AxfrSourceZoneTransferFailed
+from octodns.zone import Zone
+
+
+class TestAxfrSource(TestCase):
+    source = AxfrSource('test', 'localhost')
+
+    forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.db',
+                                          'unit.tests', relativize=False)
+
+    @patch('dns.zone.from_xfr')
+    def test_populate(self, from_xfr_mock):
+        got = Zone('unit.tests.', [])
+
+        from_xfr_mock.side_effect = [
+            self.forward_zonefile,
+            DNSException
+        ]
+
+        self.source.populate(got)
+        self.assertEquals(11, len(got.records))
+
+        with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx:
+            zone = Zone('unit.tests.', [])
+            self.source.populate(zone)
+        self.assertEquals('Unable to Perform Zone Transfer',
+                          ctx.exception.message)

tests/test_octodns_source_tinydns.py

@@ -15,7 +15,7 @@ from helpers import SimpleProvider
 
 
 class TestTinyDnsFileSource(TestCase):
-    source = TinyDnsFileSource('test', './tests/zones')
+    source = TinyDnsFileSource('test', './tests/zones/tinydns')
 
     def test_populate_normal(self):
         got = Zone('example.com.', [])

tests/zones/unit.tests.db

@@ -0,0 +1,42 @@
+$ORIGIN unit.tests.
+@           IN	SOA	ns1.unit.tests. root.unit.tests. (
+                        2018071501		; Serial
+                        3600    ; Refresh (1 hour)
+                        600     ; Retry (10 minutes)
+                        604800  ; Expire (1 week)
+                        3600    ; NXDOMAIN ttl (1 hour)
+                    )
+
+; NS Records
+@           3600  IN  NS  ns1.unit.tests.
+@           3600  IN  NS  ns2.unit.tests.
+under       3600  IN  NS  ns1.unit.tests.
+under       3600  IN  NS  ns2.unit.tests.
+
+; SRV Records
+_srv._tcp   600   IN  SRV 10 20 30 foo-1.unit.tests.
+_srv._tcp   600   IN  SRV 10 20 30 foo-2.unit.tests.
+
+; TXT Records
+txt         600   IN  TXT "Bah bah black sheep"
+txt         600   IN  TXT "have you any wool."
+txt         600   IN  TXT "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs"
+
+; MX Records
+mx          300   IN  MX  10  smtp-4.unit.tests.
+mx          300   IN  MX  20  smtp-2.unit.tests.
+mx          300   IN  MX  30  smtp-3.unit.tests.
+mx          300   IN  MX  40  smtp-1.unit.tests.
+
+; A Records
+@           300   IN  A   1.2.3.4
+@           300   IN  A   1.2.3.5
+www         300   IN  A   2.2.3.6
+wwww.sub    300   IN  A   2.2.3.6
+
+; AAAA Records
+aaaa        600   IN  AAAA  2601:644:500:e210:62f8:1dff:feb8:947a
+
+; CNAME Records
+cname       300   IN  CNAME   unit.tests.
+included    300   IN  CNAME   unit.tests.