From a803371fa4da95080b33b7c011a65e3b70ad80db Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 9 Jul 2018 01:21:46 -0700 Subject: [PATCH] add AXFR source to OctoDNS Adds a new source requested in #239. This source allows a user to pull data from a legacy system (Bind9, etc.) that does not have an API/existing provider via AXFR Zone Transfer. --- README.md | 5 +- octodns/source/axfr.py | 162 ++++++++++++++++++ tests/test_octodns_source_axfr.py | 40 +++++ tests/test_octodns_source_tinydns.py | 2 +- .../zones/{ => tinydns}/.is-needed-for-tests | Bin tests/zones/{ => tinydns}/example.com | 0 tests/zones/{ => tinydns}/other.foo | 0 tests/zones/unit.tests.db | 42 +++++ 8 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 octodns/source/axfr.py create mode 100644 tests/test_octodns_source_axfr.py rename tests/zones/{ => tinydns}/.is-needed-for-tests (100%) rename tests/zones/{ => tinydns}/example.com (100%) rename tests/zones/{ => tinydns}/other.foo (100%) create mode 100644 tests/zones/unit.tests.db diff --git a/README.md b/README.md index b263aec..ba83e2c 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,8 @@ The above command pulled the existing data out of Route53 and placed the results | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | -| [TinyDNSSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | +| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | +| [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | | All | Yes | config | #### Notes @@ -174,7 +175,7 @@ The above command pulled the existing data out of Route53 and placed the results ## Custom Sources and Providers -You can check out the [source](/octodns/source/) and [provider](/octodns/provider/) directory to see what's currently supported. Sources act as a source of record information. TinyDnsProvider is currently the only OSS source, though we have several others internally that are specific to our environment. These include something to pull host data from [gPanel](https://githubengineering.com/githubs-metal-cloud/) and a similar provider that sources information about our network gear to create both `A` & `PTR` records for their interfaces. Things that might make good OSS sources might include an `ElbSource` that pulls information about [AWS Elastic Load Balancers](https://aws.amazon.com/elasticloadbalancing/) and dynamically creates `CNAME`s for them, or `Ec2Source` that pulls instance information so that records can be created for hosts similar to how our `GPanelProvider` works. An `AxfrSource` could be really interesting as well. Another case where a source may make sense is if you'd like to export data from a legacy service that you have no plans to push changes back into. +You can check out the [source](/octodns/source/) and [provider](/octodns/provider/) directory to see what's currently supported. Sources act as a source of record information. AxfrSource and TinyDnsFileSource are currently the only OSS sources, though we have several others internally that are specific to our environment. These include something to pull host data from [gPanel](https://githubengineering.com/githubs-metal-cloud/) and a similar provider that sources information about our network gear to create both `A` & `PTR` records for their interfaces. Things that might make good OSS sources might include an `ElbSource` that pulls information about [AWS Elastic Load Balancers](https://aws.amazon.com/elasticloadbalancing/) and dynamically creates `CNAME`s for them, or `Ec2Source` that pulls instance information so that records can be created for hosts similar to how our `GPanelProvider` works. Most of the things included in OctoDNS are providers, the obvious difference being that they can serve as both sources and targets of data. We'd really like to see this list grow over time so if you use an unsupported provider then PRs are welcome. The existing providers should serve as reasonable examples. Those that have no GeoDNS support are relatively straightforward. Unfortunately most of the APIs involved to do GeoDNS style traffic management are complex and somewhat inconsistent so adding support for that function would be nice, but is optional and best done in a separate pass. diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py new file mode 100644 index 0000000..42d5fc0 --- /dev/null +++ b/octodns/source/axfr.py @@ -0,0 +1,162 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import dns.name +import dns.query +import dns.zone +import dns.rdatatype + +from dns.exception import DNSException + +from collections import defaultdict +import logging + +from ..record import Record +from .base import BaseSource + + +class AxfrBaseSource(BaseSource): + + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', + 'SRV', 'TXT')) + + def __init__(self, id): + super(AxfrBaseSource, self).__init__(id) + + def _data_for_multiple(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['value'] for r in records] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + preference, exchange = record['value'].split(' ', 1) + values.append({ + 'preference': preference, + 'exchange': exchange, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [value['value'].replace(';', '\\;') for value in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_SPF = _data_for_TXT + + def _data_for_single(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['value'] + } + + _data_for_CNAME = _data_for_single + _data_for_PTR = _data_for_single + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + priority, weight, port, target = record['value'].split(' ', 3) + values.append({ + 'priority': priority, + 'weight': weight, + 'port': port, + 'target': target, + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + continue + name = zone.hostname_from_fqdn(record['name']) + values[name][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + +class AxfrSourceException(Exception): + pass + + +class AxfrSourceZoneTransferFailed(AxfrSourceException): + + def __init__(self): + super(AxfrSourceZoneTransferFailed, self).__init__( + 'Unable to Perform Zone Transfer') + + +class AxfrSource(AxfrBaseSource): + ''' + Axfr zonefile importer to import data + + axfr: + class: octodns.source.axfr.AxfrSource + # The address of nameserver to perform zone transfer against + master: ns1.example.com + ''' + def __init__(self, id, master): + self.log = logging.getLogger('AxfrSource[{}]'.format(id)) + self.log.debug('__init__: id=%s, master=%s', id, master) + super(AxfrSource, self).__init__(id) + self.master = master + + def zone_records(self, zone): + try: + z = dns.zone.from_xfr(dns.query.xfr(self.master, zone.name, + relativize=False), + relativize=False) + except DNSException: + raise AxfrSourceZoneTransferFailed() + + records = [] + + for (name, ttl, rdata) in z.iterate_rdatas(): + rdtype = dns.rdatatype.to_text(rdata.rdtype) + records.append({ + "name": name.to_text(), + "ttl": ttl, + "type": rdtype, + "value": rdata.to_text() + }) + + return records diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py new file mode 100644 index 0000000..d90eb5b --- /dev/null +++ b/tests/test_octodns_source_axfr.py @@ -0,0 +1,40 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import dns.zone +from dns.exception import DNSException + +from mock import patch +from unittest import TestCase + +from octodns.source.axfr import AxfrSource, AxfrSourceZoneTransferFailed +from octodns.zone import Zone + + +class TestAxfrSource(TestCase): + source = AxfrSource('test', 'localhost') + + forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.db', + 'unit.tests', relativize=False) + + @patch('dns.zone.from_xfr') + def test_populate(self, from_xfr_mock): + got = Zone('unit.tests.', []) + + from_xfr_mock.side_effect = [ + self.forward_zonefile, + DNSException + ] + + self.source.populate(got) + self.assertEquals(11, len(got.records)) + + with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: + zone = Zone('unit.tests.', []) + self.source.populate(zone) + self.assertEquals('Unable to Perform Zone Transfer', + ctx.exception.message) diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index 5792b25..d2e0e21 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -15,7 +15,7 @@ from helpers import SimpleProvider class TestTinyDnsFileSource(TestCase): - source = TinyDnsFileSource('test', './tests/zones') + source = TinyDnsFileSource('test', './tests/zones/tinydns') def test_populate_normal(self): got = Zone('example.com.', []) diff --git a/tests/zones/.is-needed-for-tests b/tests/zones/tinydns/.is-needed-for-tests similarity index 100% rename from tests/zones/.is-needed-for-tests rename to tests/zones/tinydns/.is-needed-for-tests diff --git a/tests/zones/example.com b/tests/zones/tinydns/example.com similarity index 100% rename from tests/zones/example.com rename to tests/zones/tinydns/example.com diff --git a/tests/zones/other.foo b/tests/zones/tinydns/other.foo similarity index 100% rename from tests/zones/other.foo rename to tests/zones/tinydns/other.foo diff --git a/tests/zones/unit.tests.db b/tests/zones/unit.tests.db new file mode 100644 index 0000000..95828ad --- /dev/null +++ b/tests/zones/unit.tests.db @@ -0,0 +1,42 @@ +$ORIGIN unit.tests. +@ IN SOA ns1.unit.tests. root.unit.tests. ( + 2018071501 ; Serial + 3600 ; Refresh (1 hour) + 600 ; Retry (10 minutes) + 604800 ; Expire (1 week) + 3600 ; NXDOMAIN ttl (1 hour) + ) + +; NS Records +@ 3600 IN NS ns1.unit.tests. +@ 3600 IN NS ns2.unit.tests. +under 3600 IN NS ns1.unit.tests. +under 3600 IN NS ns2.unit.tests. + +; SRV Records +_srv._tcp 600 IN SRV 10 20 30 foo-1.unit.tests. +_srv._tcp 600 IN SRV 10 20 30 foo-2.unit.tests. + +; TXT Records +txt 600 IN TXT "Bah bah black sheep" +txt 600 IN TXT "have you any wool." +txt 600 IN TXT "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs" + +; MX Records +mx 300 IN MX 10 smtp-4.unit.tests. +mx 300 IN MX 20 smtp-2.unit.tests. +mx 300 IN MX 30 smtp-3.unit.tests. +mx 300 IN MX 40 smtp-1.unit.tests. + +; A Records +@ 300 IN A 1.2.3.4 +@ 300 IN A 1.2.3.5 +www 300 IN A 2.2.3.6 +wwww.sub 300 IN A 2.2.3.6 + +; AAAA Records +aaaa 600 IN AAAA 2601:644:500:e210:62f8:1dff:feb8:947a + +; CNAME Records +cname 300 IN CNAME unit.tests. +included 300 IN CNAME unit.tests.