diff --git a/CHANGELOG.md b/CHANGELOG.md index c3eaae6..eebf7bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ -## v0.9.1 - UNRELEASED +## v0.9.2 - Unreleased + +* Add lenient support to Zone.add_record, allows populate from providers that + have allowed/created invalid data and situations where a sub-zone is being + extracted from a parent, but the records still exist in the remote provider. + +## v0.9.1 - 2018-05-21 - Going backwards with setup.py ### NOTICE @@ -11,6 +17,10 @@ all health checks are passing before the first sync with `--doit`. See * Major update to geo healthchecks to allow configuring host (header), path, protocol, and port [#67](https://github.com/github/octodns/pull/67) +* SSHFP algorithm type 4 +* NS1 and DNSimple support skipping unsupported record types +* Revert back to old style setup.py & requirements.txt, setup.cfg was + causing too much pita ## v0.9.0 - 2018-03-26 - Way too long since we last met diff --git a/README.md b/README.md index 5af8e62..39946c1 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ There will be other logging information presented on the screen, but successful ### Making changes -**WARNING**: OctoDNS assumes ownership of any domain you point it to. When you tell it to act it will do whatever is necessary to try and match up states including deleting any unexpected records. Be careful when playing around with OctoDNS. It's best to experiment with a fake zone or one without any data that matters until your comfortable with the system. +**WARNING**: OctoDNS assumes ownership of any domain you point it to. When you tell it to act it will do whatever is necessary to try and match up states including deleting any unexpected records. Be careful when playing around with OctoDNS. It's best to experiment with a fake zone or one without any data that matters until you're comfortable with the system. Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. diff --git a/octodns/__init__.py b/octodns/__init__.py index da72790..b45191a 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.0' +__VERSION__ = '0.9.1' diff --git a/octodns/manager.py b/octodns/manager.py index 027df54..c3fecf4 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -65,6 +65,11 @@ class MainThreadExecutor(object): class Manager(object): log = logging.getLogger('Manager') + @classmethod + def _plan_keyer(cls, p): + plan = p[1] + return len(plan.changes[0].record.zone.name) if plan.changes else 0 + def __init__(self, config_file, max_workers=None, include_meta=False): self.log.info('__init__: config_file=%s', config_file) @@ -288,6 +293,13 @@ class Manager(object): # plan pairs. plans = [p for f in futures for p in f.result()] + # Best effort sort plans children first so that we create/update + # children zones before parents which should allow us to more safely + # extract things into sub-zones. Combining a child back into a parent + # can't really be done all that safely in general so we'll optimize for + # this direction. + plans.sort(key=self._plan_keyer, reverse=True) + for output in self.plan_outputs.values(): output.run(plans=plans, log=self.log) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 1f14c4b..b3bff6c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -345,7 +345,7 @@ class AzureProvider(BaseProvider): data['type'] = typ data['ttl'] = azrecord.ttl record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ada0c7a..2c93e49 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -17,7 +17,7 @@ class BaseProvider(BaseSource): delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT): super(BaseProvider, self).__init__(id) self.log.debug('__init__: id=%s, apply_disabled=%s, ' - 'update_pcent_threshold=%.2f' + 'update_pcent_threshold=%.2f, ' 'delete_pcent_threshold=%.2f', id, apply_disabled, diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index c51e61c..ecf3e3f 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -253,7 +253,7 @@ class CloudflareProvider(BaseProvider): self.log.info('CDN rewrite %s already in zone', name) continue - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index e71e87f..84116a0 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -230,7 +230,7 @@ class DigitalOceanProvider(BaseProvider): data_for = getattr(self, '_data_for_{}'.format(_type)) record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) - zone.add_record(record) + zone.add_record(record, lenient=lenient) exists = zone.name in self._zone_records self.log.info('populate: found %s records, exists=%s', diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index b696f75..7a9db50 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -270,7 +270,7 @@ class DnsimpleProvider(BaseProvider): data_for = getattr(self, '_data_for_{}'.format(_type)) record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) - zone.add_record(record) + zone.add_record(record, lenient=lenient) exists = zone.name in self._zone_records self.log.info('populate: found %s records, exists=%s', diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index b439931..f8cee1c 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -263,7 +263,7 @@ class DnsMadeEasyProvider(BaseProvider): data_for = getattr(self, '_data_for_{}'.format(_type)) record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) - zone.add_record(record) + zone.add_record(record, lenient=lenient) exists = zone.name in self._zone_records self.log.info('populate: found %s records, exists=%s', diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index de430db..166227b 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -399,7 +399,7 @@ class DynProvider(BaseProvider): return self._traffic_directors - def _populate_traffic_directors(self, zone): + def _populate_traffic_directors(self, zone, lenient): self.log.debug('_populate_traffic_directors: zone=%s', zone.name) td_records = set() for fqdn, types in self.traffic_directors.items(): @@ -444,7 +444,7 @@ class DynProvider(BaseProvider): name = zone.hostname_from_fqdn(fqdn) record = Record.new(zone, name, data, source=self) - zone.add_record(record) + zone.add_record(record, lenient=lenient) td_records.add(record) return td_records @@ -460,7 +460,7 @@ class DynProvider(BaseProvider): td_records = set() if self.traffic_directors_enabled: - td_records = self._populate_traffic_directors(zone) + td_records = self._populate_traffic_directors(zone, lenient) exists = True dyn_zone = _CachingDynZone.get(zone.name[:-1]) @@ -483,7 +483,7 @@ class DynProvider(BaseProvider): record = Record.new(zone, name, data, source=self, lenient=lenient) if record not in td_records: - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 82a4fcf..74c17a4 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -230,7 +230,7 @@ class GoogleCloudProvider(BaseProvider): self.log.debug('populate: adding record {} records: {!s}' .format(record_name, data)) record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 37b8d77..3ae6889 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -189,6 +189,15 @@ class Ns1Provider(BaseProvider): try: nsone_zone = self._client.loadZone(zone.name[:-1]) records = nsone_zone.data['records'] + + # change answers for certain types to always be absolute + for record in records: + if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', + 'SRV']: + for i, a in enumerate(record['short_answers']): + if not a.endswith('.'): + record['short_answers'][i] = '{}.'.format(a) + geo_records = nsone_zone.search(has_geo=True) exists = True except ResourceException as e: @@ -211,7 +220,7 @@ class Ns1Provider(BaseProvider): record = Record.new(zone, name, data_for(_type, record), source=self, lenient=lenient) zone_hash[(_type, name)] = record - [zone.add_record(r) for r in zone_hash.values()] + [zone.add_record(r, lenient=lenient) for r in zone_hash.values()] self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index ee1cef9..a74a3cd 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -82,7 +82,7 @@ class OvhProvider(BaseProvider): data_for = getattr(self, '_data_for_{}'.format(_type)) record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 02319e5..30cd01e 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -199,7 +199,7 @@ class PowerDnsBaseProvider(BaseProvider): record_name = zone.hostname_from_fqdn(rrset['name']) record = Record.new(zone, record_name, data_for(rrset), source=self, lenient=lenient) - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index b9912a3..d2b85f3 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -215,7 +215,7 @@ class RackspaceProvider(BaseProvider): record = Record.new(zone, record_name, data_for(record_set), source=self) - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=True', len(zone.records) - before) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 5fc0faa..50c734c 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -217,11 +217,14 @@ class Route53Provider(BaseProvider): route53: class: octodns.provider.route53.Route53Provider - # The AWS access key id (required) + # The AWS access key id access_key_id: - # The AWS secret access key (required) + # The AWS secret access key secret_access_key: + Alternatively, you may leave out access_key_id and secret_access_key, + this will result in boto3 deciding authentication dynamically. + In general the account used will need full permissions on Route53. ''' SUPPORTS_GEO = True @@ -232,12 +235,14 @@ class Route53Provider(BaseProvider): # health check config. HEALTH_CHECK_VERSION = '0001' - def __init__(self, id, access_key_id, secret_access_key, max_changes=1000, - client_max_attempts=None, *args, **kwargs): + def __init__(self, id, access_key_id=None, secret_access_key=None, + max_changes=1000, client_max_attempts=None, *args, **kwargs): self.max_changes = max_changes + _msg = 'access_key_id={}, secret_access_key=***'.format(access_key_id) + if access_key_id is None and secret_access_key is None: + _msg = 'auth=fallback' self.log = logging.getLogger('Route53Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, access_key_id=%s, ' - 'secret_access_key=***', id, access_key_id) + self.log.debug('__init__: id=%s, %s', id, _msg) super(Route53Provider, self).__init__(id, *args, **kwargs) config = None @@ -246,9 +251,12 @@ class Route53Provider(BaseProvider): client_max_attempts) config = Config(retries={'max_attempts': client_max_attempts}) - self._conn = client('route53', aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key, - config=config) + if access_key_id is None and secret_access_key is None: + self._conn = client('route53', config=config) + else: + self._conn = client('route53', aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key, + config=config) self._r53_zones = None self._r53_rrsets = {} @@ -489,7 +497,7 @@ class Route53Provider(BaseProvider): data = data[0] record = Record.new(zone, name, data, source=self, lenient=lenient) - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 0241d50..287fd3b 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -67,7 +67,7 @@ class YamlProvider(BaseProvider): d['ttl'] = self.default_ttl record = Record.new(zone, name, d, source=self, lenient=lenient) - zone.add_record(record) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=False', len(zone.records) - before) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 8013cd4..7b06527 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -134,7 +134,7 @@ class TinyDnsBaseSource(BaseSource): record = Record.new(zone, name, data, source=self, lenient=lenient) try: - zone.add_record(record) + zone.add_record(record, lenient=lenient) except SubzoneRecordException: self.log.debug('_populate_normal: skipping subzone ' 'record=%s', record) @@ -175,7 +175,7 @@ class TinyDnsBaseSource(BaseSource): 'value': value }, source=self, lenient=lenient) try: - zone.add_record(record) + zone.add_record(record, lenient=lenient) except DuplicateRecordException: self.log.warn('Duplicate PTR record for {}, ' 'skipping'.format(addr)) diff --git a/octodns/zone.py b/octodns/zone.py index e4dc859..916f81b 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -56,11 +56,11 @@ class Zone(object): def hostname_from_fqdn(self, fqdn): return self._name_re.sub('', fqdn) - def add_record(self, record, replace=False): + def add_record(self, record, replace=False, lenient=False): name = record.name last = name.split('.')[-1] - if last in self.sub_zones: + if not lenient and last in self.sub_zones: if name != last: # it's a record for something under a sub-zone raise SubzoneRecordException('Record {} is under a ' @@ -82,8 +82,8 @@ class Zone(object): raise DuplicateRecordException('Duplicate record {}, type {}' .format(record.fqdn, record._type)) - elif ((record._type == 'CNAME' and len(node) > 0) or - ('CNAME' in map(lambda r: r._type, node))): + elif not lenient and (((record._type == 'CNAME' and len(node) > 0) or + ('CNAME' in map(lambda r: r._type, node)))): # We're adding a CNAME to existing records or adding to an existing # CNAME raise InvalidNodeException('Invalid state, CNAME at {} cannot ' diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index d5ac5b3..22a0ee6 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -61,7 +61,7 @@ class TestBaseProvider(TestCase): class HasSupportsGeo(HasLog): SUPPORTS_GEO = False - zone = Zone('unit.tests.', []) + zone = Zone('unit.tests.', ['sub']) with self.assertRaises(NotImplementedError) as ctx: HasSupportsGeo('hassupportsgeo').populate(zone) self.assertEquals('Abstract base class, SUPPORTS property missing', @@ -81,12 +81,17 @@ class TestBaseProvider(TestCase): 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' - })) + }), lenient=lenient) zone.add_record(Record.new(zone, 'going', { 'ttl': 60, 'type': 'A', 'value': '3.4.5.6' - })) + }), lenient=lenient) + zone.add_record(Record.new(zone, 'foo.sub', { + 'ttl': 61, + 'type': 'A', + 'value': '4.5.6.7' + }), lenient=lenient) zone.add_record(Record.new(zone, '', { 'ttl': 60, @@ -98,7 +103,7 @@ class TestBaseProvider(TestCase): .supports(list(zone.records)[0])) plan = HasPopulate('haspopulate').plan(zone) - self.assertEquals(2, len(plan.changes)) + self.assertEquals(3, len(plan.changes)) with self.assertRaises(NotImplementedError) as ctx: HasPopulate('haspopulate').apply(plan) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index fc74e9e..8530b62 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -133,12 +133,12 @@ class TestNs1Provider(TestCase): }, { 'type': 'CNAME', 'ttl': 34, - 'short_answers': ['foo.unit.tests.'], + 'short_answers': ['foo.unit.tests'], 'domain': 'cname.unit.tests.', }, { 'type': 'MX', 'ttl': 35, - 'short_answers': ['10 mx1.unit.tests.', '20 mx2.unit.tests.'], + 'short_answers': ['10 mx1.unit.tests.', '20 mx2.unit.tests'], 'domain': 'unit.tests.', }, { 'type': 'NAPTR', @@ -151,18 +151,18 @@ class TestNs1Provider(TestCase): }, { 'type': 'NS', 'ttl': 37, - 'short_answers': ['ns1.unit.tests.', 'ns2.unit.tests.'], + 'short_answers': ['ns1.unit.tests.', 'ns2.unit.tests'], 'domain': 'unit.tests.', }, { 'type': 'SRV', 'ttl': 38, 'short_answers': ['12 30 30 foo-2.unit.tests.', - '10 20 30 foo-1.unit.tests.'], + '10 20 30 foo-1.unit.tests'], 'domain': '_srv._tcp.unit.tests.', }, { 'type': 'NS', 'ttl': 39, - 'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'], + 'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests'], 'domain': 'sub.unit.tests.', }, { 'type': 'CAA', diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index aec31cb..75ee991 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -167,6 +167,23 @@ class TestRoute53Provider(TestCase): return (provider, stubber) + def _get_stubbed_fallback_auth_provider(self): + provider = Route53Provider('test') + + # Use the stubber + stubber = Stubber(provider._conn) + stubber.activate() + + return (provider, stubber) + + def test_populate_with_fallback(self): + provider, stubber = self._get_stubbed_fallback_auth_provider() + + got = Zone('unit.tests.', []) + with self.assertRaises(ClientError): + stubber.add_client_error('list_hosted_zones') + provider.populate(got) + def test_populate(self): provider, stubber = self._get_stubbed_provider() diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 94faef3..b371590 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -139,9 +139,9 @@ class TestZone(TestCase): self.assertTrue('missing ending dot' in ctx.exception.message) def test_sub_zones(self): - zone = Zone('unit.tests.', set(['sub', 'barred'])) # NS for exactly the sub is allowed + zone = Zone('unit.tests.', set(['sub', 'barred'])) record = Record.new(zone, 'sub', { 'ttl': 3600, 'type': 'NS', @@ -151,6 +151,7 @@ class TestZone(TestCase): self.assertEquals(set([record]), zone.records) # non-NS for exactly the sub is rejected + zone = Zone('unit.tests.', set(['sub', 'barred'])) record = Record.new(zone, 'sub', { 'ttl': 3600, 'type': 'A', @@ -159,8 +160,12 @@ class TestZone(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) self.assertTrue('not of type NS', ctx.exception.message) + # Can add it w/lenient + zone.add_record(record, lenient=True) + self.assertEquals(set([record]), zone.records) # NS for something below the sub is rejected + zone = Zone('unit.tests.', set(['sub', 'barred'])) record = Record.new(zone, 'foo.sub', { 'ttl': 3600, 'type': 'NS', @@ -169,8 +174,12 @@ class TestZone(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) self.assertTrue('under a managed sub-zone', ctx.exception.message) + # Can add it w/lenient + zone.add_record(record, lenient=True) + self.assertEquals(set([record]), zone.records) # A for something below the sub is rejected + zone = Zone('unit.tests.', set(['sub', 'barred'])) record = Record.new(zone, 'foo.bar.sub', { 'ttl': 3600, 'type': 'A', @@ -179,6 +188,9 @@ class TestZone(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) self.assertTrue('under a managed sub-zone', ctx.exception.message) + # Can add it w/lenient + zone.add_record(record, lenient=True) + self.assertEquals(set([record]), zone.records) def test_ignored_records(self): zone_normal = Zone('unit.tests.', []) @@ -230,12 +242,18 @@ class TestZone(TestCase): zone.add_record(a) with self.assertRaises(InvalidNodeException): zone.add_record(cname) + self.assertEquals(set([a]), zone.records) + zone.add_record(cname, lenient=True) + self.assertEquals(set([a, cname]), zone.records) # add a to cname zone = Zone('unit.tests.', []) zone.add_record(cname) with self.assertRaises(InvalidNodeException): zone.add_record(a) + self.assertEquals(set([cname]), zone.records) + zone.add_record(a, lenient=True) + self.assertEquals(set([a, cname]), zone.records) def test_excluded_records(self): zone_normal = Zone('unit.tests.', [])