mirror/github-octodns
1
0
Fork 0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Code Issues Releases Activity

Merge branch 'master' into patch-1

Browse Source
This commit is contained in:
Ross McFarland
2018-06-15 06:55:06 -07:00
committed by GitHub
parent a5caf4ba5c 29ac9cb7ec
commit f9c53cc4ff
24 changed files with 124 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+11 -1
View File
@@ -1,4 +1,10 @@
## v0.9.1 - UNRELEASED
## v0.9.2 - Unreleased
* Add lenient support to Zone.add_record, allows populate from providers that
have allowed/created invalid data and situations where a sub-zone is being
extracted from a parent, but the records still exist in the remote provider.
## v0.9.1 - 2018-05-21 - Going backwards with setup.py
### NOTICE
@@ -11,6 +17,10 @@ all health checks are passing before the first sync with `--doit`. See
* Major update to geo healthchecks to allow configuring host (header), path,
protocol, and port [#67](https://github.com/github/octodns/pull/67)
* SSHFP algorithm type 4
* NS1 and DNSimple support skipping unsupported record types
* Revert back to old style setup.py & requirements.txt, setup.cfg was
causing too much pita
## v0.9.0 - 2018-03-26 - Way too long since we last met
README.md
+1 -1
View File
@@ -98,7 +98,7 @@ There will be other logging information presented on the screen, but successful
### Making changes
**WARNING**: OctoDNS assumes ownership of any domain you point it to. When you tell it to act it will do whatever is necessary to try and match up states including deleting any unexpected records. Be careful when playing around with OctoDNS. It's best to experiment with a fake zone or one without any data that matters until your comfortable with the system.
**WARNING**: OctoDNS assumes ownership of any domain you point it to. When you tell it to act it will do whatever is necessary to try and match up states including deleting any unexpected records. Be careful when playing around with OctoDNS. It's best to experiment with a fake zone or one without any data that matters until you're comfortable with the system.
Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes.
octodns/__init__.py
+1 -1
View File
@@ -3,4 +3,4 @@
from __future__ import absolute_import, division, print_function, \
unicode_literals
__VERSION__ = '0.9.0'
__VERSION__ = '0.9.1'
octodns/manager.py
+12
View File
@@ -65,6 +65,11 @@ class MainThreadExecutor(object):
class Manager(object):
log = logging.getLogger('Manager')
@classmethod
def _plan_keyer(cls, p):
plan = p[1]
return len(plan.changes[0].record.zone.name) if plan.changes else 0
def __init__(self, config_file, max_workers=None, include_meta=False):
self.log.info('__init__: config_file=%s', config_file)
@@ -288,6 +293,13 @@ class Manager(object):
# plan pairs.
plans = [p for f in futures for p in f.result()]
# Best effort sort plans children first so that we create/update
# children zones before parents which should allow us to more safely
# extract things into sub-zones. Combining a child back into a parent
# can't really be done all that safely in general so we'll optimize for
# this direction.
plans.sort(key=self._plan_keyer, reverse=True)
for output in self.plan_outputs.values():
output.run(plans=plans, log=self.log)
octodns/provider/azuredns.py
+1 -1
View File
@@ -345,7 +345,7 @@ class AzureProvider(BaseProvider):
data['type'] = typ
data['ttl'] = azrecord.ttl
record = Record.new(zone, record_name, data, source=self)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
octodns/provider/base.py
+1 -1
View File
@@ -17,7 +17,7 @@ class BaseProvider(BaseSource):
delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT):
super(BaseProvider, self).__init__(id)
self.log.debug('__init__: id=%s, apply_disabled=%s, '
'update_pcent_threshold=%.2f'
'update_pcent_threshold=%.2f, '
'delete_pcent_threshold=%.2f',
id,
apply_disabled,
octodns/provider/cloudflare.py
+1 -1
View File
@@ -253,7 +253,7 @@ class CloudflareProvider(BaseProvider):
self.log.info('CDN rewrite %s already in zone', name)
continue
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
octodns/provider/digitalocean.py
+1 -1
View File
@@ -230,7 +230,7 @@ class DigitalOceanProvider(BaseProvider):
data_for = getattr(self, '_data_for_{}'.format(_type))
record = Record.new(zone, name, data_for(_type, records),
source=self, lenient=lenient)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
exists = zone.name in self._zone_records
self.log.info('populate: found %s records, exists=%s',
octodns/provider/dnsimple.py
+1 -1
View File
@@ -270,7 +270,7 @@ class DnsimpleProvider(BaseProvider):
data_for = getattr(self, '_data_for_{}'.format(_type))
record = Record.new(zone, name, data_for(_type, records),
source=self, lenient=lenient)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
exists = zone.name in self._zone_records
self.log.info('populate: found %s records, exists=%s',
octodns/provider/dnsmadeeasy.py
+1 -1
View File
@@ -263,7 +263,7 @@ class DnsMadeEasyProvider(BaseProvider):
data_for = getattr(self, '_data_for_{}'.format(_type))
record = Record.new(zone, name, data_for(_type, records),
source=self, lenient=lenient)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
exists = zone.name in self._zone_records
self.log.info('populate: found %s records, exists=%s',
octodns/provider/dyn.py
+4 -4
View File
@@ -399,7 +399,7 @@ class DynProvider(BaseProvider):
return self._traffic_directors
def _populate_traffic_directors(self, zone):
def _populate_traffic_directors(self, zone, lenient):
self.log.debug('_populate_traffic_directors: zone=%s', zone.name)
td_records = set()
for fqdn, types in self.traffic_directors.items():
@@ -444,7 +444,7 @@ class DynProvider(BaseProvider):
name = zone.hostname_from_fqdn(fqdn)
record = Record.new(zone, name, data, source=self)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
td_records.add(record)
return td_records
@@ -460,7 +460,7 @@ class DynProvider(BaseProvider):
td_records = set()
if self.traffic_directors_enabled:
td_records = self._populate_traffic_directors(zone)
td_records = self._populate_traffic_directors(zone, lenient)
exists = True
dyn_zone = _CachingDynZone.get(zone.name[:-1])
@@ -483,7 +483,7 @@ class DynProvider(BaseProvider):
record = Record.new(zone, name, data, source=self,
lenient=lenient)
if record not in td_records:
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
octodns/provider/googlecloud.py
+1 -1
View File
@@ -230,7 +230,7 @@ class GoogleCloudProvider(BaseProvider):
self.log.debug('populate: adding record {} records: {!s}'
.format(record_name, data))
record = Record.new(zone, record_name, data, source=self)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
octodns/provider/ns1.py
+10 -1
View File
@@ -189,6 +189,15 @@ class Ns1Provider(BaseProvider):
try:
nsone_zone = self._client.loadZone(zone.name[:-1])
records = nsone_zone.data['records']
# change answers for certain types to always be absolute
for record in records:
if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR',
'SRV']:
for i, a in enumerate(record['short_answers']):
if not a.endswith('.'):
record['short_answers'][i] = '{}.'.format(a)
geo_records = nsone_zone.search(has_geo=True)
exists = True
except ResourceException as e:
@@ -211,7 +220,7 @@ class Ns1Provider(BaseProvider):
record = Record.new(zone, name, data_for(_type, record),
source=self, lenient=lenient)
zone_hash[(_type, name)] = record
[zone.add_record(r) for r in zone_hash.values()]
[zone.add_record(r, lenient=lenient) for r in zone_hash.values()]
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
return exists
octodns/provider/ovh.py
+1 -1
View File
@@ -82,7 +82,7 @@ class OvhProvider(BaseProvider):
data_for = getattr(self, '_data_for_{}'.format(_type))
record = Record.new(zone, name, data_for(_type, records),
source=self, lenient=lenient)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
octodns/provider/powerdns.py
+1 -1
View File
@@ -199,7 +199,7 @@ class PowerDnsBaseProvider(BaseProvider):
record_name = zone.hostname_from_fqdn(rrset['name'])
record = Record.new(zone, record_name, data_for(rrset),
source=self, lenient=lenient)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
octodns/provider/rackspace.py
+1 -1
View File
@@ -215,7 +215,7 @@ class RackspaceProvider(BaseProvider):
record = Record.new(zone, record_name,
data_for(record_set),
source=self)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=True',
len(zone.records) - before)
octodns/provider/route53.py
+18 -10
View File
@@ -217,11 +217,14 @@ class Route53Provider(BaseProvider):
route53:
class: octodns.provider.route53.Route53Provider
# The AWS access key id (required)
# The AWS access key id
access_key_id:
# The AWS secret access key (required)
# The AWS secret access key
secret_access_key:
Alternatively, you may leave out access_key_id and secret_access_key,
this will result in boto3 deciding authentication dynamically.
In general the account used will need full permissions on Route53.
'''
SUPPORTS_GEO = True
@@ -232,12 +235,14 @@ class Route53Provider(BaseProvider):
# health check config.
HEALTH_CHECK_VERSION = '0001'
def __init__(self, id, access_key_id, secret_access_key, max_changes=1000,
client_max_attempts=None, *args, **kwargs):
def __init__(self, id, access_key_id=None, secret_access_key=None,
max_changes=1000, client_max_attempts=None, *args, **kwargs):
self.max_changes = max_changes
_msg = 'access_key_id={}, secret_access_key=***'.format(access_key_id)
if access_key_id is None and secret_access_key is None:
_msg = 'auth=fallback'
self.log = logging.getLogger('Route53Provider[{}]'.format(id))
self.log.debug('__init__: id=%s, access_key_id=%s, '
'secret_access_key=***', id, access_key_id)
self.log.debug('__init__: id=%s, %s', id, _msg)
super(Route53Provider, self).__init__(id, *args, **kwargs)
config = None
@@ -246,9 +251,12 @@ class Route53Provider(BaseProvider):
client_max_attempts)
config = Config(retries={'max_attempts': client_max_attempts})
self._conn = client('route53', aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
config=config)
if access_key_id is None and secret_access_key is None:
self._conn = client('route53', config=config)
else:
self._conn = client('route53', aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
config=config)
self._r53_zones = None
self._r53_rrsets = {}
@@ -489,7 +497,7 @@ class Route53Provider(BaseProvider):
data = data[0]
record = Record.new(zone, name, data, source=self,
lenient=lenient)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=%s',
len(zone.records) - before, exists)
octodns/provider/yaml.py
+1 -1
View File
@@ -67,7 +67,7 @@ class YamlProvider(BaseProvider):
d['ttl'] = self.default_ttl
record = Record.new(zone, name, d, source=self,
lenient=lenient)
zone.add_record(record)
zone.add_record(record, lenient=lenient)
self.log.info('populate: found %s records, exists=False',
len(zone.records) - before)
octodns/source/tinydns.py
+2 -2
View File
@@ -134,7 +134,7 @@ class TinyDnsBaseSource(BaseSource):
record = Record.new(zone, name, data, source=self,
lenient=lenient)
try:
zone.add_record(record)
zone.add_record(record, lenient=lenient)
except SubzoneRecordException:
self.log.debug('_populate_normal: skipping subzone '
'record=%s', record)
@@ -175,7 +175,7 @@ class TinyDnsBaseSource(BaseSource):
'value': value
}, source=self, lenient=lenient)
try:
zone.add_record(record)
zone.add_record(record, lenient=lenient)
except DuplicateRecordException:
self.log.warn('Duplicate PTR record for {}, '
'skipping'.format(addr))
octodns/zone.py
+4 -4
View File
@@ -56,11 +56,11 @@ class Zone(object):
def hostname_from_fqdn(self, fqdn):
return self._name_re.sub('', fqdn)
def add_record(self, record, replace=False):
def add_record(self, record, replace=False, lenient=False):
name = record.name
last = name.split('.')[-1]
if last in self.sub_zones:
if not lenient and last in self.sub_zones:
if name != last:
# it's a record for something under a sub-zone
raise SubzoneRecordException('Record {} is under a '
@@ -82,8 +82,8 @@ class Zone(object):
raise DuplicateRecordException('Duplicate record {}, type {}'
.format(record.fqdn,
record._type))
elif ((record._type == 'CNAME' and len(node) > 0) or
('CNAME' in map(lambda r: r._type, node))):
elif not lenient and (((record._type == 'CNAME' and len(node) > 0) or
('CNAME' in map(lambda r: r._type, node)))):
# We're adding a CNAME to existing records or adding to an existing
# CNAME
raise InvalidNodeException('Invalid state, CNAME at {} cannot '
tests/test_octodns_provider_base.py
+9 -4
View File
@@ -61,7 +61,7 @@ class TestBaseProvider(TestCase):
class HasSupportsGeo(HasLog):
SUPPORTS_GEO = False
zone = Zone('unit.tests.', [])
zone = Zone('unit.tests.', ['sub'])
with self.assertRaises(NotImplementedError) as ctx:
HasSupportsGeo('hassupportsgeo').populate(zone)
self.assertEquals('Abstract base class, SUPPORTS property missing',
@@ -81,12 +81,17 @@ class TestBaseProvider(TestCase):
'ttl': 60,
'type': 'A',
'value': '2.3.4.5'
}))
}), lenient=lenient)
zone.add_record(Record.new(zone, 'going', {
'ttl': 60,
'type': 'A',
'value': '3.4.5.6'
}))
}), lenient=lenient)
zone.add_record(Record.new(zone, 'foo.sub', {
'ttl': 61,
'type': 'A',
'value': '4.5.6.7'
}), lenient=lenient)
zone.add_record(Record.new(zone, '', {
'ttl': 60,
@@ -98,7 +103,7 @@ class TestBaseProvider(TestCase):
.supports(list(zone.records)[0]))
plan = HasPopulate('haspopulate').plan(zone)
self.assertEquals(2, len(plan.changes))
self.assertEquals(3, len(plan.changes))
with self.assertRaises(NotImplementedError) as ctx:
HasPopulate('haspopulate').apply(plan)
tests/test_octodns_provider_ns1.py
+5 -5
View File
@@ -133,12 +133,12 @@ class TestNs1Provider(TestCase):
}, {
'type': 'CNAME',
'ttl': 34,
'short_answers': ['foo.unit.tests.'],
'short_answers': ['foo.unit.tests'],
'domain': 'cname.unit.tests.',
}, {
'type': 'MX',
'ttl': 35,
'short_answers': ['10 mx1.unit.tests.', '20 mx2.unit.tests.'],
'short_answers': ['10 mx1.unit.tests.', '20 mx2.unit.tests'],
'domain': 'unit.tests.',
}, {
'type': 'NAPTR',
@@ -151,18 +151,18 @@ class TestNs1Provider(TestCase):
}, {
'type': 'NS',
'ttl': 37,
'short_answers': ['ns1.unit.tests.', 'ns2.unit.tests.'],
'short_answers': ['ns1.unit.tests.', 'ns2.unit.tests'],
'domain': 'unit.tests.',
}, {
'type': 'SRV',
'ttl': 38,
'short_answers': ['12 30 30 foo-2.unit.tests.',
'10 20 30 foo-1.unit.tests.'],
'10 20 30 foo-1.unit.tests'],
'domain': '_srv._tcp.unit.tests.',
}, {
'type': 'NS',
'ttl': 39,
'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'],
'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests'],
'domain': 'sub.unit.tests.',
}, {
'type': 'CAA',
tests/test_octodns_provider_route53.py
+17
View File
@@ -167,6 +167,23 @@ class TestRoute53Provider(TestCase):
return (provider, stubber)
def _get_stubbed_fallback_auth_provider(self):
provider = Route53Provider('test')
# Use the stubber
stubber = Stubber(provider._conn)
stubber.activate()
return (provider, stubber)
def test_populate_with_fallback(self):
provider, stubber = self._get_stubbed_fallback_auth_provider()
got = Zone('unit.tests.', [])
with self.assertRaises(ClientError):
stubber.add_client_error('list_hosted_zones')
provider.populate(got)
def test_populate(self):
provider, stubber = self._get_stubbed_provider()
tests/test_octodns_zone.py
+19 -1
View File
@@ -139,9 +139,9 @@ class TestZone(TestCase):
self.assertTrue('missing ending dot' in ctx.exception.message)
def test_sub_zones(self):
zone = Zone('unit.tests.', set(['sub', 'barred']))
# NS for exactly the sub is allowed
zone = Zone('unit.tests.', set(['sub', 'barred']))
record = Record.new(zone, 'sub', {
'ttl': 3600,
'type': 'NS',
@@ -151,6 +151,7 @@ class TestZone(TestCase):
self.assertEquals(set([record]), zone.records)
# non-NS for exactly the sub is rejected
zone = Zone('unit.tests.', set(['sub', 'barred']))
record = Record.new(zone, 'sub', {
'ttl': 3600,
'type': 'A',
@@ -159,8 +160,12 @@ class TestZone(TestCase):
with self.assertRaises(SubzoneRecordException) as ctx:
zone.add_record(record)
self.assertTrue('not of type NS', ctx.exception.message)
# Can add it w/lenient
zone.add_record(record, lenient=True)
self.assertEquals(set([record]), zone.records)
# NS for something below the sub is rejected
zone = Zone('unit.tests.', set(['sub', 'barred']))
record = Record.new(zone, 'foo.sub', {
'ttl': 3600,
'type': 'NS',
@@ -169,8 +174,12 @@ class TestZone(TestCase):
with self.assertRaises(SubzoneRecordException) as ctx:
zone.add_record(record)
self.assertTrue('under a managed sub-zone', ctx.exception.message)
# Can add it w/lenient
zone.add_record(record, lenient=True)
self.assertEquals(set([record]), zone.records)
# A for something below the sub is rejected
zone = Zone('unit.tests.', set(['sub', 'barred']))
record = Record.new(zone, 'foo.bar.sub', {
'ttl': 3600,
'type': 'A',
@@ -179,6 +188,9 @@ class TestZone(TestCase):
with self.assertRaises(SubzoneRecordException) as ctx:
zone.add_record(record)
self.assertTrue('under a managed sub-zone', ctx.exception.message)
# Can add it w/lenient
zone.add_record(record, lenient=True)
self.assertEquals(set([record]), zone.records)
def test_ignored_records(self):
zone_normal = Zone('unit.tests.', [])
@@ -230,12 +242,18 @@ class TestZone(TestCase):
zone.add_record(a)
with self.assertRaises(InvalidNodeException):
zone.add_record(cname)
self.assertEquals(set([a]), zone.records)
zone.add_record(cname, lenient=True)
self.assertEquals(set([a, cname]), zone.records)
# add a to cname
zone = Zone('unit.tests.', [])
zone.add_record(cname)
with self.assertRaises(InvalidNodeException):
zone.add_record(a)
self.assertEquals(set([cname]), zone.records)
zone.add_record(a, lenient=True)
self.assertEquals(set([a, cname]), zone.records)
def test_excluded_records(self):
zone_normal = Zone('unit.tests.', [])