mirror/github-octodns
1
0
Fork 0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Code Issues Releases Activity

Merge remote-tracking branch 'origin/main' into idna-internally

Browse Source
This commit is contained in:
Ross McFarland
2022-09-12 14:56:01 -07:00
parent cda6ef3909 51b45e5911
commit b4792707f8
19 changed files with 136 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.git-blame-ignore-revs
View File

@@ -1,2 +1,4 @@
# Commit that added in black formatting support
e116d26eeca0891c31b689e43db5bb60b62f73f6
# Commit that fixed a bunch of uneeded '...' '...' string joins from ^
fa4225b625654c51c7b0be6efcfd6a1109768a72

14
CHANGELOG.md
View File

@@ -1,3 +1,17 @@
## v0.9.19 - 2022-??-?? - ???
* Addressed shortcomings with YamlProvider.SUPPORTS in that it didn't include
dynamically registered types, was a static list that could have drifted over
time even ignoring 3rd party types.
* Provider._process_desired_zone needed to call Provider.supports rather than
doing it's own `_type in provider.SUPPORTS`. The default behavior in
Source.supports is ^, but it's possible for providers to override that
behavior and do special checking and `_process_desired_zone` wasn't taking
that into account.
* Now that it's used as it needed to be YamlProvider overrides
Provider.supports and just always says Yes so that any dynamically registered
types will be supported.
## v0.9.18 - 2022-08-14 - Subzone handling
* Fixed issue with sub-zone handling introduced in 0.9.18

1
README.md
View File

@@ -363,6 +363,7 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o
- [`doddo/octodns-lexicon`](https://github.com/doddo/octodns-lexicon): Use [Lexicon](https://github.com/AnalogJ/lexicon) providers as octoDNS providers.
- [`asyncon/octoblox`](https://github.com/asyncon/octoblox): [Infoblox](https://www.infoblox.com/) provider.
- [`sukiyaki/octodns-netbox`](https://github.com/sukiyaki/octodns-netbox): [NetBox](https://github.com/netbox-community/netbox) source.
- [`jcollie/octodns-netbox-dns`](https://github.com/jcollie/octodns-netbox-dns): [NetBox-DNS Plugin](https://github.com/auroraresearchlab/netbox-dns) provider.
- [`kompetenzbolzen/octodns-custom-provider`](https://github.com/kompetenzbolzen/octodns-custom-provider): zonefile provider & phpIPAM source.
- **Resources.**
- Article: [Visualising DNS records with Neo4j](https://medium.com/@costask/querying-and-visualising-octodns-records-with-neo4j-f4f72ab2d474) + code

4
octodns/cmds/dump.py
View File

@@ -38,13 +38,13 @@ def main():
'--lenient',
action='store_true',
default=False,
help='Ignore record validations and do a best effort ' 'dump',
help='Ignore record validations and do a best effort dump',
)
parser.add_argument(
'--split',
action='store_true',
default=False,
help='Split the dumped zone into a YAML file per ' 'record',
help='Split the dumped zone into a YAML file per record',
)
parser.add_argument('zone', help='Zone to dump')
parser.add_argument('source', nargs='+', help='Source(s) to pull data from')

2
octodns/cmds/sync.py
View File

@@ -26,7 +26,7 @@ def main():
'--doit',
action='store_true',
default=False,
help='Whether to take action or just show what would ' 'change',
help='Whether to take action or just show what would change',
)
parser.add_argument(
'--force',

21
octodns/manager.py
View File

@@ -188,7 +188,7 @@ class Manager(object):
except KeyError:
self.log.exception('Invalid provider class')
raise ManagerException(
f'Provider {provider_name} is missing ' 'class'
f'Provider {provider_name} is missing class'
)
_class, module, version = self._get_named_class('provider', _class)
kwargs = self._build_kwargs(provider_config)
@@ -216,7 +216,7 @@ class Manager(object):
except KeyError:
self.log.exception('Invalid processor class')
raise ManagerException(
f'Processor {processor_name} is ' 'missing class'
f'Processor {processor_name} is missing class'
)
_class, module, version = self._get_named_class('processor', _class)
kwargs = self._build_kwargs(processor_config)
@@ -243,7 +243,7 @@ class Manager(object):
except KeyError:
self.log.exception('Invalid plan_output class')
raise ManagerException(
f'plan_output {plan_output_name} is ' 'missing class'
f'plan_output {plan_output_name} is missing class'
)
_class, module, version = self._get_named_class(
'plan_output', _class
@@ -302,7 +302,7 @@ class Manager(object):
module, version = self._import_module(module_name)
except (ImportError, ValueError):
self.log.exception(
'_get_{}_class: Unable to import ' 'module %s', _class
'_get_{}_class: Unable to import module %s', _class
)
raise ManagerException(f'Unknown {_type} class: {_class}')
@@ -310,7 +310,7 @@ class Manager(object):
return getattr(module, class_name), module_name, version
except AttributeError:
self.log.exception(
'_get_{}_class: Unable to get class %s ' 'from module %s',
'_get_{}_class: Unable to get class %s from module %s',
class_name,
module,
)
@@ -411,7 +411,7 @@ class Manager(object):
if "unexpected keyword argument 'lenient'" not in str(e):
raise
self.log.warning(
'provider %s does not accept lenient ' 'param',
'provider %s does not accept lenient param',
source.__class__.__name__,
)
source.populate(zone)
@@ -440,7 +440,7 @@ class Manager(object):
if "keyword argument 'processors'" not in str(e):
raise
self.log.warning(
'provider.plan %s does not accept processors ' 'param',
'provider.plan %s does not accept processors param',
target.__class__.__name__,
)
plan = target.plan(zone)
@@ -567,7 +567,7 @@ class Manager(object):
trg = self.providers[target]
if not isinstance(trg, BaseProvider):
raise ManagerException(
f'{trg} - "{target}" does not ' 'support targeting'
f'{trg} - "{target}" does not support targeting'
)
trgs.append(trg)
targets = trgs
@@ -741,7 +741,7 @@ class Manager(object):
raise ManagerException(msg)
target = target.copy()
self.log.info(
'dump: setting directory of output_provider ' 'copy to %s',
'dump: setting directory of output_provider copy to %s',
output_dir,
)
target.directory = output_dir
@@ -831,8 +831,7 @@ class Manager(object):
def get_zone(self, zone_name):
if not zone_name[-1] == '.':
raise ManagerException(
f'Invalid zone name {idna_decode(zone_name)}, missing '
'ending dot'
f'Invalid zone name {idna_decode(zone_name)}, missing ending dot'
)
zone = self.config['zones'].get(zone_name)

9
octodns/provider/base.py
View File

@@ -59,7 +59,7 @@ class BaseProvider(BaseSource):
'''
for record in desired.records:
if record._type not in self.SUPPORTS:
if not self.supports(record):
msg = f'{record._type} records not supported for {record.fqdn}'
fallback = 'omitting record'
self.supports_warn_or_except(msg, fallback)
@@ -187,8 +187,7 @@ class BaseProvider(BaseSource):
# If your code gets this warning see Source.populate for more
# information
self.log.warning(
'Provider %s used in target mode did not return ' 'exists',
self.id,
'Provider %s used in target mode did not return exists', self.id
)
# Make a (shallow) copy of the desired state so that everything from
@@ -254,6 +253,4 @@ class BaseProvider(BaseSource):
return len(plan.changes)
def _apply(self, plan):
raise NotImplementedError(
'Abstract base class, _apply method ' 'missing'
)
raise NotImplementedError('Abstract base class, _apply method missing')

2
octodns/provider/plan.py
View File

@@ -73,7 +73,7 @@ class Plan(object):
existing_n = 0
self.log.debug(
'__init__: Creates=%d, Updates=%d, Deletes=%d ' 'Existing=%d',
'__init__: Creates=%d, Updates=%d, Deletes=%d Existing=%d',
self.change_counts['Create'],
self.change_counts['Update'],
self.change_counts['Delete'],

36
octodns/provider/yaml.py
View File

@@ -112,26 +112,6 @@ class YamlProvider(BaseProvider):
SUPPORTS_DYNAMIC = True
SUPPORTS_POOL_VALUE_STATUS = True
SUPPORTS_MULTIVALUE_PTR = True
SUPPORTS = set(
(
'A',
'AAAA',
'ALIAS',
'CAA',
'CNAME',
'DNAME',
'LOC',
'MX',
'NAPTR',
'NS',
'PTR',
'SSHFP',
'SPF',
'SRV',
'TXT',
'URLFWD',
)
)
def __init__(
self,
@@ -168,6 +148,22 @@ class YamlProvider(BaseProvider):
del args['log']
return self.__class__(**args)
@property
def SUPPORTS(self):
# The yaml provider supports all record types even those defined by 3rd
# party modules that we know nothing about, thus we dynamically return
# the types list that is registered in Record, everything that's know as
# of the point in time we're asked
return set(Record.registered_types().keys())
def supports(self, record):
# We're overriding this as a performance tweak, namely to avoid calling
# the implementation of the SUPPORTS property to create a set from a
# dict_keys every single time something checked whether we support a
# record, the answer is always yes so that's overkill and we can just
# return True here and be done with it
return True
@property
def SUPPORTS_ROOT_NS(self):
return self.supports_root_ns

10
octodns/record/__init__.py
View File

@@ -103,6 +103,10 @@ class Record(EqualityTupleMixin):
raise RecordException(msg)
cls._CLASSES[_type] = _class
@classmethod
def registered_types(cls):
return cls._CLASSES
@classmethod
def new(cls, zone, name, data, source=None, lenient=False):
reasons = []
@@ -641,7 +645,7 @@ class _DynamicMixin(object):
if len(values) == 1 and values[0].get('weight', 1) != 1:
reasons.append(
f'pool "{_id}" has single value with ' 'weight!=1'
f'pool "{_id}" has single value with weight!=1'
)
fallback = pool.get('fallback', None)
@@ -1324,7 +1328,7 @@ class _NsValue(object):
for value in data:
if not FQDN(str(value), allow_underscores=True).is_valid:
reasons.append(
f'Invalid NS value "{value}" is not ' 'a valid FQDN.'
f'Invalid NS value "{value}" is not a valid FQDN.'
)
elif not value.endswith('.'):
reasons.append(f'NS value "{value}" missing trailing .')
@@ -1536,7 +1540,7 @@ class SrvValue(EqualityTupleMixin):
and not FQDN(str(target), allow_underscores=True).is_valid
):
reasons.append(
f'Invalid SRV target "{target}" is not ' 'a valid FQDN.'
f'Invalid SRV target "{target}" is not a valid FQDN.'
)
except KeyError:
reasons.append('missing target')

2
octodns/record/geo_data.py
View File

@@ -288,7 +288,7 @@ geo_data = {
'SD': {'name': 'South Dakota'},
'TN': {'name': 'Tennessee'},
'TX': {'name': 'Texas'},
'UM': {'name': 'United States Minor Outlying ' 'Islands'},
'UM': {'name': 'United States Minor Outlying Islands'},
'UT': {'name': 'Utah'},
'VA': {'name': 'Virginia'},
'VI': {'name': 'Virgin Islands'},

8
octodns/source/base.py
View File

@@ -20,15 +20,15 @@ class BaseSource(object):
self.id = id
if not getattr(self, 'log', False):
raise NotImplementedError(
'Abstract base class, log property ' 'missing'
'Abstract base class, log property missing'
)
if not hasattr(self, 'SUPPORTS_GEO'):
raise NotImplementedError(
'Abstract base class, SUPPORTS_GEO ' 'property missing'
'Abstract base class, SUPPORTS_GEO property missing'
)
if not hasattr(self, 'SUPPORTS'):
raise NotImplementedError(
'Abstract base class, SUPPORTS ' 'property missing'
'Abstract base class, SUPPORTS property missing'
)
@property
@@ -51,7 +51,7 @@ class BaseSource(object):
True if the zone exists or False if it does not.
'''
raise NotImplementedError(
'Abstract base class, populate method ' 'missing'
'Abstract base class, populate method missing'
)
def supports(self, record):

2
octodns/source/envvar.py
View File

@@ -67,7 +67,7 @@ class EnvVarSource(BaseSource):
klass = self.__class__.__name__
self.log = logging.getLogger(f'{klass}[{id}]')
self.log.debug(
'__init__: id=%s, variable=%s, name=%s, ' 'ttl=%d',
'__init__: id=%s, variable=%s, name=%s, ttl=%d',
id,
variable,
name,

2
octodns/source/tinydns.py
View File

@@ -174,7 +174,7 @@ class TinyDnsBaseSource(BaseSource):
zone.add_record(record, lenient=lenient)
except SubzoneRecordException:
self.log.debug(
'_populate_normal: skipping subzone ' 'record=%s',
'_populate_normal: skipping subzone record=%s',
record,
)

10
octodns/zone.py
View File

@@ -151,7 +151,7 @@ class Zone(object):
continue
elif len(record.included) > 0 and target.id not in record.included:
self.log.debug(
'changes: skipping record=%s %s - %s not' ' included ',
'changes: skipping record=%s %s - %s not included ',
record.fqdn,
record._type,
target.id,
@@ -159,7 +159,7 @@ class Zone(object):
continue
elif target.id in record.excluded:
self.log.debug(
'changes: skipping record=%s %s - %s ' 'excluded ',
'changes: skipping record=%s %s - %s excluded ',
record.fqdn,
record._type,
target.id,
@@ -174,7 +174,7 @@ class Zone(object):
and target.id not in desired_record.included
):
self.log.debug(
'changes: skipping record=%s %s - %s' 'not included ',
'changes: skipping record=%s %s - %s not included',
record.fqdn,
record._type,
target.id,
@@ -221,7 +221,7 @@ class Zone(object):
continue
elif len(record.included) > 0 and target.id not in record.included:
self.log.debug(
'changes: skipping record=%s %s - %s not' ' included ',
'changes: skipping record=%s %s - %s not included ',
record.fqdn,
record._type,
target.id,
@@ -229,7 +229,7 @@ class Zone(object):
continue
elif target.id in record.excluded:
self.log.debug(
'changes: skipping record=%s %s - %s ' 'excluded ',
'changes: skipping record=%s %s - %s excluded ',
record.fqdn,
record._type,
target.id,

11
script/bootstrap
View File

@@ -29,8 +29,17 @@ if [ "$ENV" != "production" ]; then
python -m pip install -r requirements-dev.txt
fi
if [ ! -L ".git/hooks/pre-commit" ]; then
if [ -d ".git" ]; then
if [ -f ".git-blame-ignore-revs" ]; then
echo ""
echo "Setting blame.ignoreRevsFile to .git-blame-ingore-revs"
git config --local blame.ignoreRevsFile .git-blame-ignore-revs
fi
if [ ! -L ".git/hooks/pre-commit" ]; then
echo ""
echo "Installing pre-commit hook"
ln -s "$ROOT/.git_hooks_pre-commit" ".git/hooks/pre-commit"
fi
fi
echo ""

2
tests/test_octodns_manager.py
View File

@@ -486,7 +486,7 @@ class TestManager(TestCase):
sources=['in'],
)
self.assertEqual(
'output_provider=simple, does not support ' 'copy method',
'output_provider=simple, does not support copy method',
str(ctx.exception),
)

36
tests/test_octodns_provider_yaml.py
View File

@@ -16,7 +16,7 @@ from yaml import safe_load
from yaml.constructor import ConstructorError
from octodns.idna import idna_encode
from octodns.record import Create
from octodns.record import _NsValue, Create, Record, ValuesMixin
from octodns.provider import ProviderException
from octodns.provider.base import Plan
from octodns.provider.yaml import (
@@ -267,10 +267,40 @@ xn--dj-kia8a:
with self.assertRaises(SubzoneRecordException) as ctx:
source.populate(zone)
self.assertEqual(
'Record www.sub.unit.tests. is under a managed ' 'subzone',
'Record www.sub.unit.tests. is under a managed subzone',
str(ctx.exception),
)
def test_SUPPORTS(self):
source = YamlProvider('test', join(dirname(__file__), 'config'))
# make sure the provider supports all the registered types
self.assertEqual(Record.registered_types().keys(), source.SUPPORTS)
class YamlRecord(ValuesMixin, Record):
_type = 'YAML'
_value_type = _NsValue
# don't know anything about a yaml type
self.assertTrue('YAML' not in source.SUPPORTS)
# register it
Record.register_type(YamlRecord)
# when asked again we'll now include it in our list of supports
self.assertTrue('YAML' in source.SUPPORTS)
def test_supports(self):
source = YamlProvider('test', join(dirname(__file__), 'config'))
class DummyType(object):
def __init__(self, _type):
self._type = _type
# No matter what we check it's always supported
self.assertTrue(source.supports(DummyType(None)))
self.assertTrue(source.supports(DummyType(42)))
self.assertTrue(source.supports(DummyType('A')))
self.assertTrue(source.supports(DummyType(source)))
self.assertTrue(source.supports(DummyType(self)))
class TestSplitYamlProvider(TestCase):
def test_list_all_yaml_files(self):
@@ -494,7 +524,7 @@ class TestSplitYamlProvider(TestCase):
with self.assertRaises(SubzoneRecordException) as ctx:
source.populate(zone)
self.assertEqual(
'Record www.sub.unit.tests. is under a managed ' 'subzone',
'Record www.sub.unit.tests. is under a managed subzone',
str(ctx.exception),
)

42
tests/test_octodns_record.py
View File

@@ -62,7 +62,7 @@ class TestRecord(TestCase):
with self.assertRaises(RecordException) as ctx:
Record.register_type(None, 'A')
self.assertEqual(
'Type "A" already registered by ' 'octodns.record.ARecord',
'Type "A" already registered by octodns.record.ARecord',
str(ctx.exception),
)
@@ -70,6 +70,8 @@ class TestRecord(TestCase):
_type = 'AA'
_value_type = _NsValue
self.assertTrue('AA' not in Record.registered_types())
Record.register_type(AaRecord)
aa = Record.new(
self.zone,
@@ -78,6 +80,8 @@ class TestRecord(TestCase):
)
self.assertEqual(AaRecord, aa.__class__)
self.assertTrue('AA' in Record.registered_types())
def test_lowering(self):
record = ARecord(
self.zone, 'MiXeDcAsE', {'ttl': 30, 'type': 'A', 'value': '1.2.3.4'}
@@ -1868,7 +1872,7 @@ class TestRecordValidation(TestCase):
self.assertTrue(reason.startswith('invalid fqdn, "xxxx'))
self.assertTrue(
reason.endswith(
'.unit.tests." is too long at 254' ' chars, max is 253'
'.unit.tests." is too long at 254 chars, max is 253'
)
)
@@ -1882,7 +1886,7 @@ class TestRecordValidation(TestCase):
reason = ctx.exception.reasons[0]
self.assertTrue(reason.startswith('invalid label, "xxxx'))
self.assertTrue(
reason.endswith('xxx" is too long at 64' ' chars, max is 63')
reason.endswith('xxx" is too long at 64 chars, max is 63')
)
with self.assertRaises(ValidationError) as ctx:
@@ -1893,7 +1897,7 @@ class TestRecordValidation(TestCase):
reason = ctx.exception.reasons[0]
self.assertTrue(reason.startswith('invalid label, "xxxx'))
self.assertTrue(
reason.endswith('xxx" is too long at 64' ' chars, max is 63')
reason.endswith('xxx" is too long at 64 chars, max is 63')
)
# should not raise with dots
@@ -2526,7 +2530,7 @@ class TestRecordValidation(TestCase):
{'type': 'CNAME', 'ttl': 600, 'value': 'https://google.com'},
)
self.assertEqual(
['CNAME value "https://google.com" is not a valid ' 'FQDN'],
['CNAME value "https://google.com" is not a valid FQDN'],
ctx.exception.reasons,
)
@@ -2542,7 +2546,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
['CNAME value "https://google.com/a/b/c" is not a ' 'valid FQDN'],
['CNAME value "https://google.com/a/b/c" is not a valid FQDN'],
ctx.exception.reasons,
)
@@ -2554,7 +2558,7 @@ class TestRecordValidation(TestCase):
{'type': 'CNAME', 'ttl': 600, 'value': 'google.com/some/path'},
)
self.assertEqual(
['CNAME value "google.com/some/path" is not a valid ' 'FQDN'],
['CNAME value "google.com/some/path" is not a valid FQDN'],
ctx.exception.reasons,
)
@@ -3025,7 +3029,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
['Invalid MX exchange "100 foo.bar.com." is not a ' 'valid FQDN.'],
['Invalid MX exchange "100 foo.bar.com." is not a valid FQDN.'],
ctx.exception.reasons,
)
@@ -3136,7 +3140,7 @@ class TestRecordValidation(TestCase):
{'type': 'NS', 'ttl': 600, 'value': '100 foo.bar.com.'},
)
self.assertEqual(
['Invalid NS value "100 foo.bar.com." is not a ' 'valid FQDN.'],
['Invalid NS value "100 foo.bar.com." is not a valid FQDN.'],
ctx.exception.reasons,
)
@@ -3337,7 +3341,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
['unescaped ; in "this has some; ' 'semi-colons\\; in it"'],
['unescaped ; in "this has some; semi-colons\\; in it"'],
ctx.exception.reasons,
)
@@ -3541,7 +3545,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
['Invalid SRV target "100 foo.bar.com." is not a ' 'valid FQDN.'],
['Invalid SRV target "100 foo.bar.com." is not a valid FQDN.'],
ctx.exception.reasons,
)
@@ -3643,7 +3647,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
'invalid certificate_usage ' '"{value["certificate_usage"]}"',
'invalid certificate_usage "{value["certificate_usage"]}"',
ctx.exception.reasons,
)
@@ -3664,7 +3668,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
'invalid certificate_usage ' '"{value["certificate_usage"]}"',
'invalid certificate_usage "{value["certificate_usage"]}"',
ctx.exception.reasons,
)
@@ -3702,8 +3706,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
'invalid selector ' '"{value["selector"]}"',
ctx.exception.reasons,
'invalid selector "{value["selector"]}"', ctx.exception.reasons
)
# Invalid selector
@@ -3723,8 +3726,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
'invalid selector ' '"{value["selector"]}"',
ctx.exception.reasons,
'invalid selector "{value["selector"]}"', ctx.exception.reasons
)
# missing matching_type
@@ -3761,7 +3763,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
'invalid matching_type ' '"{value["matching_type"]}"',
'invalid matching_type "{value["matching_type"]}"',
ctx.exception.reasons,
)
@@ -3782,7 +3784,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
'invalid matching_type ' '"{value["matching_type"]}"',
'invalid matching_type "{value["matching_type"]}"',
ctx.exception.reasons,
)
@@ -3819,7 +3821,7 @@ class TestRecordValidation(TestCase):
},
)
self.assertEqual(
['unescaped ; in "this has some; semi-colons\\; ' 'in it"'],
['unescaped ; in "this has some; semi-colons\\; in it"'],
ctx.exception.reasons,
)