diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa9128..8a9b785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ ## v0.9.19 - 2022-??-?? - ??? +#### Noteworthy changes + +* Added support for automatic handling of IDNA (utf-8) zones. Everything is + stored IDNA encoded internally. For ASCII zones that's a noop. For zones with + utf-8 chars they will be converted and all internals/providers will see the + encoded version and work with it without any knowledge of it having been + converted. This means that all providers will automatically support IDNA as of + this version. IDNA zones will generally be displayed in the logs in their + decoded form. Both forms should be accepted in command line arguments. + Providers may need to be updated to display the decoded form in their logs, + until then they'd display the IDNA version. +* Support for configuring global processors that apply to all zones with + `manager.processors` + +#### Stuff + * Addressed shortcomings with YamlProvider.SUPPORTS in that it didn't include dynamically registered types, was a static list that could have drifted over time even ignoring 3rd party types. @@ -11,8 +27,9 @@ * Now that it's used as it needed to be YamlProvider overrides Provider.supports and just always says Yes so that any dynamically registered types will be supported. -* Support for configuring global processors that apply to all zones with - `manager.processors` +* Add TtlRestrictionFilter processor for adding ttl restriction/checking +* NameAllowlistFilter & NameRejectlistFilter implementations to support + filtering on record names to include/exclude records from management. ## v0.9.18 - 2022-08-14 - Subzone handling diff --git a/octodns/processor/base.py b/octodns/processor/base.py index ac5c155..c6c368e 100644 --- a/octodns/processor/base.py +++ b/octodns/processor/base.py @@ -10,6 +10,10 @@ from __future__ import ( ) +class ProcessorException(Exception): + pass + + class BaseProcessor(object): def __init__(self, name): self.name = name diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index f3aabf5..fdbd8ec 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -9,6 +9,8 @@ from __future__ import ( unicode_literals, ) +from re import compile as re_compile + from .base import BaseProcessor @@ -19,8 +21,8 @@ class TypeAllowlistFilter(BaseProcessor): processors: only-a-and-aaaa: - class: octodns.processor.filter.TypeRejectlistFilter - rejectlist: + class: octodns.processor.filter.TypeAllowlistFilter + allowlist: - A - AAAA @@ -35,7 +37,7 @@ class TypeAllowlistFilter(BaseProcessor): ''' def __init__(self, name, allowlist): - super(TypeAllowlistFilter, self).__init__(name) + super().__init__(name) self.allowlist = set(allowlist) def _process(self, zone, *args, **kwargs): @@ -71,7 +73,7 @@ class TypeRejectlistFilter(BaseProcessor): ''' def __init__(self, name, rejectlist): - super(TypeRejectlistFilter, self).__init__(name) + super().__init__(name) self.rejectlist = set(rejectlist) def _process(self, zone, *args, **kwargs): @@ -83,3 +85,113 @@ class TypeRejectlistFilter(BaseProcessor): process_source_zone = _process process_target_zone = _process + + +class _NameBaseFilter(BaseProcessor): + def __init__(self, name, _list): + super().__init__(name) + exact = set() + regex = [] + for pattern in _list: + if pattern.startswith('/'): + regex.append(re_compile(pattern[1:-1])) + else: + exact.add(pattern) + self.exact = exact + self.regex = regex + + +class NameAllowlistFilter(_NameBaseFilter): + '''Only manage records with names that match the provider patterns + + Example usage: + + processors: + only-these: + class: octodns.processor.filter.NameAllowlistFilter + allowlist: + # exact string match + - www + # contains/substring match + - /substring/ + # regex pattern match + - /some-pattern-\\d\\+/ + # regex - anchored so has to match start to end + - /^start-.+-end$/ + + zones: + exxampled.com.: + sources: + - config + processors: + - only-these + targets: + - route53 + ''' + + def __init__(self, name, allowlist): + super().__init__(name, allowlist) + + def _process(self, zone, *args, **kwargs): + for record in zone.records: + name = record.name + if name in self.exact: + continue + elif any(r.search(name) for r in self.regex): + continue + + zone.remove_record(record) + + return zone + + process_source_zone = _process + process_target_zone = _process + + +class NameRejectlistFilter(_NameBaseFilter): + '''Reject managing records with names that match the provider patterns + + Example usage: + + processors: + not-these: + class: octodns.processor.filter.NameRejectlistFilter + rejectlist: + # exact string match + - www + # contains/substring match + - /substring/ + # regex pattern match + - /some-pattern-\\d\\+/ + # regex - anchored so has to match start to end + - /^start-.+-end$/ + + zones: + exxampled.com.: + sources: + - config + processors: + - not-these + targets: + - route53 + ''' + + def __init__(self, name, rejectlist): + super().__init__(name, rejectlist) + + def _process(self, zone, *args, **kwargs): + for record in zone.records: + name = record.name + if name in self.exact: + zone.remove_record(record) + continue + + for regex in self.regex: + if regex.search(name): + zone.remove_record(record) + break + + return zone + + process_source_zone = _process + process_target_zone = _process diff --git a/octodns/processor/restrict.py b/octodns/processor/restrict.py new file mode 100644 index 0000000..1903995 --- /dev/null +++ b/octodns/processor/restrict.py @@ -0,0 +1,83 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from .base import BaseProcessor, ProcessorException + + +class RestrictionException(ProcessorException): + pass + + +class TtlRestrictionFilter(BaseProcessor): + ''' + Ensure that configured TTLs are between a configured minimum and maximum or + in an allowed set of values. + + The default minimum is 1 (the behavior of 0 is undefined spec-wise) and the + default maximum is 604800 (seven days.) allowed_ttls is only used when + explicitly configured and min and max are ignored in that case. + + Example usage: + + processors: + min-max-ttl: + class: octodns.processor.restrict.TtlRestrictionFilter + min_ttl: 60 + max_ttl: 3600 + # allowed_ttls: [300, 900, 3600] + + zones: + exxampled.com.: + sources: + - config + processors: + - min-max-ttl + targets: + - azure + + The restriction can be skipped for specific records by setting the lenient + flag, e.g. + + a: + octodns: + lenient: true + ttl: 0 + value: 1.2.3.4 + + The higher level lenient flags are not checked as it would make more sense + to just avoid enabling the processor in those cases. + ''' + + SEVEN_DAYS = 60 * 60 * 24 * 7 + + def __init__(self, name, min_ttl=1, max_ttl=SEVEN_DAYS, allowed_ttls=None): + super().__init__(name) + self.min_ttl = min_ttl + self.max_ttl = max_ttl + self.allowed_ttls = set(allowed_ttls) if allowed_ttls else None + + def process_source_zone(self, zone, *args, **kwargs): + for record in zone.records: + if record._octodns.get('lenient'): + continue + if self.allowed_ttls and record.ttl not in self.allowed_ttls: + raise RestrictionException( + f'{record.fqdn} ttl={record.ttl} not an allowed value, allowed_ttls={self.allowed_ttls}' + ) + elif record.ttl < self.min_ttl: + raise RestrictionException( + f'{record.fqdn} ttl={record.ttl} too low, min_ttl={self.min_ttl}' + ) + elif record.ttl > self.max_ttl: + raise RestrictionException( + f'{record.fqdn} ttl={record.ttl} too high, max_ttl={self.max_ttl}' + ) + return zone diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 859677d..aa6f5ff 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -11,7 +11,12 @@ from __future__ import ( from unittest import TestCase -from octodns.processor.filter import TypeAllowlistFilter, TypeRejectlistFilter +from octodns.processor.filter import ( + NameAllowlistFilter, + NameRejectlistFilter, + TypeAllowlistFilter, + TypeRejectlistFilter, +) from octodns.record import Record from octodns.zone import Zone @@ -76,3 +81,83 @@ class TestTypeRejectListFilter(TestCase): filter_a_aaaa = TypeRejectlistFilter('not-a-aaaa', set(('A', 'AAAA'))) got = filter_a_aaaa.process_target_zone(zone.copy()) self.assertEqual(['txt', 'txt2'], sorted([r.name for r in got.records])) + + +class TestNameAllowListFilter(TestCase): + zone = Zone('unit.tests.', []) + matches = Record.new( + zone, 'matches', {'type': 'A', 'ttl': 42, 'value': '1.2.3.4'} + ) + zone.add_record(matches) + doesnt = Record.new( + zone, 'doesnt', {'type': 'A', 'ttl': 42, 'value': '2.3.4.5'} + ) + zone.add_record(doesnt) + matchable1 = Record.new( + zone, 'start-f43ad96-end', {'type': 'A', 'ttl': 42, 'value': '3.4.5.6'} + ) + zone.add_record(matchable1) + matchable2 = Record.new( + zone, 'start-a3b444c-end', {'type': 'A', 'ttl': 42, 'value': '4.5.6.7'} + ) + zone.add_record(matchable2) + + def test_exact(self): + allows = NameAllowlistFilter('exact', ('matches',)) + + self.assertEqual(4, len(self.zone.records)) + filtered = allows.process_source_zone(self.zone.copy()) + self.assertEqual(1, len(filtered.records)) + self.assertEqual(['matches'], [r.name for r in filtered.records]) + + def test_regex(self): + allows = NameAllowlistFilter('exact', ('/^start-.+-end$/',)) + + self.assertEqual(4, len(self.zone.records)) + filtered = allows.process_source_zone(self.zone.copy()) + self.assertEqual(2, len(filtered.records)) + self.assertEqual( + ['start-a3b444c-end', 'start-f43ad96-end'], + sorted([r.name for r in filtered.records]), + ) + + +class TestNameRejectListFilter(TestCase): + zone = Zone('unit.tests.', []) + matches = Record.new( + zone, 'matches', {'type': 'A', 'ttl': 42, 'value': '1.2.3.4'} + ) + zone.add_record(matches) + doesnt = Record.new( + zone, 'doesnt', {'type': 'A', 'ttl': 42, 'value': '2.3.4.5'} + ) + zone.add_record(doesnt) + matchable1 = Record.new( + zone, 'start-f43ad96-end', {'type': 'A', 'ttl': 42, 'value': '3.4.5.6'} + ) + zone.add_record(matchable1) + matchable2 = Record.new( + zone, 'start-a3b444c-end', {'type': 'A', 'ttl': 42, 'value': '4.5.6.7'} + ) + zone.add_record(matchable2) + + def test_exact(self): + rejects = NameRejectlistFilter('exact', ('matches',)) + + self.assertEqual(4, len(self.zone.records)) + filtered = rejects.process_source_zone(self.zone.copy()) + self.assertEqual(3, len(filtered.records)) + self.assertEqual( + ['doesnt', 'start-a3b444c-end', 'start-f43ad96-end'], + sorted([r.name for r in filtered.records]), + ) + + def test_regex(self): + rejects = NameRejectlistFilter('exact', ('/^start-.+-end$/',)) + + self.assertEqual(4, len(self.zone.records)) + filtered = rejects.process_source_zone(self.zone.copy()) + self.assertEqual(2, len(filtered.records)) + self.assertEqual( + ['doesnt', 'matches'], sorted([r.name for r in filtered.records]) + ) diff --git a/tests/test_octodns_processor_restrict.py b/tests/test_octodns_processor_restrict.py new file mode 100644 index 0000000..4848ae6 --- /dev/null +++ b/tests/test_octodns_processor_restrict.py @@ -0,0 +1,113 @@ +from unittest import TestCase + +from octodns.processor.restrict import ( + RestrictionException, + TtlRestrictionFilter, +) +from octodns.record import Record +from octodns.zone import Zone + + +class TestTtlRestrictionFilter(TestCase): + def test_restrict_ttl(self): + # configured values + restrictor = TtlRestrictionFilter('test', min_ttl=32, max_ttl=1024) + + zone = Zone('unit.tests.', []) + good = Record.new( + zone, 'good', {'type': 'A', 'ttl': 42, 'value': '1.2.3.4'} + ) + zone.add_record(good) + + restricted = restrictor.process_source_zone(zone) + self.assertEqual(zone.records, restricted.records) + + # too low + low = Record.new( + zone, 'low', {'type': 'A', 'ttl': 16, 'value': '1.2.3.4'} + ) + copy = zone.copy() + copy.add_record(low) + with self.assertRaises(RestrictionException) as ctx: + restrictor.process_source_zone(copy) + self.assertEqual( + 'low.unit.tests. ttl=16 too low, min_ttl=32', str(ctx.exception) + ) + + # with lenient set, we can go lower + lenient = Record.new( + zone, + 'low', + { + 'octodns': {'lenient': True}, + 'type': 'A', + 'ttl': 16, + 'value': '1.2.3.4', + }, + ) + copy = zone.copy() + copy.add_record(lenient) + restricted = restrictor.process_source_zone(copy) + self.assertEqual(copy.records, restricted.records) + + # too high + high = Record.new( + zone, 'high', {'type': 'A', 'ttl': 2048, 'value': '1.2.3.4'} + ) + copy = zone.copy() + copy.add_record(high) + with self.assertRaises(RestrictionException) as ctx: + restrictor.process_source_zone(copy) + self.assertEqual( + 'high.unit.tests. ttl=2048 too high, max_ttl=1024', + str(ctx.exception), + ) + + # too low defaults + restrictor = TtlRestrictionFilter('test') + low = Record.new( + zone, 'low', {'type': 'A', 'ttl': 0, 'value': '1.2.3.4'} + ) + copy = zone.copy() + copy.add_record(low) + with self.assertRaises(RestrictionException) as ctx: + restrictor.process_source_zone(copy) + self.assertEqual( + 'low.unit.tests. ttl=0 too low, min_ttl=1', str(ctx.exception) + ) + + # too high defaults + high = Record.new( + zone, 'high', {'type': 'A', 'ttl': 999999, 'value': '1.2.3.4'} + ) + copy = zone.copy() + copy.add_record(high) + with self.assertRaises(RestrictionException) as ctx: + restrictor.process_source_zone(copy) + self.assertEqual( + 'high.unit.tests. ttl=999999 too high, max_ttl=604800', + str(ctx.exception), + ) + + # allowed_ttls + restrictor = TtlRestrictionFilter('test', allowed_ttls=[42, 300]) + + # add 300 (42 is already there) + another = Record.new( + zone, 'another', {'type': 'A', 'ttl': 300, 'value': '4.5.6.7'} + ) + zone.add_record(another) + + # 42 and 300 are allowed through + restricted = restrictor.process_source_zone(zone) + self.assertEqual(zone.records, restricted.records) + + # 16 is not + copy = zone.copy() + copy.add_record(low) + with self.assertRaises(RestrictionException) as ctx: + restrictor.process_source_zone(copy) + self.assertEqual( + 'low.unit.tests. ttl=0 not an allowed value, allowed_ttls={42, 300}', + str(ctx.exception), + )