From b139d266c9a2ec1c236c4962bdb5b70029039b80 Mon Sep 17 00:00:00 2001 From: Aquifoliales <103569748+Aquifoliales@users.noreply.github.com> Date: Wed, 4 May 2022 11:01:50 +0200 Subject: [PATCH 01/16] Support for TLSA record, https://www.rfc-editor.org/rfc/rfc6698.txt --- docs/records.md | 1 + octodns/record/__init__.py | 65 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/docs/records.md b/docs/records.md index 182a109..30d689a 100644 --- a/docs/records.md +++ b/docs/records.md @@ -18,6 +18,7 @@ OctoDNS supports the following record types: * `SPF` * `SRV` * `SSHFP` +* `TLSA` * `TXT` * `URLFWD` diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 6fe3f90..ce73752 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1517,6 +1517,71 @@ class SrvRecord(ValuesMixin, Record): Record.register_type(SrvRecord) +class TlsaValue(EqualityTupleMixin): + + @classmethod + def validate(cls, data, _type): + if not isinstance(data, (list, tuple)): + data = (data,) + reasons = [] + for value in data: + try: + certificate_usage = int(value.get('certificate_usage', 0)) + if certificate_usage < 0 or certificate_usage > 3: + reasons.append(f'invalid certificate_usage "{certificate_usage}"') + except ValueError: + reasons.append(f'invalid certificate_usage "{value["certificate_usage"]}"') + + try: + selector = int(value.get('selector', 0)) + if selector < 0 or selector > 1: + reasons.append(f'invalid selector "{selector}"') + except ValueError: + reasons.append(f'invalid selector "{value["selector"]}"') + + try: + matching_type = int(value.get('matching_type', 0)) + if matching_type < 0 or matching_type > 2: + reasons.append(f'invalid matching_type "{matching_type}"') + except ValueError: + reasons.append(f'invalid matching_type "{value["matching_type"]}"') + + if 'certificate_association_data' not in value: + reasons.append('missing certificate_association_data') + return reasons + + @classmethod + def process(cls, values): + return [TlsaValue(v) for v in values] + + def __init__(self, value): + self.certificate_usage = int(value.get('certificate_usage', 0)) + self.selector = int(value.get('selector', 0)) + self.matching_type = int(value.get('matching_type', 0)) + self.certificate_association_data = value['certificate_association_data'] + + @property + def data(self): + return { + 'certificate_usage': self.certificate_usage, + 'selector': self.selector, + 'matching_type': self.matching_type, + 'certificate_association_data': self.certificate_association_data, + } + + def _equality_tuple(self): + return (self.certificate_usage, self.selector, self.matching_type, self.certificate_association_data) + + def __repr__(self): + return f'{self.certificate_usage} {self.selector} {self.matching_type}"{self.certificate_association_data}"' + + +class TlsaRecord(ValuesMixin, Record): + _type = 'TLSA' + _value_type = TlsaValue + +Record.register_type(TlsaRecord) + class _TxtValue(_ChunkedValue): pass From 084d537c943fc282411d08cdc1f83b9d4aadd755 Mon Sep 17 00:00:00 2001 From: Aquifoliales <103569748+Aquifoliales@users.noreply.github.com> Date: Tue, 10 May 2022 09:50:57 +0200 Subject: [PATCH 02/16] Fixed testing for TLSA record type. --- octodns/record/__init__.py | 26 +++- tests/test_octodns_record.py | 245 ++++++++++++++++++++++++++++++++++- 2 files changed, 262 insertions(+), 9 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index ce73752..ddb8b92 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1528,9 +1528,11 @@ class TlsaValue(EqualityTupleMixin): try: certificate_usage = int(value.get('certificate_usage', 0)) if certificate_usage < 0 or certificate_usage > 3: - reasons.append(f'invalid certificate_usage "{certificate_usage}"') + reasons.append(f'invalid certificate_usage ' + f'"{certificate_usage}"') except ValueError: - reasons.append(f'invalid certificate_usage "{value["certificate_usage"]}"') + reasons.append(f'invalid certificate_usage ' + f'"{value["certificate_usage"]}"') try: selector = int(value.get('selector', 0)) @@ -1544,8 +1546,15 @@ class TlsaValue(EqualityTupleMixin): if matching_type < 0 or matching_type > 2: reasons.append(f'invalid matching_type "{matching_type}"') except ValueError: - reasons.append(f'invalid matching_type "{value["matching_type"]}"') + reasons.append(f'invalid matching_type ' + f'"{value["matching_type"]}"') + if 'certificate_usage' not in value: + reasons.append('missing certificate_usage') + if 'selector' not in value: + reasons.append('missing selector') + if 'matching_type' not in value: + reasons.append('missing matching_type') if 'certificate_association_data' not in value: reasons.append('missing certificate_association_data') return reasons @@ -1558,7 +1567,8 @@ class TlsaValue(EqualityTupleMixin): self.certificate_usage = int(value.get('certificate_usage', 0)) self.selector = int(value.get('selector', 0)) self.matching_type = int(value.get('matching_type', 0)) - self.certificate_association_data = value['certificate_association_data'] + self.certificate_association_data = \ + value['certificate_association_data'] @property def data(self): @@ -1570,18 +1580,22 @@ class TlsaValue(EqualityTupleMixin): } def _equality_tuple(self): - return (self.certificate_usage, self.selector, self.matching_type, self.certificate_association_data) + return (self.certificate_usage, self.selector, self.matching_type, + self.certificate_association_data) def __repr__(self): - return f'{self.certificate_usage} {self.selector} {self.matching_type}"{self.certificate_association_data}"' + return f"'{self.certificate_usage} {self.selector} '" \ + f"'{self.matching_type} {self.certificate_association_data}'" class TlsaRecord(ValuesMixin, Record): _type = 'TLSA' _value_type = TlsaValue + Record.register_type(TlsaRecord) + class _TxtValue(_ChunkedValue): pass diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index fd3f70f..dbd2666 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -11,9 +11,9 @@ from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, LocRecord, \ LocValue, MxRecord, MxValue, NaptrRecord, NaptrValue, NsRecord, \ PtrRecord, Record, RecordException, SshfpRecord, SshfpValue, SpfRecord, \ - SrvRecord, SrvValue, TxtRecord, Update, UrlfwdRecord, UrlfwdValue, \ - ValidationError, _Dynamic, _DynamicPool, _DynamicRule, _NsValue, \ - ValuesMixin + SrvRecord, SrvValue, TlsaRecord, TxtRecord, Update, UrlfwdRecord, \ + UrlfwdValue, ValidationError, _Dynamic, _DynamicPool, _DynamicRule, \ + _NsValue, ValuesMixin from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -907,6 +907,80 @@ class TestRecord(TestCase): # __repr__ doesn't blow up a.__repr__() + def test_tlsa(self): + a_values = [{ + 'certificate_usage': 1, + 'selector': 1, + 'matching_type': 1, + 'certificate_association_data': 'ABABABABABABABABAB', + }, { + 'certificate_usage': 2, + 'selector': 0, + 'matching_type': 2, + 'certificate_association_data': 'ABABABABABABABABAC', + }] + a_data = {'ttl': 30, 'values': a_values} + a = TlsaRecord(self.zone, 'a', a_data) + self.assertEqual('a.unit.tests.', a.fqdn) + self.assertEqual('a', a.name) + self.assertEqual(30, a.ttl) + self.assertEqual(a_values[0]['certificate_usage'], a.values[0].certificate_usage) + self.assertEqual(a_values[0]['selector'], a.values[0].selector) + self.assertEqual(a_values[0]['matching_type'], a.values[0].matching_type) + self.assertEqual(a_values[0]['certificate_association_data'], a.values[0].certificate_association_data) + + self.assertEqual(a_values[1]['certificate_usage'], a.values[1].certificate_usage) + self.assertEqual(a_values[1]['selector'], a.values[1].selector) + self.assertEqual(a_values[1]['matching_type'], a.values[1].matching_type) + self.assertEqual(a_values[1]['certificate_association_data'], a.values[1].certificate_association_data) + self.assertEqual(a_data, a.data) + + b_value = { + 'certificate_usage': 0, + 'selector': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAAAA', + } + b_data = {'ttl': 30, 'value': b_value} + b = TlsaRecord(self.zone, 'b', b_data) + self.assertEqual(b_value['certificate_usage'], b.values[0].certificate_usage) + self.assertEqual(b_value['selector'], b.values[0].selector) + self.assertEqual(b_value['matching_type'], b.values[0].matching_type) + self.assertEqual(b_value['certificate_association_data'], b.values[0].certificate_association_data) + self.assertEqual(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in certificate_usage causes change + other = TlsaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].certificate_usage = 0 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in selector causes change + other = TlsaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].selector = 0 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in matching_type causes change + other = TlsaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].matching_type = 0 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in certificate_association_data causes change + other = TlsaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].certificate_association_data = 'AAAAAAAAAAAAA' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # __repr__ doesn't blow up + a.__repr__() + + def test_txt(self): a_values = ['a one', 'a two'] b_value = 'b other' @@ -3187,6 +3261,171 @@ class TestRecordValidation(TestCase): self.assertEqual(['Invalid SRV target "100 foo.bar.com." is not a ' 'valid FQDN.'], ctx.exception.reasons) + def test_TLSA(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'selector' : 0, + 'matching_type' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + + # missing certificate_association_data + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'selector' : 0, + 'matching_type' : 0 + } + }) + self.assertEqual(['missing certificate_association_data'], + ctx.exception.reasons) + + # missing certificate_usage + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'selector' : 0, + 'matching_type' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual(['missing certificate_usage'], + ctx.exception.reasons) + + # False certificate_usage + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 4, + 'selector' : 0, + 'matching_type' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual('invalid certificate_usage ' + '"{value["certificate_usage"]}"', + ctx.exception.reasons) + + # Invalid certificate_usage + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 'XYZ', + 'selector' : 0, + 'matching_type' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual('invalid certificate_usage ' + '"{value["certificate_usage"]}"', + ctx.exception.reasons) + + # missing selector + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'matching_type' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual(['missing selector'], + ctx.exception.reasons) + + # False selector + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'selector' : 4, + 'matching_type' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual('invalid selector ' + '"{value["selector"]}"', + ctx.exception.reasons) + + # Invalid selector + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'selector' : 'XYZ', + 'matching_type' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual('invalid selector ' + '"{value["selector"]}"', + ctx.exception.reasons) + + # missing matching_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'selector' : 0, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual(['missing matching_type'], + ctx.exception.reasons) + + # False matching_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'selector' : 1, + 'matching_type' : 3, + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual('invalid matching_type ' + '"{value["matching_type"]}"', + ctx.exception.reasons) + + # Invalid matching_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'value' : { + 'certificate_usage' : 0, + 'selector' : 1, + 'matching_type' : 'XYZ', + 'certificate_association_data' : 'AAAAAAAAAAAAA' + } + }) + self.assertEqual('invalid matching_type ' + '"{value["matching_type"]}"', + ctx.exception.reasons) + def test_TXT(self): # doesn't blow up (name & zone here don't make any sense, but not # important) From 3cdefc505844e3253a7e2b179b7a41306ca557a1 Mon Sep 17 00:00:00 2001 From: Aquifoliales <103569748+Aquifoliales@users.noreply.github.com> Date: Tue, 10 May 2022 14:58:35 +0200 Subject: [PATCH 03/16] Adaption for Linting --- tests/test_octodns_record.py | 187 ++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 89 deletions(-) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index dbd2666..2649fb9 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -924,17 +924,24 @@ class TestRecord(TestCase): self.assertEqual('a.unit.tests.', a.fqdn) self.assertEqual('a', a.name) self.assertEqual(30, a.ttl) - self.assertEqual(a_values[0]['certificate_usage'], a.values[0].certificate_usage) + self.assertEqual(a_values[0]['certificate_usage'], + a.values[0].certificate_usage) self.assertEqual(a_values[0]['selector'], a.values[0].selector) - self.assertEqual(a_values[0]['matching_type'], a.values[0].matching_type) - self.assertEqual(a_values[0]['certificate_association_data'], a.values[0].certificate_association_data) - - self.assertEqual(a_values[1]['certificate_usage'], a.values[1].certificate_usage) - self.assertEqual(a_values[1]['selector'], a.values[1].selector) - self.assertEqual(a_values[1]['matching_type'], a.values[1].matching_type) - self.assertEqual(a_values[1]['certificate_association_data'], a.values[1].certificate_association_data) + self.assertEqual(a_values[0]['matching_type'], + a.values[0].matching_type) + self.assertEqual(a_values[0]['certificate_association_data'], + a.values[0].certificate_association_data) + + self.assertEqual(a_values[1]['certificate_usage'], + a.values[1].certificate_usage) + self.assertEqual(a_values[1]['selector'], + a.values[1].selector) + self.assertEqual(a_values[1]['matching_type'], + a.values[1].matching_type) + self.assertEqual(a_values[1]['certificate_association_data'], + a.values[1].certificate_association_data) self.assertEqual(a_data, a.data) - + b_value = { 'certificate_usage': 0, 'selector': 0, @@ -943,10 +950,13 @@ class TestRecord(TestCase): } b_data = {'ttl': 30, 'value': b_value} b = TlsaRecord(self.zone, 'b', b_data) - self.assertEqual(b_value['certificate_usage'], b.values[0].certificate_usage) + self.assertEqual(b_value['certificate_usage'], + b.values[0].certificate_usage) self.assertEqual(b_value['selector'], b.values[0].selector) - self.assertEqual(b_value['matching_type'], b.values[0].matching_type) - self.assertEqual(b_value['certificate_association_data'], b.values[0].certificate_association_data) + self.assertEqual(b_value['matching_type'], + b.values[0].matching_type) + self.assertEqual(b_value['certificate_association_data'], + b.values[0].certificate_association_data) self.assertEqual(b_data, b.data) target = SimpleProvider() @@ -976,11 +986,10 @@ class TestRecord(TestCase): change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) - + # __repr__ doesn't blow up a.__repr__() - def test_txt(self): a_values = ['a one', 'a two'] b_value = 'b other' @@ -3266,11 +3275,11 @@ class TestRecordValidation(TestCase): Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'selector' : 0, - 'matching_type' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 0, + 'selector': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) @@ -3279,152 +3288,152 @@ class TestRecordValidation(TestCase): Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'selector' : 0, - 'matching_type' : 0 + 'value': { + 'certificate_usage': 0, + 'selector': 0, + 'matching_type': 0 } }) self.assertEqual(['missing certificate_association_data'], - ctx.exception.reasons) + ctx.exception.reasons) - # missing certificate_usage + # missing certificate_usage with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'selector' : 0, - 'matching_type' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'selector': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual(['missing certificate_usage'], - ctx.exception.reasons) + ctx.exception.reasons) # False certificate_usage with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 4, - 'selector' : 0, - 'matching_type' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 4, + 'selector': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual('invalid certificate_usage ' - '"{value["certificate_usage"]}"', - ctx.exception.reasons) + '"{value["certificate_usage"]}"', + ctx.exception.reasons) # Invalid certificate_usage with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 'XYZ', - 'selector' : 0, - 'matching_type' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 'XYZ', + 'selector': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual('invalid certificate_usage ' - '"{value["certificate_usage"]}"', - ctx.exception.reasons) + '"{value["certificate_usage"]}"', + ctx.exception.reasons) - # missing selector + # missing selector with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'matching_type' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual(['missing selector'], - ctx.exception.reasons) + ctx.exception.reasons) - # False selector + # False selector with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'selector' : 4, - 'matching_type' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 0, + 'selector': 4, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual('invalid selector ' - '"{value["selector"]}"', - ctx.exception.reasons) + '"{value["selector"]}"', + ctx.exception.reasons) - # Invalid selector + # Invalid selector with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'selector' : 'XYZ', - 'matching_type' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 0, + 'selector': 'XYZ', + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual('invalid selector ' - '"{value["selector"]}"', - ctx.exception.reasons) - - # missing matching_type + '"{value["selector"]}"', + ctx.exception.reasons) + + # missing matching_type with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'selector' : 0, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 0, + 'selector': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual(['missing matching_type'], - ctx.exception.reasons) + ctx.exception.reasons) - # False matching_type + # False matching_type with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'selector' : 1, - 'matching_type' : 3, - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 0, + 'selector': 1, + 'matching_type': 3, + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual('invalid matching_type ' - '"{value["matching_type"]}"', - ctx.exception.reasons) + '"{value["matching_type"]}"', + ctx.exception.reasons) - # Invalid matching_type + # Invalid matching_type with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'TLSA', 'ttl': 600, - 'value' : { - 'certificate_usage' : 0, - 'selector' : 1, - 'matching_type' : 'XYZ', - 'certificate_association_data' : 'AAAAAAAAAAAAA' + 'value': { + 'certificate_usage': 0, + 'selector': 1, + 'matching_type': 'XYZ', + 'certificate_association_data': 'AAAAAAAAAAAAA' } }) self.assertEqual('invalid matching_type ' - '"{value["matching_type"]}"', - ctx.exception.reasons) + '"{value["matching_type"]}"', + ctx.exception.reasons) def test_TXT(self): # doesn't blow up (name & zone here don't make any sense, but not From 2401a7318c95ae794ddc2ec58b011461d8d386d6 Mon Sep 17 00:00:00 2001 From: Aquifoliales <103569748+Aquifoliales@users.noreply.github.com> Date: Tue, 10 May 2022 15:56:33 +0200 Subject: [PATCH 04/16] Fixed testing, TLSA record ready. --- tests/test_octodns_record.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 2649fb9..a406d96 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -3282,6 +3282,25 @@ class TestRecordValidation(TestCase): 'certificate_association_data': 'AAAAAAAAAAAAA' } }) + # Multi value, second missing certificate usage + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TLSA', + 'ttl': 600, + 'values': [{ + 'certificate_usage': 0, + 'selector': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' + }, { + 'selector': 0, + 'matching_type': 0, + 'certificate_association_data': 'AAAAAAAAAAAAA' + } + ] + }) + self.assertEqual(['missing certificate_usage'], + ctx.exception.reasons) # missing certificate_association_data with self.assertRaises(ValidationError) as ctx: From 8b2bfa5deaaa4c057d755ba1eaebc852fb1c3a58 Mon Sep 17 00:00:00 2001 From: Aquifoliales <103569748+Aquifoliales@users.noreply.github.com> Date: Wed, 18 May 2022 14:43:12 +0200 Subject: [PATCH 05/16] SSHFP support for Source AXFR/Zonefile --- octodns/source/axfr.py | 18 +++++++++++++++++- tests/test_octodns_source_axfr.py | 8 ++++---- tests/zones/unit.tests.tst | 4 ++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index 57645e3..4c33d97 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -26,7 +26,7 @@ class AxfrBaseSource(BaseSource): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', 'PTR', - 'SPF', 'SRV', 'TXT')) + 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id): super(AxfrBaseSource, self).__init__(id) @@ -135,6 +135,22 @@ class AxfrBaseSource(BaseSource): 'values': values } + def _data_for_SSHFP(self, _type, records): + values = [] + for record in records: + algorithm, fingerprint_type, fingerprint = \ + record['value'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint_type': fingerprint_type, + 'fingerprint': fingerprint, + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 9e34d06..bb548b1 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -35,7 +35,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEqual(15, len(got.records)) + self.assertEqual(16, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -72,18 +72,18 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEqual(15, len(valid.records)) + self.assertEqual(16, len(valid.records)) def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEqual(15, len(valid.records)) + self.assertEqual(16, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEqual(15, len(again.records)) + self.assertEqual(16, len(again.records)) # bust the cache del self.source._zone_records[valid.name] diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index b916b81..82549ea 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -13,6 +13,10 @@ $ORIGIN unit.tests. under 3600 IN NS ns1.unit.tests. under 3600 IN NS ns2.unit.tests. +; SSHFP Records +@ 600 IN SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 +@ 600 IN SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 + ; CAA Records caa 1800 IN CAA 0 issue "ca.unit.tests" caa 1800 IN CAA 0 iodef "mailto:admin@unit.tests" From 4241cb52779504a810a70be7240906a3cfed65bd Mon Sep 17 00:00:00 2001 From: Johan Kok Date: Fri, 27 May 2022 16:36:39 +0200 Subject: [PATCH 06/16] Updated link to providers in README.md The supported providers table was renamed in 9b79c98a094c0c713b5c1cffd492ffd68e83f7b1, this would fix the link to the table. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 43e1db1..93ff704 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ The architecture is pluggable and the tooling is flexible to make it applicable ### Workspace -Running through the following commands will install the latest release of OctoDNS and set up a place for your config files to live. To determine if provider specific requirements are necessary see the [Supported providers table](#supported-providers) below. +Running through the following commands will install the latest release of OctoDNS and set up a place for your config files to live. To determine if provider specific requirements are necessary see the [providers table](#providers) below. ```shell $ mkdir dns From 04b7bf0ac2f9cf132d8466e32feab4d83363d9a2 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Sun, 29 May 2022 13:16:01 +0800 Subject: [PATCH 07/16] Fix typos --- CHANGELOG.md | 6 +++--- README.md | 8 ++++---- octodns/processor/awsacm.py | 2 +- octodns/processor/base.py | 2 +- octodns/provider/azuredns.py | 2 +- octodns/provider/cloudflare.py | 2 +- octodns/provider/constellix.py | 2 +- octodns/provider/digitalocean.py | 2 +- octodns/provider/dnsimple.py | 2 +- octodns/provider/dnsmadeeasy.py | 2 +- octodns/provider/dyn.py | 2 +- octodns/provider/easydns.py | 2 +- octodns/provider/edgedns.py | 2 +- octodns/provider/etc_hosts.py | 2 +- octodns/provider/fastdns.py | 2 +- octodns/provider/gandi.py | 2 +- octodns/provider/gcore.py | 2 +- octodns/provider/googlecloud.py | 2 +- octodns/provider/hetzner.py | 2 +- octodns/provider/mythicbeasts.py | 2 +- octodns/provider/ns1.py | 2 +- octodns/provider/ovh.py | 2 +- octodns/provider/powerdns.py | 2 +- octodns/provider/rackspace.py | 2 +- octodns/provider/route53.py | 2 +- octodns/provider/selectel.py | 2 +- octodns/provider/transip.py | 2 +- octodns/provider/ultra.py | 2 +- octodns/source/base.py | 2 +- tests/test_octodns_manager.py | 6 +++--- tests/test_octodns_provider_base.py | 6 +++--- tests/test_octodns_record.py | 2 +- 32 files changed, 41 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44209eb..5bf11a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -162,7 +162,7 @@ unless the `processors` key is present in zone configs. * Fixes NS1 provider's geotarget limitation of using `NA` continent. Now, when `NA` is used in geos it considers **all** the countries of `North America` - insted of just `us-east`, `us-west` and `us-central` regions + instead of just `us-east`, `us-west` and `us-central` regions * `SX' & 'UM` country support added to NS1Provider, not yet in the North America list for backwards compatibility reasons. They will be added in the next releaser. @@ -278,7 +278,7 @@ * AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, & TransipPovider providers added -* Route53Provider seperator fix +* Route53Provider separator fix * YamlProvider export error around stringification * PyPi markdown rendering fix @@ -351,7 +351,7 @@ Using this version on existing records with `geo` will result in recreating all health checks. This process has been tested pretty thoroughly to -try and ensure a seemless upgrade without any traffic shifting around. It's +try and ensure a seamless upgrade without any traffic shifting around. It's probably best to take extra care when updating and to try and make sure that all health checks are passing before the first sync with `--doit`. See [#67](https://github.com/octodns/octodns/pull/67) for more information. diff --git a/README.md b/README.md index 93ff704..1c88df4 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ The architecture is pluggable and the tooling is flexible to make it applicable * [Updating to use extracted providers](#updating-to-use-extracted-providers) - [Sources](#sources) + [Notes](#notes) -- [Compatibilty and Compliance](#compatibilty-and-compliance) +- [Compatibility and Compliance](#compatibilty-and-compliance) * [`lenient`](#-lenient-) * [`strict_supports` (Work In Progress)](#-strict-supports---work-in-progress-) * [Configuring `strict_supports`](#configuring--strict-supports-) @@ -55,7 +55,7 @@ $ mkdir config #### Installing a specific commit SHA -If you'd like to install a version that has not yet been released in a repetable/safe manner you can do the following. In general octoDNS is fairly stable inbetween releases thanks to the plan and apply process, but care should be taken regardless. +If you'd like to install a version that has not yet been released in a repetable/safe manner you can do the following. In general octoDNS is fairly stable in between releases thanks to the plan and apply process, but care should be taken regardless. ```shell $ pip install -e git+https://git@github.com/octodns/octodns.git@#egg=octodns @@ -195,7 +195,7 @@ The above command pulled the existing data out of Route53 and placed the results ## Providers -The table below lists the providers octoDNS supports. They are maintained in their own repositories and released as independant modules. +The table below lists the providers octoDNS supports. They are maintained in their own repositories and released as independent modules. | Provider | Module | Notes | |--|--|--| @@ -250,7 +250,7 @@ Similar to providers, but can only serve to populate records into a zone, cannot * Dnsimple's uses the configured TTL when serving things through the ALIAS, there's also a secondary TXT record created alongside the ALIAS that octoDNS ignores * octoDNS itself supports non-ASCII character sets, but in testing Cloudflare is the only provider where that is currently functional end-to-end. Others have failures either in the client libraries or API calls -## Compatibilty and Compliance +## Compatibility and Compliance ### `lenient` diff --git a/octodns/processor/awsacm.py b/octodns/processor/awsacm.py index 63376fd..f147c00 100644 --- a/octodns/processor/awsacm.py +++ b/octodns/processor/awsacm.py @@ -15,7 +15,7 @@ try: from octodns_route53.processor import AwsAcmMangingProcessor AwsAcmMangingProcessor # pragma: no cover except ModuleNotFoundError: - logger.exception('AwsAcmMangingProcessor has been moved into a seperate ' + logger.exception('AwsAcmMangingProcessor has been moved into a separate ' 'module, octodns_route53 is now required. Processor ' 'class should be updated to ' 'octodns_route53.processor.AwsAcmMangingProcessor') diff --git a/octodns/processor/base.py b/octodns/processor/base.py index 98f2baa..82ee66a 100644 --- a/octodns/processor/base.py +++ b/octodns/processor/base.py @@ -15,7 +15,7 @@ class BaseProcessor(object): ''' Called after all sources have completed populate. Provides an opportunity for the processor to modify the desired `Zone` that targets - will recieve. + will receive. - Will see `desired` after any modifications done by `Provider._process_desired_zone` and processors configured to run diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4dbf213..3210685 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -15,7 +15,7 @@ try: from octodns_azure import AzureProvider AzureProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('AzureProvider has been moved into a seperate module, ' + logger.exception('AzureProvider has been moved into a separate module, ' 'octodns_azure is now required. Provider class should ' 'be updated to octodns_azure.AzureProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 7e0986f..c1a8f56 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -15,7 +15,7 @@ try: from octodns_cloudflare import CloudflareProvider CloudflareProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('CloudflareProvider has been moved into a seperate ' + logger.exception('CloudflareProvider has been moved into a separate ' 'module, octodns_cloudflare is now required. Provider ' 'class should be updated to ' 'octodns_cloudflare.CloudflareProvider. See ' diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 76e01ea..5159951 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -15,7 +15,7 @@ try: from octodns_constellix import ConstellixProvider ConstellixProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('ConstellixProvider has been moved into a seperate ' + logger.exception('ConstellixProvider has been moved into a separate ' 'module, octodns_constellix is now required. Provider ' 'class should be updated to ' 'octodns_constellix.ConstellixProvider. See ' diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index d7152ff..9ea59b1 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -15,7 +15,7 @@ try: from octodns_digitalocean import DigitalOceanProvider DigitalOceanProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('DigitalOceanProvider has been moved into a seperate ' + logger.exception('DigitalOceanProvider has been moved into a separate ' 'module, octodns_digitalocean is now required. Provider ' 'class should be updated to ' 'octodns_digitalocean.DigitalOceanProvider. See ' diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 7a2a97c..cd379a1 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -15,7 +15,7 @@ try: from octodns_dnsimple import DnsimpleProvider DnsimpleProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('DnsimpleProvider has been moved into a seperate module, ' + logger.exception('DnsimpleProvider has been moved into a separate module, ' 'octodns_dnsimple is now required. Provider class should ' 'be updated to octodns_dnsimple.DnsimpleProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 459767a..fc54c38 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -15,7 +15,7 @@ try: from octodns_dnsmadeeasy import DnsMadeEasyProvider DnsMadeEasyProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('DnsMadeEasyProvider has been moved into a seperate ' + logger.exception('DnsMadeEasyProvider has been moved into a separate ' 'module, octodns_dnsmadeeasy is now required. Provider ' 'class should be updated to ' 'octodns_dnsmadeeasy.DnsMadeEasyProvider. See ' diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index befb1e3..eef3a7a 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -15,7 +15,7 @@ try: from octodns_dyn import DynProvider DynProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('DynProvider has been moved into a seperate module, ' + logger.exception('DynProvider has been moved into a separate module, ' 'octodns_dyn is now required. Provider class should ' 'be updated to octodns_dyn.DynProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py index 5f78570..7be551b 100644 --- a/octodns/provider/easydns.py +++ b/octodns/provider/easydns.py @@ -16,7 +16,7 @@ try: EasyDnsProvider # pragma: no cover EasyDNSProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('EasyDNSProvider has been moved into a seperate module, ' + logger.exception('EasyDNSProvider has been moved into a separate module, ' 'octodns_easydns is now required. Provider class should ' 'be updated to octodns_easydns.EasyDnsProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/edgedns.py b/octodns/provider/edgedns.py index 020f7c3..8e1e58c 100644 --- a/octodns/provider/edgedns.py +++ b/octodns/provider/edgedns.py @@ -15,7 +15,7 @@ try: from octodns_edgedns import AkamaiProvider AkamaiProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('AkamaiProvider has been moved into a seperate module, ' + logger.exception('AkamaiProvider has been moved into a separate module, ' 'octodns_edgedns is now required. Provider class should ' 'be updated to octodns_edgedns.AkamaiProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/etc_hosts.py b/octodns/provider/etc_hosts.py index 9f02988..4ae9ee9 100644 --- a/octodns/provider/etc_hosts.py +++ b/octodns/provider/etc_hosts.py @@ -15,7 +15,7 @@ try: from octodns_etchosts import EtcHostsProvider EtcHostsProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('EtcHostsProvider has been moved into a seperate module, ' + logger.exception('EtcHostsProvider has been moved into a separate module, ' 'octodns_etchosts is now required. Provider class should ' 'be updated to octodns_etchosts.EtcHostsProvider. See ' 'See https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index f795d40..9dd1490 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -8,7 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger logger = getLogger('Akamai') -logger.warning('AkamaiProvider has been moved into a seperate module, ' +logger.warning('AkamaiProvider has been moved into a separate module, ' 'octodns_edgedns is now required. Provider class should ' 'be updated to octodns_edgedns.AkamaiProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py index 121bdd4..2429bfa 100644 --- a/octodns/provider/gandi.py +++ b/octodns/provider/gandi.py @@ -15,7 +15,7 @@ try: from octodns_gandi import GandiProvider GandiProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('GandiProvider has been moved into a seperate module, ' + logger.exception('GandiProvider has been moved into a separate module, ' 'octodns_gandi is now required. Provider class should ' 'be updated to octodns_gandi.GandiProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 83b3dd3..141ed10 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -15,7 +15,7 @@ try: from octodns_gcore import GCoreProvider GCoreProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('GCoreProvider has been moved into a seperate module, ' + logger.exception('GCoreProvider has been moved into a separate module, ' 'octodns_gcore is now required. Provider class should ' 'be updated to octodns_gcore.GCoreProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index e997533..22cb07a 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -15,7 +15,7 @@ try: from octodns_googlecloud import GoogleCloudProvider GoogleCloudProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('GoogleCloudProvider has been moved into a seperate ' + logger.exception('GoogleCloudProvider has been moved into a separate ' 'module, octodns_googlecloud is now required. Provider ' 'class should be updated to ' 'octodns_googlecloud.GoogleCloudProvider. See ' diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 78603c7..38577e4 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -15,7 +15,7 @@ try: from octodns_hetzner import HetznerProvider HetznerProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('HetznerProvider has been moved into a seperate module, ' + logger.exception('HetznerProvider has been moved into a separate module, ' 'octodns_hetzner is now required. Provider class should ' 'be updated to octodns_hetzner.HetznerProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index d9eed8a..0d1378e 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -15,7 +15,7 @@ try: from octodns_mythicbeasts import MythicBeastsProvider MythicBeastsProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('MythicBeastsProvider has been moved into a seperate ' + logger.exception('MythicBeastsProvider has been moved into a separate ' 'module, octodns_mythicbeasts is now required. Provider ' 'class should be updated to ' 'octodns_mythicbeasts.MythicBeastsProvider. See ' diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 79f9252..76398a8 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -15,7 +15,7 @@ try: from octodns_ns1 import Ns1Provider Ns1Provider # pragma: no cover except ModuleNotFoundError: - logger.exception('Ns1Provider has been moved into a seperate module, ' + logger.exception('Ns1Provider has been moved into a separate module, ' 'octodns_ns1 is now required. Provider class should ' 'be updated to octodns_ns1.Ns1Provider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 6147c57..5f83560 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -15,7 +15,7 @@ try: from octodns_ovh import OvhProvider OvhProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('OvhProvider has been moved into a seperate module, ' + logger.exception('OvhProvider has been moved into a separate module, ' 'octodns_ovh is now required. Provider class should ' 'be updated to octodns_ovh.OvhProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index cb52c6d..7a28f46 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -16,7 +16,7 @@ try: PowerDnsProvider # pragma: no cover PowerDnsBaseProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('PowerDnsProvider has been moved into a seperate module, ' + logger.exception('PowerDnsProvider has been moved into a separate module, ' 'octodns_powerdns is now required. Provider class should ' 'be updated to octodns_powerdns.PowerDnsProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 586fd5b..77436be 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -15,7 +15,7 @@ try: from octodns_rackspace import RackspaceProvider RackspaceProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('RackspaceProvider has been moved into a seperate ' + logger.exception('RackspaceProvider has been moved into a separate ' 'module, octodns_rackspace is now required. Provider ' 'class should be updated to ' 'octodns_rackspace.RackspaceProvider. See ' diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 9d5f42a..744281e 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -15,7 +15,7 @@ try: from octodns_route53 import Route53Provider Route53Provider # pragma: no cover except ModuleNotFoundError: - logger.exception('Route53Provider has been moved into a seperate module, ' + logger.exception('Route53Provider has been moved into a separate module, ' 'octodns_route53 is now required. Provider class should ' 'be updated to octodns_route53.Route53Provider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py index 4728a75..44325db 100644 --- a/octodns/provider/selectel.py +++ b/octodns/provider/selectel.py @@ -15,7 +15,7 @@ try: from octodns_selectel import SelectelProvider SelectelProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('SelectelProvider has been moved into a seperate module, ' + logger.exception('SelectelProvider has been moved into a separate module, ' 'octodns_selectel is now required. Provider class should ' 'be updated to octodns_selectel.SelectelProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 278aeff..5282b42 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -15,7 +15,7 @@ try: from octodns_transip import TransipProvider TransipProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('TransipProvider has been moved into a seperate module, ' + logger.exception('TransipProvider has been moved into a separate module, ' 'octodns_transip is now required. Provider class should ' 'be updated to octodns_transip.TransipProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 51c388b..6ccaf7a 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -15,7 +15,7 @@ try: from octodns_ultra import UltraProvider UltraProvider # pragma: no cover except ModuleNotFoundError: - logger.exception('UltraProvider has been moved into a seperate module, ' + logger.exception('UltraProvider has been moved into a separate module, ' 'octodns_ultra is now required. Provider class should ' 'be updated to octodns_ultra.UltraProvider. See ' 'https://github.com/octodns/octodns#updating-' diff --git a/octodns/source/base.py b/octodns/source/base.py index dfc1613..e328b04 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -38,7 +38,7 @@ class BaseSource(object): When `lenient` is True the populate call may skip record validation and do a "best effort" load of data. That will allow through some common, but not best practices stuff that we otherwise would reject. E.g. no - trailing . or mising escapes for ;. + trailing . or missing escapes for ;. When target is True (loading current state) this method should return True if the zone exists or False if it does not. diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 6d85924..1c01af9 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -290,7 +290,7 @@ class TestManager(TestCase): manager.dump('unit.tests.', tmpdir.dirname, False, False, 'in') # make sure this fails with an IOError and not a KeyError when - # tyring to find sub zones + # trying to find sub zones with self.assertRaises(IOError): manager.dump('unknown.zone.', tmpdir.dirname, False, False, 'in') @@ -319,7 +319,7 @@ class TestManager(TestCase): manager.dump('unit.tests.', tmpdir.dirname, False, True, 'in') # make sure this fails with an OSError and not a KeyError when - # tyring to find sub zones + # trying to find sub zones with self.assertRaises(OSError): manager.dump('unknown.zone.', tmpdir.dirname, False, True, 'in') @@ -447,7 +447,7 @@ class TestManager(TestCase): manager.sync(['unit.tests.']) with self.assertRaises(ManagerException) as ctx: - # This zone specifies a non-existant processor + # This zone specifies a non-existent processor manager.sync(['bad.unit.tests.']) self.assertTrue('Zone bad.unit.tests., unknown processor: ' 'doesnt-exist' in str(ctx.exception)) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 398e300..e1140c5 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -744,13 +744,13 @@ class TestBaseProviderSupportsRootNs(TestCase): # different root is in the desired plan = provider.plan(self.has_root) - # the mis-match doesn't matter since we can't manage the records + # the mismatch doesn't matter since we can't manage the records # anyway, they will have been removed from the desired and existing. self.assertFalse(plan) # plan again with strict_supports enabled, we should get an exception # b/c we have something configured that can't be managed (doesn't - # matter that it's a mis-match) + # matter that it's a mismatch) provider.strict_supports = True with self.assertRaises(SupportsException) as ctx: provider.plan(self.has_root) @@ -765,7 +765,7 @@ class TestBaseProviderSupportsRootNs(TestCase): # desired doesn't have a root plan = provider.plan(self.no_root) - # the mis-match doesn't matter since we can't manage the records + # the mismatch doesn't matter since we can't manage the records # anyway, they will have been removed from the desired and existing. self.assertFalse(plan) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index a406d96..4649745 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1759,7 +1759,7 @@ class TestRecordValidation(TestCase): reason = ctx.exception.reasons[0] self.assertTrue(reason.startswith('invalid name "@", use "" instead')) - # fqdn length, DNS defins max as 253 + # fqdn length, DNS defines max as 253 with self.assertRaises(ValidationError) as ctx: # The . will put this over the edge name = 'x' * (253 - len(self.zone.name)) From a17c4f8e0c970a9ecb3201e313f7aefec738508f Mon Sep 17 00:00:00 2001 From: William Jackson <565174+williamjacksn@users.noreply.github.com> Date: Sun, 29 May 2022 16:22:52 -0500 Subject: [PATCH 08/16] Add documentation for `include_meta` config key Update README.md to include documentation for the `include_meta` key in the `manager` section of the config. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 1c88df4..666e4b9 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ We start by creating a config file to tell OctoDNS about our providers and the z ```yaml --- manager: + include_meta: False max_workers: 2 providers: @@ -102,6 +103,8 @@ zones: Further information can be found in the `docstring` of each source and provider class. +The `include_meta` key in the `manager` section of the config controls the creation of a TXT record at the root of a zone that is managed by OctoDNS. If set to `True`, OctoDNS will create a TXT record for the root of the zone with the value `provider=`. If not specified, the default value for `include_meta` is `False`. + The `max_workers` key in the `manager` section of the config enables threading to parallelize the planning portion of the sync. In this example, `example.net` is an alias of zone `example.com`, which means they share the same sources and targets. They will therefore have identical records. From e299ceced245af72b6195954556022be42813478 Mon Sep 17 00:00:00 2001 From: Sachi King Date: Thu, 23 Jun 2022 18:40:01 +1000 Subject: [PATCH 09/16] Prepare tests with failing "managed subzone" error The is not a zone between delegated.subzone.unit.tests. and unit.tests., however we get a delegated subzone error. This modifies the tests to succeed with the added record, however the tests fail as it incorrectly throws the managed subzone error. Change the name of delegated.subzone, and the tests will pass cleanly. --- tests/config/simple.yaml | 5 +++++ tests/config/sub.txt.unit.tests.yaml | 1 + tests/config/unit.tests.yaml | 5 +++++ tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_yaml.py | 9 +++++---- 5 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 tests/config/sub.txt.unit.tests.yaml diff --git a/tests/config/simple.yaml b/tests/config/simple.yaml index fc4ad9f..4cd19c8 100644 --- a/tests/config/simple.yaml +++ b/tests/config/simple.yaml @@ -33,6 +33,11 @@ zones: targets: - dump - dump2 + sub.txt.unit.tests.: + sources: + - in + targets: + - dump empty.: sources: - in diff --git a/tests/config/sub.txt.unit.tests.yaml b/tests/config/sub.txt.unit.tests.yaml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/tests/config/sub.txt.unit.tests.yaml @@ -0,0 +1 @@ +--- diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index aa28ee5..b88ff82 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -162,6 +162,11 @@ sub: values: - 6.2.3.4. - 7.2.3.4. +sub.txt: + type: 'NS' + values: + - ns1.test. + - ns2.test. txt: ttl: 600 type: TXT diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 1c01af9..64aab9a 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -121,12 +121,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEqual(26, tc) + self.assertEqual(27, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEqual(20, tc) + self.assertEqual(21, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -141,18 +141,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEqual(26, tc) + self.assertEqual(27, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEqual(26, tc) + self.assertEqual(27, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEqual(30, tc) + self.assertEqual(32, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -220,13 +220,13 @@ class TestManager(TestCase): # compare doesn't use _process_desired_zone and thus doesn't filter # out root NS records, that seems fine/desirable changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEqual(21, len(changes)) + self.assertEqual(22, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEqual(20, len(changes)) + self.assertEqual(21, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 51e55eb..05fe90b 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -34,7 +34,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEqual(23, len(zone.records)) + self.assertEqual(24, len(zone.records)) source.populate(dynamic_zone) self.assertEqual(6, len(dynamic_zone.records)) @@ -57,12 +57,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEqual(20, len([c for c in plan.changes + self.assertEqual(21, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEqual(20, target.apply(plan)) + self.assertEqual(21, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -90,7 +90,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEqual(20, len([c for c in plan.changes + self.assertEqual(21, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -111,6 +111,7 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('txt')) self.assertTrue('values' in data.pop('loc')) self.assertTrue('values' in data.pop('urlfwd')) + self.assertTrue('values' in data.pop('sub.txt')) # these are stored as singular 'value' self.assertTrue('value' in data.pop('_imap._tcp')) self.assertTrue('value' in data.pop('_pop3._tcp')) From 5592f5da96a297f6563c82ea35d74f790f6dedd3 Mon Sep 17 00:00:00 2001 From: Sachi King Date: Thu, 23 Jun 2022 20:31:49 +1000 Subject: [PATCH 10/16] Support dotted subdomains for subzones Currently if there are two zones configured; - example.com. - delegated.subdomain.example.com. When an NS record is created in example.com.yaml as such: delegated.subdomain: type: NS values: - ns1.example.org. The NS record for delegated.subdomain.example.com cannot be created as it throws an exception: octodns.zone.SubzoneRecordException: Record delegated.subdomain.example.com is under a managed subzone Additionally, all records other than NS are rejected for subdomain.example.com.. This is caused by zone_tree being the result of all zones split on '.' and being added to the tree, even if a zone does not exist at that point. To support records where a subzone is dotted, the the map is built such that each node represents the subdomain of the closest subzone. Before: {"com", {"example": {"subdomain": {"delegated": {}}}}} After: {"example.com": {"delegated.subdomain": {}}} Fixes: #378 --- octodns/manager.py | 50 ++++++++++++++++++---------------------------- octodns/zone.py | 6 +++--- 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index e640a8d..27b384d 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -162,23 +162,18 @@ class Manager(object): processor_name) zone_tree = {} - # sort by reversed strings so that parent zones always come first - for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): - # ignore trailing dots, and reverse - pieces = name[:-1].split('.')[::-1] - # where starts out at the top - where = zone_tree - # for all the pieces - for piece in pieces: - try: - where = where[piece] - # our current piece already exists, just point where at - # it's value - except KeyError: - # our current piece doesn't exist, create it - where[piece] = {} - # and then point where at it's newly created value - where = where[piece] + # Sort so we iterate on the deepest nodes first, ensuring if a parent + # zone exists it will be seen after the subzone, thus we can easily + # reparent children to their parent zone from the tree root. + for name in sorted(self.config['zones'].keys(), + key=lambda s: 0 - s.count('.')): + name = name[:-1] + this = {} + for sz in filter( + lambda k: k.endswith(name), set(zone_tree.keys()) + ): + this[sz[:-(len(name) + 1)]] = zone_tree.pop(sz) + zone_tree[name] = this self.zone_tree = zone_tree self.plan_outputs = {} @@ -274,21 +269,14 @@ class Manager(object): return kwargs def configured_sub_zones(self, zone_name): - # Reversed pieces of the zone name - pieces = zone_name[:-1].split('.')[::-1] - # Point where at the root of the tree + name = zone_name[:-1] where = self.zone_tree - # Until we've hit the bottom of this zone - try: - while pieces: - # Point where at the value of our current piece - where = where[pieces.pop(0)] - except KeyError: - self.log.debug('configured_sub_zones: unknown zone, %s, no subs', - zone_name) - return set() - # We're not pointed at the dict for our name, the keys of which will be - # any subzones + while True: + parent = next(filter(lambda k: name.endswith(k), where), None) + if not parent: + break + where = where[parent] + name = name[:-(len(parent) + 1)] sub_zone_names = where.keys() self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) diff --git a/octodns/zone.py b/octodns/zone.py index 4cd5e91..41cd6ec 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -68,10 +68,10 @@ class Zone(object): self.hydrate() name = record.name - last = name.split('.')[-1] - if not lenient and last in self.sub_zones: - if name != last: + if not lenient and any(map(lambda sz: name.endswith(sz), + self.sub_zones)): + if name not in self.sub_zones: # it's a record for something under a sub-zone raise SubzoneRecordException(f'Record {record.fqdn} is under ' 'a managed subzone') From d5363e8045f1f810d187e50df8ea6f274de1f249 Mon Sep 17 00:00:00 2001 From: Sachi King Date: Mon, 27 Jun 2022 10:50:05 +1000 Subject: [PATCH 11/16] Add comments and use list comprehensions Per PR review, use list comprehensions as they are prefered in py3 over use of filter. Add comments to describe the building of the zone tree. --- octodns/manager.py | 17 +++++++++++++---- octodns/zone.py | 3 +-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 27b384d..030298e 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -167,12 +167,15 @@ class Manager(object): # reparent children to their parent zone from the tree root. for name in sorted(self.config['zones'].keys(), key=lambda s: 0 - s.count('.')): + # Trim the trailing dot from FQDN name = name[:-1] this = {} - for sz in filter( - lambda k: k.endswith(name), set(zone_tree.keys()) - ): + for sz in [k for k in zone_tree.keys() if k.endswith(name)]: + # Found a zone in tree root that is our child, slice the + # name and move its tree under ours. this[sz[:-(len(name) + 1)]] = zone_tree.pop(sz) + # Add to tree root where it will be reparented as we iterate up + # the tree. zone_tree[name] = this self.zone_tree = zone_tree @@ -272,11 +275,17 @@ class Manager(object): name = zone_name[:-1] where = self.zone_tree while True: - parent = next(filter(lambda k: name.endswith(k), where), None) + # Find parent if it exists + parent = next((k for k in where if name.endswith(k)), None) if not parent: + # The zone_name in the tree has been reached, stop searching. break + # Move down the tree and slice name to get the remainder for the + # next round of the search. where = where[parent] name = name[:-(len(parent) + 1)] + # `where` is now pointing at the dictionary of children for zone_name + # in the tree sub_zone_names = where.keys() self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) diff --git a/octodns/zone.py b/octodns/zone.py index 41cd6ec..7e42fa3 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -69,8 +69,7 @@ class Zone(object): name = record.name - if not lenient and any(map(lambda sz: name.endswith(sz), - self.sub_zones)): + if not lenient and any((name.endswith(sz) for sz in self.sub_zones)): if name not in self.sub_zones: # it's a record for something under a sub-zone raise SubzoneRecordException(f'Record {record.fqdn} is under ' From 04be906c3cb53711b61b41521117477c4c17dfe9 Mon Sep 17 00:00:00 2001 From: Sachi King Date: Mon, 27 Jun 2022 10:56:20 +1000 Subject: [PATCH 12/16] Add test to validate non-dotted subdomain zones are vaild This confirms that in addition to the recently added support for dotted subdomains that subdomains that are not dotted are supported. From RFC1034 Section 3.5 this would be a that contains a single