From a948b8233e64af3145120de255c3460e53543cd6 Mon Sep 17 00:00:00 2001 From: William Gauthier Date: Fri, 26 Apr 2024 17:07:58 +0200 Subject: [PATCH 1/2] AutoArpa: Add an optional inherit_ttl option --- docs/auto_arpa.md | 5 ++- octodns/processor/arpa.py | 17 +++++--- tests/test_octodns_processor_arpa.py | 60 +++++++++++++++++----------- 3 files changed, 52 insertions(+), 30 deletions(-) diff --git a/docs/auto_arpa.md b/docs/auto_arpa.md index a5411c2..a1f7053 100644 --- a/docs/auto_arpa.md +++ b/docs/auto_arpa.md @@ -15,12 +15,15 @@ Alternatively the value can be a dictionary with configuration options for the A manager: auto_arpa: # Whether duplicate records should replace rather than error - # (optiona, default False) + # (optional, default False) populate_should_replace: false # Explicitly set the TTL of auto-created records, default is 3600s, 1hr ttl: 1800 # Set how many PTR records will be created for the same IP, default: 999 max_auto_arpa: 1 + # Inherit the TTL value of the corresponding A/AAAA record (optional, + # default False) + inherit_ttl: True ``` Once enabled, a singleton `AutoArpa` instance, `auto-arpa`, will be added to the pool of providers and globally configured to run as the very last global processor so that it will see all records as they will be seen by targets. Further all zones ending with `arpa.` will be held back and processed after all other zones have been completed so that all `A` and `AAAA` records will have been seen prior to planning the `arpa.` zones. diff --git a/octodns/processor/arpa.py b/octodns/processor/arpa.py index bd56ec9..2675f0d 100644 --- a/octodns/processor/arpa.py +++ b/octodns/processor/arpa.py @@ -12,19 +12,21 @@ from .base import BaseProcessor class AutoArpa(BaseProcessor): def __init__( - self, name, ttl=3600, populate_should_replace=False, max_auto_arpa=999 + self, name, ttl=3600, populate_should_replace=False, max_auto_arpa=999, inherit_ttl=False ): super().__init__(name) self.log = getLogger(f'AutoArpa[{name}]') self.log.info( - '__init__: ttl=%d, populate_should_replace=%s, max_auto_arpa=%d', + '__init__: ttl=%d, populate_should_replace=%s, max_auto_arpa=%d, inherit_ttl=%s', ttl, populate_should_replace, max_auto_arpa, + inherit_ttl, ) self.ttl = ttl self.populate_should_replace = populate_should_replace self.max_auto_arpa = max_auto_arpa + self.inherit_ttl = inherit_ttl self._records = defaultdict(list) def process_source_zone(self, desired, sources): @@ -44,8 +46,12 @@ class AutoArpa(BaseProcessor): auto_arpa_priority = record.octodns.get( 'auto_arpa_priority', 999 ) + if self.inherit_ttl: + record_ttl = record.ttl + else: + record_ttl = self.ttl self._records[f'{ptr}.'].append( - (auto_arpa_priority, record.fqdn) + (auto_arpa_priority, record_ttl, record.fqdn) ) return desired @@ -55,7 +61,7 @@ class AutoArpa(BaseProcessor): # order the fqdns making a copy so we can reset the list below ordered = sorted(fqdns) fqdns = [] - for _, fqdn in ordered: + for _, _, fqdn in ordered: if fqdn in seen: continue fqdns.append(fqdn) @@ -79,12 +85,13 @@ class AutoArpa(BaseProcessor): for arpa, fqdns in self._records.items(): if arpa.endswith(f'.{zone_name}'): name = arpa[:-n] + _, record_ttl, _ = fqdns[0] # Note: this takes a list of (priority, fqdn) tuples and returns the ordered and uniqified list of fqdns. fqdns = self._order_and_unique_fqdns(fqdns, self.max_auto_arpa) record = Record.new( zone, name, - {'ttl': self.ttl, 'type': 'PTR', 'values': fqdns}, + {'ttl': record_ttl, 'type': 'PTR', 'values': fqdns}, lenient=lenient, ) zone.add_record( diff --git a/tests/test_octodns_processor_arpa.py b/tests/test_octodns_processor_arpa.py index 328d05f..463ecc6 100644 --- a/tests/test_octodns_processor_arpa.py +++ b/tests/test_octodns_processor_arpa.py @@ -27,7 +27,7 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) self.assertEqual( - {'4.3.2.1.in-addr.arpa.': [(999, 'a.unit.tests.')]}, aa._records + {'4.3.2.1.in-addr.arpa.': [(999, 3600, 'a.unit.tests.')]}, aa._records ) # matching zone @@ -56,8 +56,8 @@ class TestAutoArpa(TestCase): aa.process_source_zone(zone, []) self.assertEqual( { - '4.3.2.1.in-addr.arpa.': [(999, 'a.unit.tests.')], - '5.3.2.1.in-addr.arpa.': [(999, 'a.unit.tests.')], + '4.3.2.1.in-addr.arpa.': [(999, 1600, 'a.unit.tests.')], + '5.3.2.1.in-addr.arpa.': [(999, 1600, 'a.unit.tests.')], }, aa._records, ) @@ -81,7 +81,7 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) ip6_arpa = '2.0.0.0.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.0.0.0.f.f.0.0.ip6.arpa.' - self.assertEqual({ip6_arpa: [(999, 'aaaa.unit.tests.')]}, aa._records) + self.assertEqual({ip6_arpa: [(999, 3600, 'aaaa.unit.tests.')]}, aa._records) # matching zone arpa = Zone('c.0.0.0.f.f.0.0.ip6.arpa.', []) @@ -117,13 +117,13 @@ class TestAutoArpa(TestCase): aa.process_source_zone(zone, []) self.assertEqual( { - '4.3.2.1.in-addr.arpa.': [(999, 'geo.unit.tests.')], - '5.3.2.1.in-addr.arpa.': [(999, 'geo.unit.tests.')], - '1.1.1.1.in-addr.arpa.': [(999, 'geo.unit.tests.')], - '2.2.2.2.in-addr.arpa.': [(999, 'geo.unit.tests.')], - '3.3.3.3.in-addr.arpa.': [(999, 'geo.unit.tests.')], - '4.4.4.4.in-addr.arpa.': [(999, 'geo.unit.tests.')], - '5.5.5.5.in-addr.arpa.': [(999, 'geo.unit.tests.')], + '4.3.2.1.in-addr.arpa.': [(999, 3600, 'geo.unit.tests.')], + '5.3.2.1.in-addr.arpa.': [(999, 3600, 'geo.unit.tests.')], + '1.1.1.1.in-addr.arpa.': [(999, 3600, 'geo.unit.tests.')], + '2.2.2.2.in-addr.arpa.': [(999, 3600, 'geo.unit.tests.')], + '3.3.3.3.in-addr.arpa.': [(999, 3600, 'geo.unit.tests.')], + '4.4.4.4.in-addr.arpa.': [(999, 3600, 'geo.unit.tests.')], + '5.5.5.5.in-addr.arpa.': [(999, 3600, 'geo.unit.tests.')], }, aa._records, ) @@ -194,8 +194,8 @@ class TestAutoArpa(TestCase): self.assertEqual( { '4.3.2.1.in-addr.arpa.': [ - (999, 'a1.unit.tests.'), - (999, 'a2.unit.tests.'), + (999, 3600, 'a1.unit.tests.'), + (999, 3600, 'a2.unit.tests.'), ] }, {'4.3.2.1.in-addr.arpa.': sorted_records}, @@ -219,7 +219,7 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) self.assertEqual( - {'4.3.20.10.in-addr.arpa.': [(999, 'a.unit.tests.')]}, aa._records + {'4.3.20.10.in-addr.arpa.': [(999, 3600, 'a.unit.tests.')]}, aa._records ) # matching zone @@ -259,7 +259,7 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) self.assertEqual( - {'4.3.2.1.in-addr.arpa.': [(999, 'a with spaces.unit.tests.')]}, + {'4.3.2.1.in-addr.arpa.': [(999, 3600, 'a with spaces.unit.tests.')]}, aa._records, ) @@ -275,18 +275,18 @@ class TestAutoArpa(TestCase): def test_arpa_priority(self): aa = AutoArpa('auto-arpa') - duplicate_values = [(999, 'a.unit.tests.'), (1, 'a.unit.tests.')] + duplicate_values = [(999, 3600, 'a.unit.tests.'), (1, 3600, 'a.unit.tests.')] self.assertEqual( ['a.unit.tests.'], aa._order_and_unique_fqdns(duplicate_values, max_auto_arpa=999), ) duplicate_values = [ - (50, 'd.unit.tests.'), - (999, 'dup.unit.tests.'), - (3, 'a.unit.tests.'), - (1, 'dup.unit.tests.'), - (2, 'c.unit.tests.'), + (50, 3600, 'd.unit.tests.'), + (999, 3600, 'dup.unit.tests.'), + (3, 3600, 'a.unit.tests.'), + (1, 3600, 'dup.unit.tests.'), + (2, 3600, 'c.unit.tests.'), ] self.assertEqual( [ @@ -298,20 +298,32 @@ class TestAutoArpa(TestCase): aa._order_and_unique_fqdns(duplicate_values, max_auto_arpa=999), ) - duplicate_values_2 = [(999, 'a.unit.tests.'), (999, 'a.unit.tests.')] + duplicate_values_2 = [(999, 3600, 'a.unit.tests.'), (999, 3600, 'a.unit.tests.')] self.assertEqual( ['a.unit.tests.'], aa._order_and_unique_fqdns(duplicate_values_2, max_auto_arpa=999), ) - ordered_values = [(999, 'a.unit.tests.'), (1, 'b.unit.tests.')] + ordered_values = [(999, 3600, 'a.unit.tests.'), (1, 3600, 'b.unit.tests.')] self.assertEqual( ['b.unit.tests.', 'a.unit.tests.'], aa._order_and_unique_fqdns(ordered_values, max_auto_arpa=999), ) - max_one_value = [(999, 'a.unit.tests.'), (1, 'b.unit.tests.')] + max_one_value = [(999, 3600, 'a.unit.tests.'), (1, 3600, 'b.unit.tests.')] self.assertEqual( ['b.unit.tests.'], aa._order_and_unique_fqdns(max_one_value, max_auto_arpa=1), ) + + def test_arpa_inherit_ttl(self): + zone = Zone('unit.tests.', []) + record = Record.new( + zone, 'a1', {'ttl': 32, 'type': 'A', 'value': '1.2.3.4'} + ) + zone.add_record(record) + + aa = AutoArpa('auto-arpa', 3600, 999, True) + aa.populate(zone) + (ptr,) = zone.records + self.assertEqual(32, ptr.ttl) From b15e04380d87f04b042a46973bc4dd883d30e8aa Mon Sep 17 00:00:00 2001 From: William Gauthier Date: Sun, 28 Apr 2024 21:33:42 +0200 Subject: [PATCH 2/2] Use the TTL value of the first ordered FQDN --- octodns/processor/arpa.py | 21 ++++--- tests/test_octodns_processor_arpa.py | 82 +++++++++++++++++++++------- 2 files changed, 75 insertions(+), 28 deletions(-) diff --git a/octodns/processor/arpa.py b/octodns/processor/arpa.py index 2675f0d..d360b19 100644 --- a/octodns/processor/arpa.py +++ b/octodns/processor/arpa.py @@ -12,7 +12,12 @@ from .base import BaseProcessor class AutoArpa(BaseProcessor): def __init__( - self, name, ttl=3600, populate_should_replace=False, max_auto_arpa=999, inherit_ttl=False + self, + name, + ttl=3600, + populate_should_replace=False, + max_auto_arpa=999, + inherit_ttl=False, ): super().__init__(name) self.log = getLogger(f'AutoArpa[{name}]') @@ -61,10 +66,10 @@ class AutoArpa(BaseProcessor): # order the fqdns making a copy so we can reset the list below ordered = sorted(fqdns) fqdns = [] - for _, _, fqdn in ordered: + for _, record_ttl, fqdn in ordered: if fqdn in seen: continue - fqdns.append(fqdn) + fqdns.append((record_ttl, fqdn)) seen.add(fqdn) if len(seen) >= max_auto_arpa: break @@ -85,13 +90,16 @@ class AutoArpa(BaseProcessor): for arpa, fqdns in self._records.items(): if arpa.endswith(f'.{zone_name}'): name = arpa[:-n] - _, record_ttl, _ = fqdns[0] - # Note: this takes a list of (priority, fqdn) tuples and returns the ordered and uniqified list of fqdns. + # Note: this takes a list of (priority, ttl, fqdn) tuples and returns the ordered and uniqified list of fqdns. fqdns = self._order_and_unique_fqdns(fqdns, self.max_auto_arpa) record = Record.new( zone, name, - {'ttl': record_ttl, 'type': 'PTR', 'values': fqdns}, + { + 'ttl': fqdns[0][0], + 'type': 'PTR', + 'values': [fqdn[1] for fqdn in fqdns], + }, lenient=lenient, ) zone.add_record( @@ -99,7 +107,6 @@ class AutoArpa(BaseProcessor): replace=self.populate_should_replace, lenient=lenient, ) - self.log.info( 'populate: found %s records', len(zone.records) - before ) diff --git a/tests/test_octodns_processor_arpa.py b/tests/test_octodns_processor_arpa.py index 463ecc6..be4c2fd 100644 --- a/tests/test_octodns_processor_arpa.py +++ b/tests/test_octodns_processor_arpa.py @@ -27,7 +27,8 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) self.assertEqual( - {'4.3.2.1.in-addr.arpa.': [(999, 3600, 'a.unit.tests.')]}, aa._records + {'4.3.2.1.in-addr.arpa.': [(999, 3600, 'a.unit.tests.')]}, + aa._records, ) # matching zone @@ -81,7 +82,9 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) ip6_arpa = '2.0.0.0.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.0.0.0.f.f.0.0.ip6.arpa.' - self.assertEqual({ip6_arpa: [(999, 3600, 'aaaa.unit.tests.')]}, aa._records) + self.assertEqual( + {ip6_arpa: [(999, 3600, 'aaaa.unit.tests.')]}, aa._records + ) # matching zone arpa = Zone('c.0.0.0.f.f.0.0.ip6.arpa.', []) @@ -176,7 +179,7 @@ class TestAutoArpa(TestCase): unique_values = aa._order_and_unique_fqdns( aa._records[f'{zone}'], 999 ) - self.assertEqual([('dynamic.unit.tests.')], unique_values) + self.assertEqual([(3600, 'dynamic.unit.tests.')], unique_values) def test_multiple_names(self): zone = Zone('unit.tests.', []) @@ -219,7 +222,8 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) self.assertEqual( - {'4.3.20.10.in-addr.arpa.': [(999, 3600, 'a.unit.tests.')]}, aa._records + {'4.3.20.10.in-addr.arpa.': [(999, 3600, 'a.unit.tests.')]}, + aa._records, ) # matching zone @@ -259,7 +263,11 @@ class TestAutoArpa(TestCase): aa = AutoArpa('auto-arpa') aa.process_source_zone(zone, []) self.assertEqual( - {'4.3.2.1.in-addr.arpa.': [(999, 3600, 'a with spaces.unit.tests.')]}, + { + '4.3.2.1.in-addr.arpa.': [ + (999, 3600, 'a with spaces.unit.tests.') + ] + }, aa._records, ) @@ -275,9 +283,12 @@ class TestAutoArpa(TestCase): def test_arpa_priority(self): aa = AutoArpa('auto-arpa') - duplicate_values = [(999, 3600, 'a.unit.tests.'), (1, 3600, 'a.unit.tests.')] + duplicate_values = [ + (999, 3600, 'a.unit.tests.'), + (1, 3600, 'a.unit.tests.'), + ] self.assertEqual( - ['a.unit.tests.'], + [(3600, 'a.unit.tests.')], aa._order_and_unique_fqdns(duplicate_values, max_auto_arpa=999), ) @@ -290,29 +301,38 @@ class TestAutoArpa(TestCase): ] self.assertEqual( [ - 'dup.unit.tests.', - 'c.unit.tests.', - 'a.unit.tests.', - 'd.unit.tests.', + (3600, 'dup.unit.tests.'), + (3600, 'c.unit.tests.'), + (3600, 'a.unit.tests.'), + (3600, 'd.unit.tests.'), ], aa._order_and_unique_fqdns(duplicate_values, max_auto_arpa=999), ) - duplicate_values_2 = [(999, 3600, 'a.unit.tests.'), (999, 3600, 'a.unit.tests.')] + duplicate_values_2 = [ + (999, 3600, 'a.unit.tests.'), + (999, 3600, 'a.unit.tests.'), + ] self.assertEqual( - ['a.unit.tests.'], + [(3600, 'a.unit.tests.')], aa._order_and_unique_fqdns(duplicate_values_2, max_auto_arpa=999), ) - ordered_values = [(999, 3600, 'a.unit.tests.'), (1, 3600, 'b.unit.tests.')] + ordered_values = [ + (999, 3600, 'a.unit.tests.'), + (1, 3600, 'b.unit.tests.'), + ] self.assertEqual( - ['b.unit.tests.', 'a.unit.tests.'], + [(3600, 'b.unit.tests.'), (3600, 'a.unit.tests.')], aa._order_and_unique_fqdns(ordered_values, max_auto_arpa=999), ) - max_one_value = [(999, 3600, 'a.unit.tests.'), (1, 3600, 'b.unit.tests.')] + max_one_value = [ + (999, 3600, 'a.unit.tests.'), + (1, 3600, 'b.unit.tests.'), + ] self.assertEqual( - ['b.unit.tests.'], + [(3600, 'b.unit.tests.')], aa._order_and_unique_fqdns(max_one_value, max_auto_arpa=1), ) @@ -323,7 +343,27 @@ class TestAutoArpa(TestCase): ) zone.add_record(record) - aa = AutoArpa('auto-arpa', 3600, 999, True) - aa.populate(zone) - (ptr,) = zone.records - self.assertEqual(32, ptr.ttl) + record2 = Record.new( + zone, + 'a2', + { + 'ttl': 64, + 'type': 'A', + 'value': '1.2.3.4', + 'octodns': {'auto_arpa_priority': 1}, + }, + ) + zone.add_record(record2) + + record3 = Record.new( + zone, 'a3', {'ttl': 128, 'type': 'A', 'value': '1.2.3.4'} + ) + zone.add_record(record3) + + aa = AutoArpa('auto-arpa', 3600, False, 999, True) + aa.process_source_zone(zone, []) + + arpa = Zone('3.2.1.in-addr.arpa.', []) + aa.populate(arpa) + (ptr,) = arpa.records + self.assertEqual(64, ptr.ttl)