mirror/github-octodns
1
0
Fork 0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Code Issues Releases Activity

POC of auto-arpa concept, think there's too many complications though

Browse Source
This commit is contained in:
Ross McFarland
2022-09-29 15:33:01 -07:00
parent 6a19dc554f
commit 3fe6025462
5 changed files with 298 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
octodns/auto_arpa.py
+51
View File
@@ -0,0 +1,51 @@
#
#
#
from collections import defaultdict
from ipaddress import ip_address
from logging import getLogger
from .processor.base import BaseProcessor
from .record import Record
from .source.base import BaseSource
class AutoArpa(BaseProcessor, BaseSource):
SUPPORTS = set(('PTR',))
SUPPORTS_GEO = False
log = getLogger('AutoArpa')
def __init__(self, ttl=3600):
super().__init__('auto-arpa')
self.ttl = ttl
self._addrs = defaultdict(list)
def process_source_zone(self, desired, sources):
for record in desired.records:
if record._type in ('A', 'AAAA'):
for value in record.values:
addr = ip_address(value)
self._addrs[f'{addr.reverse_pointer}.'].append(record.fqdn)
return desired
def populate(self, zone, target=False, lenient=False):
self.log.debug('populate: zone=%s', zone.name)
before = len(zone.records)
name = zone.name
for arpa, fqdns in self._addrs.items():
if arpa.endswith(name):
record = Record.new(
zone,
zone.hostname_from_fqdn(arpa),
{'ttl': self.ttl, 'type': 'PTR', 'values': fqdns},
)
zone.add_record(record)
self.log.info(
'populate: found %s records', len(zone.records) - before
)
octodns/manager.py
+62 -10
View File
@@ -10,6 +10,7 @@ from sys import stdout
import logging
from . import __VERSION__
from .auto_arpa import AutoArpa
from .idna import IdnaDict, idna_decode, idna_encode
from .provider.base import BaseProvider
from .provider.plan import Plan
@@ -99,7 +100,13 @@ class Manager(object):
# TODO: all of this should get broken up, mainly so that it's not so huge
# and each bit can be cleanly tested independently
def __init__(self, config_file, max_workers=None, include_meta=False):
def __init__(
self,
config_file,
max_workers=None,
include_meta=False,
enable_auto_arpa=False,
):
version = self._try_version('octodns', version=__VERSION__)
self.log.info(
'__init__: config_file=%s (octoDNS %s)', config_file, version
@@ -119,8 +126,15 @@ class Manager(object):
self.include_meta = self._config_include_meta(
manager_config, include_meta
)
self.auto_arpa = self._config_auto_arpa(
manager_config, enable_auto_arpa
)
self.global_processors = manager_config.get('processors', [])
if self.auto_arpa:
# if enabled this need to run last on every zone so that it can get
# the final picture of any A/AAAA records
self.global_processors.append('auto-arpa')
self.log.info('__init__: global_processors=%s', self.global_processors)
providers_config = self.config['providers']
@@ -174,6 +188,17 @@ class Manager(object):
self.log.info('_config_include_meta: include_meta=%s', include_meta)
return include_meta
def _config_auto_arpa(self, manager_config, enable_auto_arpa=False):
enable_auto_arpa = enable_auto_arpa or manager_config.get(
'enable_auto_arpa', False
)
self.log.info(
'_config_auto_arpa: enable_auto_arpa=%s', enable_auto_arpa
)
if enable_auto_arpa:
return AutoArpa()
return None
def _config_providers(self, providers_config):
self.log.debug('_config_providers: configuring providers')
providers = {}
@@ -202,6 +227,9 @@ class Manager(object):
'Incorrect provider config for ' + provider_name
)
if self.auto_arpa:
providers['auto-arpa'] = self.auto_arpa
return providers
def _config_processors(self, processors_config):
@@ -229,6 +257,10 @@ class Manager(object):
raise ManagerException(
'Incorrect processor config for ' + processor_name
)
if self.auto_arpa:
processors['auto-arpa'] = self.auto_arpa
return processors
def _config_plan_outputs(self, plan_outputs_config):
@@ -476,6 +508,7 @@ class Manager(object):
zones = IdnaDict({n: zones.get(n) for n in eligible_zones})
aliased_zones = {}
deferred_kwargs = []
futures = []
for zone_name, config in zones.items():
decoded_zone_name = idna_decode(zone_name)
@@ -506,6 +539,10 @@ class Manager(object):
f'Zone {decoded_zone_name} is missing sources'
)
# if auto-arpa is in the list (while it's still ids) this zone needs
# to be deferred until after everything else has run
deferred = 'auto-arpa' in sources
try:
targets = config['targets']
except KeyError:
@@ -572,16 +609,20 @@ class Manager(object):
f'Zone {decoded_zone_name}, unknown ' f'target: {target}'
)
futures.append(
self._executor.submit(
self._populate_and_plan,
zone_name,
processors,
sources,
targets,
lenient=lenient,
kwargs = {
'zone_name': zone_name,
'processors': processors,
'sources': sources,
'targets': targets,
'lenient': lenient,
}
if deferred:
self.log.debug('sync: deferring %s', zone_name)
deferred_kwargs.append(kwargs)
else:
futures.append(
self._executor.submit(self._populate_and_plan, **kwargs)
)
)
# Wait on all results and unpack/flatten the plans and store the
# desired states in case we need them below
@@ -621,6 +662,17 @@ class Manager(object):
# as these are aliased zones
plans += [p for f in futures for p in f.result()[0]]
if deferred_kwargs:
self.log.debug('sync: planning deferred zones')
# now we need to run any deferred planning
futures = []
for kwargs in deferred_kwargs:
futures.append(
self._executor.submit(self._populate_and_plan, **kwargs)
)
# wait for them to finish
plans += [p for f in futures for p in f.result()[0]]
# Best effort sort plans children first so that we create/update
# children zones before parents which should allow us to more safely
# extract things into sub-zones. Combining a child back into a parent
tests/config/auto-arpa.yaml
+26
View File
@@ -0,0 +1,26 @@
manager:
enable_auto_arpa: true
max_workers: 2
providers:
in:
class: octodns.provider.yaml.YamlProvider
directory: tests/config
supports_root_ns: False
dump:
class: octodns.provider.yaml.YamlProvider
directory: env/YAML_TMP_DIR
default_ttl: 999
supports_root_ns: False
zones:
1.in-addr.arpa.:
sources:
- auto-arpa
targets:
- dump
unit.tests.:
sources:
- in
targets:
- dump
tests/test_octodns_auto_arpa.py
+149
View File
@@ -0,0 +1,149 @@
#
#
#
from unittest import TestCase
from octodns.auto_arpa import AutoArpa
from octodns.record import Record
from octodns.zone import Zone
class TestAutoArpa(TestCase):
def test_v4(self):
aa = AutoArpa()
# a record it won't be interested in b/c of type
zone = Zone('unit.tests.', [])
record = Record.new(
zone, 'ns', {'type': 'NS', 'ttl': 1800, 'value': 'ns1.unit.tests.'}
)
zone.add_record(record)
aa.process_source_zone(zone, [])
# nothing recorded
self.assertFalse(aa._addrs)
# a record it will record
zone = Zone('unit.tests.', [])
record = Record.new(
zone, 'a', {'type': 'A', 'ttl': 1800, 'value': '10.0.0.1'}
)
zone.add_record(record)
aa.process_source_zone(zone, [])
self.assertEqual(
{'1.0.0.10.in-addr.arpa.': ['a.unit.tests.']}, dict(aa._addrs)
)
# another record it will record
zone = Zone('unit.tests.', [])
record = Record.new(
zone, 'b', {'type': 'A', 'ttl': 1800, 'value': '10.0.42.1'}
)
zone.add_record(record)
aa.process_source_zone(zone, [])
self.assertEqual(
{
'1.0.0.10.in-addr.arpa.': ['a.unit.tests.'],
'1.42.0.10.in-addr.arpa.': ['b.unit.tests.'],
},
dict(aa._addrs),
)
# a second record pointed to the same IP
zone = Zone('unit.tests.', [])
record = Record.new(
zone, 'c', {'type': 'A', 'ttl': 1800, 'value': '10.0.42.1'}
)
zone.add_record(record)
aa.process_source_zone(zone, [])
self.assertEqual(
{
'1.0.0.10.in-addr.arpa.': ['a.unit.tests.'],
'1.42.0.10.in-addr.arpa.': ['b.unit.tests.', 'c.unit.tests.'],
},
dict(aa._addrs),
)
# subnet with just 1 record
zone = Zone('0.0.10.in-addr.arpa.', [])
aa.populate(zone)
self.assertEqual(
{'1.0.0.10.in-addr.arpa.': ['a.unit.tests.']},
{r.fqdn: r.values for r in zone.records},
)
# subnet with 2 records
zone = Zone('0.10.in-addr.arpa.', [])
aa.populate(zone)
self.assertEqual(
{
'1.0.0.10.in-addr.arpa.': ['a.unit.tests.'],
'1.42.0.10.in-addr.arpa.': ['b.unit.tests.', 'c.unit.tests.'],
},
{r.fqdn: r.values for r in zone.records},
)
def test_v6(self):
aa = AutoArpa()
# a v6 record it will record
zone = Zone('unit.tests.', [])
record = Record.new(
zone, 'aaaa', {'type': 'AAAA', 'ttl': 1800, 'value': 'fc00::1'}
)
zone.add_record(record)
aa.process_source_zone(zone, [])
self.assertEqual(
{
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
'aaaa.unit.tests.'
]
},
dict(aa._addrs),
)
# another v6 record it will record
zone = Zone('unit.tests.', [])
record = Record.new(
zone, 'bbbb', {'type': 'AAAA', 'ttl': 1800, 'value': 'fc42::1'}
)
zone.add_record(record)
aa.process_source_zone(zone, [])
self.assertEqual(
{
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
'aaaa.unit.tests.'
],
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.4.c.f.ip6.arpa.': [
'bbbb.unit.tests.'
],
},
dict(aa._addrs),
)
# subnet with just 1 record
zone = Zone('0.0.c.f.ip6.arpa.', [])
aa.populate(zone)
self.assertEqual(
{
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
'aaaa.unit.tests.'
]
},
{r.fqdn: r.values for r in zone.records},
)
# subnet with 2 records
zone = Zone('c.f.ip6.arpa.', [])
aa.populate(zone)
self.assertEqual(
{
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
'aaaa.unit.tests.'
],
'1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.4.c.f.ip6.arpa.': [
'bbbb.unit.tests.'
],
},
{r.fqdn: r.values for r in zone.records},
)
tests/test_octodns_manager.py
+10
View File
@@ -176,6 +176,16 @@ class TestManager(TestCase):
).sync(dry_run=False, force=True)
self.assertEqual(33, tc)
def test_auto_arpa(self):
with TemporaryDirectory() as tmpdir:
environ['YAML_TMP_DIR'] = tmpdir.dirname
environ['YAML_TMP_DIR2'] = tmpdir.dirname
manager = Manager(get_config_filename('auto-arpa.yaml'))
tc = manager.sync(dry_run=False)
# we expect 22 records from unit.tests and 2 PTRs in the arpa zone
self.assertEqual(24, tc)
def test_idna_eligible_zones(self):
# loading w/simple, but we'll be blowing it away and doing some manual
# stuff