WIP/expiriments with auto-arpa

octodns/auto_arpa.py

@@ -0,0 +1,51 @@
+#
+#
+#
+
+from collections import defaultdict
+from ipaddress import ip_address
+from logging import getLogger
+
+from .processor.base import BaseProcessor
+from .record import Record
+from .source.base import BaseSource
+
+
+class AutoArpa(BaseProcessor, BaseSource):
+    SUPPORTS = set(('PTR',))
+    SUPPORTS_GEO = False
+
+    log = getLogger('AutoArpa')
+
+    def __init__(self, ttl=3600):
+        super().__init__('auto-arpa')
+        self.ttl = ttl
+
+        self._addrs = defaultdict(list)
+
+    def process_source_zone(self, desired, sources):
+        for record in desired.records:
+            if record._type in ('A', 'AAAA'):
+                for value in record.values:
+                    addr = ip_address(value)
+                    self._addrs[f'{addr.reverse_pointer}.'].append(record.fqdn)
+
+        return desired
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: zone=%s', zone.name)
+        before = len(zone.records)
+
+        name = zone.name
+        for arpa, fqdns in self._addrs.items():
+            if arpa.endswith(name):
+                record = Record.new(
+                    zone,
+                    zone.hostname_from_fqdn(arpa),
+                    {'ttl': self.ttl, 'type': 'PTR', 'values': fqdns},
+                )
+                zone.add_record(record)
+
+        self.log.info(
+            'populate:   found %s records', len(zone.records) - before
+        )

octodns/manager.py

@@ -10,6 +10,7 @@ from sys import stdout
 import logging
 
 from . import __VERSION__
+from .auto_arpa import AutoArpa
 from .idna import IdnaDict, idna_decode, idna_encode
 from .provider.base import BaseProvider
 from .provider.plan import Plan
@@ -62,7 +63,7 @@ class _AggregateTarget(object):
         raise AttributeError(f'{klass} object has no attribute {name}')
 
 
-class MakeThreadFuture(object):
+class FakeThreadFuture(object):
     def __init__(self, func, args, kwargs):
         self.func = func
         self.args = args
@@ -82,7 +83,7 @@ class MainThreadExecutor(object):
     '''
 
     def submit(self, func, *args, **kwargs):
-        return MakeThreadFuture(func, args, kwargs)
+        return FakeThreadFuture(func, args, kwargs)
 
 
 class ManagerException(Exception):
@@ -99,7 +100,9 @@ class Manager(object):
 
     # TODO: all of this should get broken up, mainly so that it's not so huge
     # and each bit can be cleanly tested independently
-    def __init__(self, config_file, max_workers=None, include_meta=False):
+    def __init__(
+        self, config_file, max_workers=None, include_meta=False, auto_arpa=False
+    ):
         version = self._try_version('octodns', version=__VERSION__)
         self.log.info(
             '__init__: config_file=%s (octoDNS %s)', config_file, version
@@ -119,6 +122,7 @@ class Manager(object):
         self.include_meta = self._config_include_meta(
             manager_config, include_meta
         )
+        self.auto_arpa = self._config_auto_arpa(manager_config, auto_arpa)
 
         self.global_processors = manager_config.get('processors', [])
         self.log.info('__init__: global_processors=%s', self.global_processors)
@@ -174,6 +178,15 @@ class Manager(object):
         self.log.info('_config_include_meta: include_meta=%s', include_meta)
         return include_meta
 
+    def _config_auto_arpa(self, manager_config, auto_arpa=False):
+        auto_arpa = auto_arpa or manager_config.get('auto-arpa', False)
+        self.log.info('_config_auto_arpa: auto_arpa=%s', auto_arpa)
+        if auto_arpa:
+            if not isinstance(auto_arpa, dict):
+                auto_arpa = {}
+            return AutoArpa(**auto_arpa)
+        return None
+
     def _config_providers(self, providers_config):
         self.log.debug('_config_providers: configuring providers')
         providers = {}
@@ -202,6 +215,9 @@ class Manager(object):
                     'Incorrect provider config for ' + provider_name
                 )
 
+        if self.auto_arpa:
+            providers['auto-arpa'] = self.auto_arpa
+
         return providers
 
     def _config_processors(self, processors_config):
@@ -229,6 +245,10 @@ class Manager(object):
                 raise ManagerException(
                     'Incorrect processor config for ' + processor_name
                 )
+
+        if self.auto_arpa:
+            processors['auto-arpa'] = self.auto_arpa
+
         return processors
 
     def _config_plan_outputs(self, plan_outputs_config):
@@ -477,6 +497,7 @@ class Manager(object):
 
         aliased_zones = {}
         futures = []
+        arpa_kwargs = []
         for zone_name, config in zones.items():
             decoded_zone_name = idna_decode(zone_name)
             self.log.info('sync:   zone=%s', decoded_zone_name)
@@ -537,6 +558,9 @@ class Manager(object):
                 collected = []
                 for processor in self.global_processors + processors:
                     collected.append(self.processors[processor])
+                # always goes last
+                if self.auto_arpa:
+                    collected.append(self.auto_arpa)
                 processors = collected
             except KeyError:
                 raise ManagerException(
@@ -572,16 +596,25 @@ class Manager(object):
                     f'Zone {decoded_zone_name}, unknown ' f'target: {target}'
                 )
 
-            futures.append(
+            kwargs = {
-                self._executor.submit(
+                'zone_name': zone_name,
-                    self._populate_and_plan,
+                'processors': processors,
-                    zone_name,
+                'sources': sources,
-                    processors,
+                'targets': targets,
-                    sources,
+                'lenient': lenient,
-                    targets,
+            }
-                    lenient=lenient,
+            if self.auto_arpa and (
+                zone_name.endswith('in-addr.arpa.')
+                or zone_name.endswith('ip6.arpa.')
+            ):
+                # auto arpa is enabled so we need to defer processing all arpa
+                # zones until after general ones have completed (async) so that
+                # they'll have access to all the recorded A/AAAA values
+                arpa_kwargs.append(kwargs)
+            else:
+                futures.append(
+                    self._executor.submit(self._populate_and_plan, **kwargs)
                 )
-            )
 
         # Wait on all results and unpack/flatten the plans and store the
         # desired states in case we need them below
@@ -621,6 +654,24 @@ class Manager(object):
         # as these are aliased zones
         plans += [p for f in futures for p in f.result()[0]]
 
+        if self.auto_arpa and arpa_kwargs:
+            self.log.info(
+                'sync: processing %d arpa reverse dns zones', len(arpa_kwargs)
+            )
+            # all the general zones are done and we've recorded the A/AAAA
+            # records with the AutoPtr. We can no process ptr zones
+            futures = []
+            for kwargs in arpa_kwargs:
+                futures.append(
+                    self._executor.submit(self._populate_and_plan, **kwargs)
+                )
+
+            for future in futures:
+                ps, d = future.result()
+                desired[d.name] = d
+                for plan in ps:
+                    plans.append(plan)
+
         # Best effort sort plans children first so that we create/update
         # children zones before parents which should allow us to more safely
         # extract things into sub-zones. Combining a child back into a parent

tests/config/simple.yaml

@@ -1,4 +1,5 @@
 manager:
+  auto-arpa: true
   max_workers: 2
 providers:
   in:
@@ -44,3 +45,8 @@ zones:
     - in
     targets:
     - dump
+  3.2.1.in-addr.arpa.:
+    sources:
+      - auto-arpa
+    targets:
+    - dump

tests/test_octodns_manager.py

@@ -136,7 +136,7 @@ class TestManager(TestCase):
             environ['YAML_TMP_DIR'] = tmpdir.dirname
             environ['YAML_TMP_DIR2'] = tmpdir.dirname
             tc = Manager(get_config_filename('simple.yaml')).sync(dry_run=False)
-            self.assertEqual(28, tc)
+            self.assertEqual(30, tc)
 
             # try with just one of the zones
             tc = Manager(get_config_filename('simple.yaml')).sync(
@@ -160,13 +160,13 @@ class TestManager(TestCase):
             tc = Manager(get_config_filename('simple.yaml')).sync(
                 dry_run=False, force=True
             )
-            self.assertEqual(28, tc)
+            self.assertEqual(30, tc)
 
             # Again with max_workers = 1
             tc = Manager(
                 get_config_filename('simple.yaml'), max_workers=1
             ).sync(dry_run=False, force=True)
-            self.assertEqual(28, tc)
+            self.assertEqual(30, tc)
 
             # Include meta
             tc = Manager(
@@ -174,7 +174,7 @@ class TestManager(TestCase):
                 max_workers=1,
                 include_meta=True,
             ).sync(dry_run=False, force=True)
-            self.assertEqual(33, tc)
+            self.assertEqual(36, tc)
 
     def test_idna_eligible_zones(self):
         # loading w/simple, but we'll be blowing it away and doing some manual
@@ -186,9 +186,6 @@ class TestManager(TestCase):
         manager.config['zones'] = manager._config_zones(
             {'déjà.vu.': {}, 'deja.vu.': {}, idna_encode('こんにちは.jp.'): {}}
         )
-        from pprint import pprint
-
-        pprint(manager.config['zones'])
 
         # refer to them with utf-8
         with self.assertRaises(ManagerException) as ctx: