WIP: full tinydns spec-compliant source implementation

CHANGELOG.md

@@ -6,6 +6,8 @@
 * octodns-report access --lenient flag to allow running reports with records
   sourced from providers with non-compliant record data.
 * Correctly handle FQDNs in TinyDNS config files that end with trailing .'s
+* Complete rewrite of TinyDnsBaseSource to fully implement the spec and the ipv6
+  extensions
 
 ## v1.0.0.rc0 - 2023-05-16 - First of the ones
 

octodns/source/tinydns.py

@@ -9,19 +9,26 @@ from collections import defaultdict
 from ipaddress import ip_address
 from os import listdir
 from os.path import join
+from pprint import pprint
 
 from ..record import Record
 from ..zone import DuplicateRecordException, SubzoneRecordException
 from .base import BaseSource
 
 
+def _decode_octal(s):
+    return re.sub(r'\\(\d\d\d)', lambda m: chr(int(m.group(1), 8)), s).replace(
+        ';', '\\;'
+    )
+
+
 class TinyDnsBaseSource(BaseSource):
+    # spec https://cr.yp.to/djbdns/tinydns-data.html
+    # ipv6 addon spec https://docs.bytemark.co.uk/article/tinydns-format/
     SUPPORTS_GEO = False
     SUPPORTS_DYNAMIC = False
     SUPPORTS = set(('A', 'CNAME', 'MX', 'NS', 'TXT', 'AAAA'))
 
-    split_re = re.compile(r':+')
-
     def __init__(self, id, default_ttl=3600):
         super().__init__(id)
         self.default_ttl = default_ttl
@@ -121,48 +128,282 @@ class TinyDnsBaseSource(BaseSource):
             'populate:   found %s records', len(zone.records) - before
         )
 
+    def _records_for_at(self, zone, name, lines, in_addr=False, lenient=False):
+        # @fqdn:ip:x:dist:ttl:timestamp:lo
+        # MX (and optional A)
+        if in_addr:
+            return []
+
+        # see if we can find a ttl on any of the lines, first one wins
+        ttl = self.default_ttl
+        for line in lines:
+            try:
+                ttl = int(lines[0][4])
+                break
+            except IndexError:
+                pass
+
+        values = []
+        for line in lines:
+            mx = line[2]
+            # if there's a . in the mx we hit a special case and use it as-is
+            if '.' not in mx:
+                # otherwise we treat it as the MX hostnam and construct the rest
+                mx = f'{mx}.ns.{zone.name}'
+            elif mx[-1] != '.':
+                mx = f'{mx}.'
+
+            # default distance is 0
+            try:
+                dist = line[3] or 0
+            except IndexError:
+                dist = 0
+
+            # if we have an IP then we need to create an A for the MX
+            ip = line[1]
+            if ip:
+                mx_name = zone.hostname_from_fqdn(mx)
+                yield Record.new(
+                    zone, mx_name, {'type': 'A', 'ttl': ttl, 'value': ip}
+                )
+
+            values.append({'preference': dist, 'exchange': mx})
+
+        yield Record.new(
+            zone, name, {'ttl': ttl, 'type': 'MX', 'values': values}
+        )
+
+    def _records_for_C(self, zone, name, lines, in_addr=False, lenient=False):
+        # Cfqdn:p:ttl:timestamp:lo
+        # CNAME
+        if in_addr:
+            return []
+
+        value = lines[0][1]
+        if value[-1] != '.':
+            value = f'{value}.'
+
+        # see if we can find a ttl on any of the lines, first one wins
+        ttl = self.default_ttl
+        for line in lines:
+            try:
+                ttl = int(lines[0][2])
+                break
+            except IndexError:
+                pass
+
+        return [
+            Record.new(
+                zone, name, {'ttl': ttl, 'type': 'CNAME', 'value': value}
+            )
+        ]
+
+    def _records_for_caret(
+        self, zone, name, lines, in_addr=False, lenient=False
+    ):
+        # .fqdn:ip:x:ttl:timestamp:lo
+        # NS (and optional A)
+        if not in_addr:
+            return []
+
+        raise NotImplementedError()
+
+    def _records_for_equal(
+        self, zone, name, lines, in_addr=False, lenient=False
+    ):
+        # =fqdn:ip:ttl:timestamp:lo
+        # A (in_addr False) & PTR (in_addr True)
+        return self._records_for_plus(
+            zone, name, lines, in_addr, lenient
+        ) + self._records_for_caret(zone, name, lines, in_addr, lenient)
+
+    def _records_for_dot(self, zone, name, lines, in_addr=False, lenient=False):
+        # .fqdn:ip:x:ttl:timestamp:lo
+        # NS (and optional A)
+        if not in_addr:
+            return []
+
+        # see if we can find a ttl on any of the lines, first one wins
+        ttl = self.default_ttl
+        for line in lines:
+            try:
+                ttl = int(lines[0][3])
+                break
+            except IndexError:
+                pass
+
+        values = []
+        for line in lines:
+            ns = line[2]
+            # if there's a . in the ns we hit a special case and use it as-is
+            if '.' not in ns:
+                # otherwise we treat it as the NS hostnam and construct the rest
+                ns = f'{ns}.ns.{zone.name}'
+            elif ns[-1] != '.':
+                ns = f'{ns}.'
+
+            # if we have an IP then we need to create an A for the MX
+            ip = line[1]
+            if ip:
+                ns_name = zone.hostname_from_fqdn(ns)
+                yield Record.new(
+                    zone, ns_name, {'type': 'A', 'ttl': ttl, 'value': ip}
+                )
+
+            values.append(ns)
+
+        yield Record.new(
+            zone, name, {'ttl': ttl, 'type': 'NS', 'values': values}
+        )
+
+    _records_for_amp = _records_for_dot
+
+    def _records_for_plus(
+        self, zone, name, lines, in_addr=False, lenient=False
+    ):
+        # +fqdn:ip:ttl:timestamp:lo
+        # A
+        if in_addr:
+            return []
+
+        # collect our ip(s)
+        ips = [l[1] for l in lines if l[1] != '0.0.0.0']
+
+        if not ips:
+            # we didn't find any value ips so nothing to do
+            return []
+
+        # see if we can find a ttl on any of the lines, first one wins
+        ttl = self.default_ttl
+        for line in lines:
+            try:
+                ttl = int(lines[0][2])
+                break
+            except IndexError:
+                pass
+
+        return [
+            Record.new(zone, name, {'ttl': ttl, 'type': 'A', 'values': ips})
+        ]
+
+    def _records_for_quote(
+        self, zone, name, lines, in_addr=False, lenient=False
+    ):
+        # 'fqdn:s:ttl:timestamp:lo
+        # TXT
+        if in_addr:
+            return []
+
+        # collect our ip(s)
+        values = [_decode_octal(l[1]) for l in lines]
+
+        # see if we can find a ttl on any of the lines, first one wins
+        ttl = self.default_ttl
+        for line in lines:
+            try:
+                ttl = int(lines[0][2])
+                break
+            except IndexError:
+                pass
+
+        return [
+            Record.new(
+                zone, name, {'ttl': ttl, 'type': 'TXT', 'values': values}
+            )
+        ]
+
+    def _records_for_three(
+        self, zone, name, lines, in_addr=False, lenient=False
+    ):
+        # 3fqdn:ip:ttl:timestamp:lo
+        # AAAA
+        if in_addr:
+            return []
+
+        # collect our ip(s)
+        ips = []
+        for line in lines:
+            # TinyDNS files have the ipv6 address written in full, but with the
+            # colons removed. This inserts a colon every 4th character to make
+            # the address correct.
+            ips.append(u':'.join(textwrap.wrap(line[1], 4)))
+
+        # see if we can find a ttl on any of the lines, first one wins
+        ttl = self.default_ttl
+        for line in lines:
+            try:
+                ttl = int(lines[0][2])
+                break
+            except IndexError:
+                pass
+
+        return [
+            Record.new(zone, name, {'ttl': ttl, 'type': 'AAAA', 'values': ips})
+        ]
+
+    def _records_for_six(self, zone, name, lines, in_addr=False, lenient=False):
+        # 6fqdn:ip:ttl:timestamp:lo
+        # AAAA (in_addr False) & PTR (in_addr True)
+        return self._records_for_three(
+            zone, name, lines, in_addr, lenient
+        ) + self._records_for_caret(zone, name, lines, in_addr, lenient)
+
+    TYPE_MAP = {
+        '=': _records_for_equal,  # A
+        '^': _records_for_caret,  # PTR
+        '.': _records_for_dot,  # NS
+        'C': _records_for_C,  # CNAME
+        '+': _records_for_plus,  # A
+        '@': _records_for_at,  # MX
+        '&': _records_for_amp,  # NS
+        '\'': _records_for_quote,  # TXT
+        '3': _records_for_three,  # AAAA
+        '6': _records_for_six,  # AAAA
+        # TODO:
+        #'S': _records_for_S, # SRV
+        # Sfqdn:ip:x:port:priority:weight:ttl:timestamp:lo
+        #':': _record_for_semicolon # arbitrary
+        # :fqdn:n:rdata:ttl:timestamp:lo
+    }
+
     def _populate_normal(self, zone, lenient):
-        type_map = {
-            '=': 'A',
-            '^': None,
-            '.': 'NS',
-            'C': 'CNAME',
-            '+': 'A',
-            '@': 'MX',
-            '\'': 'TXT',
-            '3': 'AAAA',
-            '6': 'AAAA',
-        }
         name_re = re.compile(fr'((?P<name>.+)\.)?{zone.name[:-1]}\.?$')
 
         data = defaultdict(lambda: defaultdict(list))
         for line in self._lines():
             _type = line[0]
-            if _type not in type_map:
-                # Something we don't care about
-                continue
-            _type = type_map[_type]
-            if not _type:
-                continue
 
             # Skip type, remove trailing comments, and omit newline
             line = line[1:].split('#', 1)[0]
             # Split on :'s including :: and strip leading/trailing ws
-            line = [p.strip() for p in self.split_re.split(line)]
+            line = [p.strip() for p in line.split(':')]
-            match = name_re.match(line[0])
+            # make sure the name portion matches the zone we're currently
-            if not match:
+            # working on
+            name = line[0]
+            if not name_re.match(name):
+                self.log.info('skipping name %s, not a match, %s: ', name, line)
                 continue
-            name = zone.hostname_from_fqdn(line[0])
+            # remove the zone name
-            data[name][_type].append(line[1:])
+            name = zone.hostname_from_fqdn(name)
+            data[_type][name].append(line)
 
-        for name, types in data.items():
+        pprint(data)
-            for _type, d in types.items():
+
-                data_for = getattr(self, f'_data_for_{_type}')
+        for _type, names in data.items():
-                data = data_for(_type, d)
+            records_for = self.TYPE_MAP.get(_type, None)
-                if data:
+            if _type not in self.TYPE_MAP:
-                    record = Record.new(
+                # Something we don't care about
-                        zone, name, data, source=self, lenient=lenient
+                self.log.info(
-                    )
+                    'skipping type %s, not supported/interested', _type
+                )
+                continue
+
+            print(_type)
+            for name, lines in names.items():
+                for record in records_for(
+                    self, zone, name, lines, lenient=lenient
+                ):
+                    pprint({'record': record})
                     try:
                         zone.add_record(record, lenient=lenient)
                     except SubzoneRecordException:
@@ -177,7 +418,7 @@ class TinyDnsBaseSource(BaseSource):
         for line in self._lines():
             _type = line[0]
             # We're only interested in = (A+PTR), and ^ (PTR) records
-            if _type not in ('=', '^'):
+            if _type not in ('=', '^', '&'):
                 continue
 
             # Skip type, remove trailing comments, and omit newline

tests/zones/tinydns/example.com

@@ -27,20 +27,26 @@ Ccname.other.foo:www.other.foo
 # MX
 @example.com::smtp-1-host.example.com:10
 @example.com.::smtp-2-host.example.com:20
-# MX with ttl
+# MX with ttl and ip
-@smtp.example.com::smtp-1-host.example.com:30:1800
+@smtp.example.com:21.22.23.24:smtp-1-host:30:1800
-@smtp.example.com.::smtp-2-host.example.com:40:1800
+@smtp.example.com.:22.23.24.25:smtp-2-host:40:1800
 
-# NS
+# NS for sub
 .sub.example.com::ns3.ns.com:30
 .sub.example.com.::ns4.ns.com:30
+# NS with ip
+.other.example.com:14.15.16.17:ns5:30
+.other.example.com.:15.16.17.18:ns6:30
 
 # A, under sub
-+www.sub.example.com::1.2.3.4
++www.sub.example.com:1.2.3.4
 
 # Top-level NS
 .example.com::ns1.ns.com
 .example.com.::ns2.ns.com
+# Top-level NS with automatic A
+&example.com:42.43.44.45:a
+&example.com.:43.44.45.46:b:31
 
 # sub special cases
 +a1.blah-asdf.subtest.com:10.2.3.5