From ea943e606ed358cc23cd91bd0ba7f2508ab81fcf Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:45:23 -0800 Subject: [PATCH 01/39] Avoid . on the end of files, but still test axfr default --- .gitignore | 3 ++- tests/test_octodns_source_axfr.py | 13 +++++++++++-- .../zones/{invalid.records. => invalid.records.tst} | 0 tests/zones/{invalid.zone. => invalid.zone.tst} | 0 tests/zones/{unit.tests. => unit.tests.tst} | 0 5 files changed, 13 insertions(+), 3 deletions(-) rename tests/zones/{invalid.records. => invalid.records.tst} (100%) rename tests/zones/{invalid.zone. => invalid.zone.tst} (100%) rename tests/zones/{unit.tests. => unit.tests.tst} (100%) diff --git a/.gitignore b/.gitignore index 715b687..5192821 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ *.pyc .coverage .env -/config/ /build/ +/config/ coverage.xml dist/ env/ @@ -14,4 +14,5 @@ htmlcov/ nosetests.xml octodns.egg-info/ output/ +tests/zones/unit.tests. tmp/ diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index a1d2e1c..8d0a527 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from shutil import copyfile from six import text_type from unittest import TestCase @@ -21,7 +22,7 @@ from octodns.record import ValidationError class TestAxfrSource(TestCase): source = AxfrSource('test', 'localhost') - forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.', + forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.tst', 'unit.tests', relativize=False) @patch('dns.zone.from_xfr') @@ -44,7 +45,7 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): - source = ZoneFileSource('test', './tests/zones') + source = ZoneFileSource('test', './tests/zones', file_extension='tst') def test_zonefiles_with_extension(self): source = ZoneFileSource('test', './tests/zones', 'extension') @@ -53,6 +54,14 @@ class TestZoneFileSource(TestCase): source.populate(valid) self.assertEquals(1, len(valid.records)) + def test_zonefiles_without_extension(self): + copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + source = ZoneFileSource('test', './tests/zones') + # Load zonefiles without a specified file extension + valid = Zone('unit.tests.', []) + source.populate(valid) + self.assertEquals(12, len(valid.records)) + def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) diff --git a/tests/zones/invalid.records. b/tests/zones/invalid.records.tst similarity index 100% rename from tests/zones/invalid.records. rename to tests/zones/invalid.records.tst diff --git a/tests/zones/invalid.zone. b/tests/zones/invalid.zone.tst similarity index 100% rename from tests/zones/invalid.zone. rename to tests/zones/invalid.zone.tst diff --git a/tests/zones/unit.tests. b/tests/zones/unit.tests.tst similarity index 100% rename from tests/zones/unit.tests. rename to tests/zones/unit.tests.tst From 37381bd2740a899c1944c5a5a274814ee807d1ff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:50:34 -0800 Subject: [PATCH 02/39] Skip the axfr default name test if we can't create the needed tests file --- tests/test_octodns_source_axfr.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 8d0a527..f732060 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -55,7 +55,12 @@ class TestZoneFileSource(TestCase): self.assertEquals(1, len(valid.records)) def test_zonefiles_without_extension(self): - copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + try: + copyfile('./tests/zones/unit.tests.tst', + './tests/zones/unit.tests.') + except: + self.skipTest('Unable to create unit.tests. (ending with .) so ' + 'skipping default filename testing.') source = ZoneFileSource('test', './tests/zones') # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) From dd1dbfbfdd1d66ee4c62487a6394fa5d10f2e443 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 28 Jan 2021 12:23:13 -0800 Subject: [PATCH 03/39] Rework copyfile and skip based on feedback from windows test --- tests/test_octodns_source_axfr.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index f732060..e0871ee 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from os.path import exists from shutil import copyfile from six import text_type from unittest import TestCase @@ -55,12 +56,19 @@ class TestZoneFileSource(TestCase): self.assertEquals(1, len(valid.records)) def test_zonefiles_without_extension(self): - try: - copyfile('./tests/zones/unit.tests.tst', - './tests/zones/unit.tests.') - except: + # Windows doesn't let files end with a `.` so we add a .tst to them in + # the repo and then try and create the `.` version we need for the + # default case (no extension.) + copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + # Unfortunately copyfile silently works and create the file without + # the `.` so we have to check to see if it did that + if exists('./tests/zones/unit.tests'): + # It did so we need to skip this test, that means windows won't + # have full code coverage, but skipping the test is going out of + # our way enough for a os-specific/oddball case. self.skipTest('Unable to create unit.tests. (ending with .) so ' 'skipping default filename testing.') + source = ZoneFileSource('test', './tests/zones') # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) From 4ce2563d2ec1fdd5b6e3cfbaf4e938f17b2a26e2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 28 Jan 2021 13:24:35 -0800 Subject: [PATCH 04/39] Remove the rest of the . ending files, clean up code and tests for better coverage --- octodns/provider/yaml.py | 6 ++-- octodns/source/axfr.py | 9 ++---- tests/config/simple-split.yaml | 3 ++ .../a.yaml | 0 .../aaaa.yaml | 0 .../cname.yaml | 0 .../real-ish-a.yaml | 0 .../simple-weighted.yaml | 0 .../split/{empty. => empty.tst}/.gitkeep | 0 .../12.yaml | 0 .../2.yaml | 0 .../test.yaml | 0 .../$unit.tests.yaml | 0 .../_srv._tcp.yaml | 0 .../{unit.tests. => unit.tests.tst}/aaaa.yaml | 0 .../cname.yaml | 0 .../dname.yaml | 0 .../excluded.yaml | 0 .../ignored.yaml | 0 .../included.yaml | 0 .../{unit.tests. => unit.tests.tst}/mx.yaml | 0 .../naptr.yaml | 0 .../{unit.tests. => unit.tests.tst}/ptr.yaml | 0 .../{unit.tests. => unit.tests.tst}/spf.yaml | 0 .../{unit.tests. => unit.tests.tst}/sub.yaml | 0 .../{unit.tests. => unit.tests.tst}/txt.yaml | 0 .../www.sub.yaml | 0 .../{unit.tests. => unit.tests.tst}/www.yaml | 0 .../{unordered. => unordered.tst}/abc.yaml | 0 .../{unordered. => unordered.tst}/xyz.yaml | 0 tests/test_octodns_provider_yaml.py | 31 ++++++++++++------- tests/test_octodns_source_axfr.py | 4 +-- 32 files changed, 30 insertions(+), 23 deletions(-) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/a.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/aaaa.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/cname.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/real-ish-a.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/simple-weighted.yaml (100%) rename tests/config/split/{empty. => empty.tst}/.gitkeep (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/12.yaml (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/2.yaml (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/test.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/$unit.tests.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/_srv._tcp.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/aaaa.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/cname.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/dname.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/excluded.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/ignored.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/included.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/mx.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/naptr.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/ptr.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/spf.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/sub.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/txt.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/www.sub.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/www.yaml (100%) rename tests/config/split/{unordered. => unordered.tst}/abc.yaml (100%) rename tests/config/split/{unordered. => unordered.tst}/xyz.yaml (100%) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 55a1632..3deca01 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -239,11 +239,13 @@ class SplitYamlProvider(YamlProvider): # instead of a file matching the record name. CATCHALL_RECORD_NAMES = ('*', '') - def __init__(self, id, directory, *args, **kwargs): + def __init__(self, id, directory, extension='.', *args, **kwargs): super(SplitYamlProvider, self).__init__(id, directory, *args, **kwargs) + self.extension = extension def _zone_directory(self, zone): - return join(self.directory, zone.name) + filename = '{}{}'.format(zone.name[:-1], self.extension) + return join(self.directory, filename) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index ed3f98f..e21f29f 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -216,7 +216,7 @@ class ZoneFileSource(AxfrBaseSource): # (optional, default true) check_origin: false ''' - def __init__(self, id, directory, file_extension=None, check_origin=True): + def __init__(self, id, directory, file_extension='.', check_origin=True): self.log = logging.getLogger('ZoneFileSource[{}]'.format(id)) self.log.debug('__init__: id=%s, directory=%s, file_extension=%s, ' 'check_origin=%s', id, @@ -229,12 +229,7 @@ class ZoneFileSource(AxfrBaseSource): self._zone_records = {} def _load_zone_file(self, zone_name): - - zone_filename = zone_name - if self.file_extension: - zone_filename = '{}{}'.format(zone_name, - self.file_extension.lstrip('.')) - + zone_filename = '{}{}'.format(zone_name[:-1], self.file_extension) zonefiles = listdir(self.directory) if zone_filename in zonefiles: try: diff --git a/tests/config/simple-split.yaml b/tests/config/simple-split.yaml index d106506..a798258 100644 --- a/tests/config/simple-split.yaml +++ b/tests/config/simple-split.yaml @@ -4,14 +4,17 @@ providers: in: class: octodns.provider.yaml.SplitYamlProvider directory: tests/config/split + extension: .tst dump: class: octodns.provider.yaml.SplitYamlProvider directory: env/YAML_TMP_DIR + extension: .tst # This is sort of ugly, but it shouldn't hurt anything. It'll just write out # the target file twice where it and dump are both used dump2: class: octodns.provider.yaml.SplitYamlProvider directory: env/YAML_TMP_DIR + extension: .tst simple: class: helpers.SimpleProvider geo: diff --git a/tests/config/split/dynamic.tests./a.yaml b/tests/config/split/dynamic.tests.tst/a.yaml similarity index 100% rename from tests/config/split/dynamic.tests./a.yaml rename to tests/config/split/dynamic.tests.tst/a.yaml diff --git a/tests/config/split/dynamic.tests./aaaa.yaml b/tests/config/split/dynamic.tests.tst/aaaa.yaml similarity index 100% rename from tests/config/split/dynamic.tests./aaaa.yaml rename to tests/config/split/dynamic.tests.tst/aaaa.yaml diff --git a/tests/config/split/dynamic.tests./cname.yaml b/tests/config/split/dynamic.tests.tst/cname.yaml similarity index 100% rename from tests/config/split/dynamic.tests./cname.yaml rename to tests/config/split/dynamic.tests.tst/cname.yaml diff --git a/tests/config/split/dynamic.tests./real-ish-a.yaml b/tests/config/split/dynamic.tests.tst/real-ish-a.yaml similarity index 100% rename from tests/config/split/dynamic.tests./real-ish-a.yaml rename to tests/config/split/dynamic.tests.tst/real-ish-a.yaml diff --git a/tests/config/split/dynamic.tests./simple-weighted.yaml b/tests/config/split/dynamic.tests.tst/simple-weighted.yaml similarity index 100% rename from tests/config/split/dynamic.tests./simple-weighted.yaml rename to tests/config/split/dynamic.tests.tst/simple-weighted.yaml diff --git a/tests/config/split/empty./.gitkeep b/tests/config/split/empty.tst/.gitkeep similarity index 100% rename from tests/config/split/empty./.gitkeep rename to tests/config/split/empty.tst/.gitkeep diff --git a/tests/config/split/subzone.unit.tests./12.yaml b/tests/config/split/subzone.unit.tests.tst/12.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./12.yaml rename to tests/config/split/subzone.unit.tests.tst/12.yaml diff --git a/tests/config/split/subzone.unit.tests./2.yaml b/tests/config/split/subzone.unit.tests.tst/2.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./2.yaml rename to tests/config/split/subzone.unit.tests.tst/2.yaml diff --git a/tests/config/split/subzone.unit.tests./test.yaml b/tests/config/split/subzone.unit.tests.tst/test.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./test.yaml rename to tests/config/split/subzone.unit.tests.tst/test.yaml diff --git a/tests/config/split/unit.tests./$unit.tests.yaml b/tests/config/split/unit.tests.tst/$unit.tests.yaml similarity index 100% rename from tests/config/split/unit.tests./$unit.tests.yaml rename to tests/config/split/unit.tests.tst/$unit.tests.yaml diff --git a/tests/config/split/unit.tests./_srv._tcp.yaml b/tests/config/split/unit.tests.tst/_srv._tcp.yaml similarity index 100% rename from tests/config/split/unit.tests./_srv._tcp.yaml rename to tests/config/split/unit.tests.tst/_srv._tcp.yaml diff --git a/tests/config/split/unit.tests./aaaa.yaml b/tests/config/split/unit.tests.tst/aaaa.yaml similarity index 100% rename from tests/config/split/unit.tests./aaaa.yaml rename to tests/config/split/unit.tests.tst/aaaa.yaml diff --git a/tests/config/split/unit.tests./cname.yaml b/tests/config/split/unit.tests.tst/cname.yaml similarity index 100% rename from tests/config/split/unit.tests./cname.yaml rename to tests/config/split/unit.tests.tst/cname.yaml diff --git a/tests/config/split/unit.tests./dname.yaml b/tests/config/split/unit.tests.tst/dname.yaml similarity index 100% rename from tests/config/split/unit.tests./dname.yaml rename to tests/config/split/unit.tests.tst/dname.yaml diff --git a/tests/config/split/unit.tests./excluded.yaml b/tests/config/split/unit.tests.tst/excluded.yaml similarity index 100% rename from tests/config/split/unit.tests./excluded.yaml rename to tests/config/split/unit.tests.tst/excluded.yaml diff --git a/tests/config/split/unit.tests./ignored.yaml b/tests/config/split/unit.tests.tst/ignored.yaml similarity index 100% rename from tests/config/split/unit.tests./ignored.yaml rename to tests/config/split/unit.tests.tst/ignored.yaml diff --git a/tests/config/split/unit.tests./included.yaml b/tests/config/split/unit.tests.tst/included.yaml similarity index 100% rename from tests/config/split/unit.tests./included.yaml rename to tests/config/split/unit.tests.tst/included.yaml diff --git a/tests/config/split/unit.tests./mx.yaml b/tests/config/split/unit.tests.tst/mx.yaml similarity index 100% rename from tests/config/split/unit.tests./mx.yaml rename to tests/config/split/unit.tests.tst/mx.yaml diff --git a/tests/config/split/unit.tests./naptr.yaml b/tests/config/split/unit.tests.tst/naptr.yaml similarity index 100% rename from tests/config/split/unit.tests./naptr.yaml rename to tests/config/split/unit.tests.tst/naptr.yaml diff --git a/tests/config/split/unit.tests./ptr.yaml b/tests/config/split/unit.tests.tst/ptr.yaml similarity index 100% rename from tests/config/split/unit.tests./ptr.yaml rename to tests/config/split/unit.tests.tst/ptr.yaml diff --git a/tests/config/split/unit.tests./spf.yaml b/tests/config/split/unit.tests.tst/spf.yaml similarity index 100% rename from tests/config/split/unit.tests./spf.yaml rename to tests/config/split/unit.tests.tst/spf.yaml diff --git a/tests/config/split/unit.tests./sub.yaml b/tests/config/split/unit.tests.tst/sub.yaml similarity index 100% rename from tests/config/split/unit.tests./sub.yaml rename to tests/config/split/unit.tests.tst/sub.yaml diff --git a/tests/config/split/unit.tests./txt.yaml b/tests/config/split/unit.tests.tst/txt.yaml similarity index 100% rename from tests/config/split/unit.tests./txt.yaml rename to tests/config/split/unit.tests.tst/txt.yaml diff --git a/tests/config/split/unit.tests./www.sub.yaml b/tests/config/split/unit.tests.tst/www.sub.yaml similarity index 100% rename from tests/config/split/unit.tests./www.sub.yaml rename to tests/config/split/unit.tests.tst/www.sub.yaml diff --git a/tests/config/split/unit.tests./www.yaml b/tests/config/split/unit.tests.tst/www.yaml similarity index 100% rename from tests/config/split/unit.tests./www.yaml rename to tests/config/split/unit.tests.tst/www.yaml diff --git a/tests/config/split/unordered./abc.yaml b/tests/config/split/unordered.tst/abc.yaml similarity index 100% rename from tests/config/split/unordered./abc.yaml rename to tests/config/split/unordered.tst/abc.yaml diff --git a/tests/config/split/unordered./xyz.yaml b/tests/config/split/unordered.tst/xyz.yaml similarity index 100% rename from tests/config/split/unordered./xyz.yaml rename to tests/config/split/unordered.tst/xyz.yaml diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 15e90da..38dfc11 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -207,18 +207,20 @@ class TestSplitYamlProvider(TestCase): def test_zone_directory(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unit.tests.', []) self.assertEqual( - join(dirname(__file__), 'config/split/unit.tests.'), + join(dirname(__file__), 'config/split/unit.tests.tst'), source._zone_directory(zone)) def test_apply_handles_existing_zone_directory(self): with TemporaryDirectory() as td: - provider = SplitYamlProvider('test', join(td.dirname, 'config')) - makedirs(join(td.dirname, 'config', 'does.exist.')) + provider = SplitYamlProvider('test', join(td.dirname, 'config'), + extension='.tst') + makedirs(join(td.dirname, 'config', 'does.exist.tst')) zone = Zone('does.exist.', []) self.assertTrue(isdir(provider._zone_directory(zone))) @@ -227,7 +229,8 @@ class TestSplitYamlProvider(TestCase): def test_provider(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unit.tests.', []) dynamic_zone = Zone('dynamic.tests.', []) @@ -246,9 +249,10 @@ class TestSplitYamlProvider(TestCase): with TemporaryDirectory() as td: # Add some subdirs to make sure that it can create them directory = join(td.dirname, 'sub', 'dir') - zone_dir = join(directory, 'unit.tests.') - dynamic_zone_dir = join(directory, 'dynamic.tests.') - target = SplitYamlProvider('test', directory) + zone_dir = join(directory, 'unit.tests.tst') + dynamic_zone_dir = join(directory, 'dynamic.tests.tst') + target = SplitYamlProvider('test', directory, + extension='.tst') # We add everything plan = target.plan(zone) @@ -335,7 +339,8 @@ class TestSplitYamlProvider(TestCase): def test_empty(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('empty.', []) @@ -345,7 +350,8 @@ class TestSplitYamlProvider(TestCase): def test_unsorted(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unordered.', []) @@ -356,14 +362,15 @@ class TestSplitYamlProvider(TestCase): source = SplitYamlProvider( 'test', join(dirname(__file__), 'config/split'), - enforce_order=False) + extension='.tst', enforce_order=False) # no exception source.populate(zone) self.assertEqual(2, len(zone.records)) def test_subzone_handling(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') # If we add `sub` as a sub-zone we'll reject `www.sub` zone = Zone('unit.tests.', ['sub']) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index e0871ee..44e04d0 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -46,10 +46,10 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): - source = ZoneFileSource('test', './tests/zones', file_extension='tst') + source = ZoneFileSource('test', './tests/zones', file_extension='.tst') def test_zonefiles_with_extension(self): - source = ZoneFileSource('test', './tests/zones', 'extension') + source = ZoneFileSource('test', './tests/zones', '.extension') # Load zonefiles with a specified file extension valid = Zone('ext.unit.tests.', []) source.populate(valid) From cda56a3ca7ffed9b62a5f960682a9c25b9173c05 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 4 Feb 2021 10:48:45 -0800 Subject: [PATCH 05/39] Force the value passed to FQDN to be a str --- octodns/record/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index f22eebf..7beb570 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -758,7 +758,9 @@ class _TargetValue(object): reasons.append('empty value') elif not data: reasons.append('missing value') - elif not FQDN(data, allow_underscores=True).is_valid: + # NOTE: FQDN complains if the data it receives isn't a str, it doesn't + # allow unicode... This is likely specific to 2.7 + elif not FQDN(str(data), allow_underscores=True).is_valid: reasons.append('{} value "{}" is not a valid FQDN' .format(_type, data)) elif not data.endswith('.'): From 4081c7b31b1ea31855b94c1572cc18e5a46410b7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 5 Feb 2021 11:55:37 -0800 Subject: [PATCH 06/39] Add the number of changes and zone name to "making changes" --- octodns/provider/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ae87844..eb097a2 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -91,7 +91,10 @@ class BaseProvider(BaseSource): self.log.info('apply: disabled') return 0 - self.log.info('apply: making changes') + zone_name = plan.desired.name + num_changes = len(plan.changes) + self.log.info('apply: making %d changes to %s', num_changes, + zone_name) self._apply(plan) return len(plan.changes) From 858628a867218e566013cde9d5a5fa920ecce382 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 5 Feb 2021 12:06:46 -0800 Subject: [PATCH 07/39] Update yaml test path to work on windows --- tests/test_octodns_provider_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 38dfc11..f255238 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -213,7 +213,7 @@ class TestSplitYamlProvider(TestCase): zone = Zone('unit.tests.', []) self.assertEqual( - join(dirname(__file__), 'config/split/unit.tests.tst'), + join(dirname(__file__), 'config/split', 'unit.tests.tst'), source._zone_directory(zone)) def test_apply_handles_existing_zone_directory(self): From 2346ebc1ce72edca8d066218cafb29e46d1f7ee9 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Mon, 8 Feb 2021 21:24:41 -0500 Subject: [PATCH 08/39] Added list of projects & resources. --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index a65e28f..84dc033 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator). - [Dynamic sources](#dynamic-sources) - [Contributing](#contributing) - [Getting help](#getting-help) +- [Related Projects & Resources](#related-projects--resources) - [License](#license) - [Authors](#authors) @@ -225,6 +226,8 @@ Most of the things included in OctoDNS are providers, the obvious difference bei The `class` key in the providers config section can be used to point to arbitrary classes in the python path so internal or 3rd party providers can easily be included with no coordination beyond getting them into PYTHONPATH, most likely installed into the virtualenv with OctoDNS. +For examples of building third-party sources and providers, see [Related Projects & Resources](#related-projects--resources). + ## Other Uses ### Syncing between providers @@ -286,6 +289,29 @@ Please see our [contributing document](/CONTRIBUTING.md) if you would like to pa If you have a problem or suggestion, please [open an issue](https://github.com/octodns/octodns/issues/new) in this repository, and we will do our best to help. Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). +## Related Projects & Resources + +- **GitHub Action:** [OctoDNS-Sync](https://github.com/marketplace/actions/octodns-sync) +- **Sample Implementations.** See how others are using it + - [`hackclub/dns`](https://github.com/hackclub/dns) + - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) + - [`g0v-network/domains`](https://github.com/g0v-network/domains) + - [`jekyll/dns`](https://github.com/jekyll/dns) + - [`parkr/dns`](https://github.com/parkr/dns) +- **Custom Sources & Providers.** + - [`octodns/octodns-ddns`](https://github.com/octodns/octodns-ddns): A simple Dynamic DNS source. + - [`doddo/octodns-lexicon`](https://github.com/doddo/octodns-lexicon): Use [Lexicon](https://github.com/AnalogJ/lexicon) providers as octoDNS providers. + - [`asyncon/octoblox`](https://github.com/asyncon/octoblox): [Infoblox](https://www.infoblox.com/) provider. + - [`sukiyaki/octodns-netbox`](https://github.com/sukiyaki/octodns-netbox): [NetBox](https://github.com/netbox-community/netbox) source. + - [`kompetenzbolzen/octodns-custom-provider`](https://github.com/kompetenzbolzen/octodns-custom-provider): zonefile provider & phpIPAM source. +- **Resources.** + - Article: [Visualising DNS records with Neo4j](https://medium.com/@costask/querying-and-visualising-octodns-records-with-neo4j-f4f72ab2d474) + code + - Video: [FOSDEM 2019 - DNS as code with octodns](https://archive.fosdem.org/2019/schedule/event/dns_octodns/) + - GitHub Blog: [Enabling DNS split authority with OctoDNS](https://github.blog/2017-04-27-enabling-split-authority-dns-with-octodns/) + - Tutorial: [How To Deploy and Manage Your DNS using OctoDNS on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-deploy-and-manage-your-dns-using-octodns-on-ubuntu-18-04) + +If you know of any other resources, please do let us know! + ## License OctoDNS is licensed under the [MIT license](LICENSE). From 9d4bd0aaec43764077a20ba23a2065cd1011c92a Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sun, 29 Nov 2020 23:42:51 +0800 Subject: [PATCH 09/39] Add support for LOC records --- docs/records.md | 1 + octodns/provider/yaml.py | 2 +- octodns/record/__init__.py | 190 ++++++++ tests/config/unit.tests.yaml | 28 ++ tests/test_octodns_manager.py | 14 +- tests/test_octodns_provider_constellix.py | 2 +- tests/test_octodns_provider_digitalocean.py | 2 +- tests/test_octodns_provider_dnsimple.py | 4 +- tests/test_octodns_provider_dnsmadeeasy.py | 2 +- tests/test_octodns_provider_easydns.py | 2 +- tests/test_octodns_provider_gandi.py | 4 +- tests/test_octodns_provider_powerdns.py | 4 +- tests/test_octodns_provider_yaml.py | 9 +- tests/test_octodns_record.py | 488 +++++++++++++++++++- tests/zones/unit.tests.tst | 4 + 15 files changed, 730 insertions(+), 26 deletions(-) diff --git a/docs/records.md b/docs/records.md index 4cf1e4b..e39a85d 100644 --- a/docs/records.md +++ b/docs/records.md @@ -10,6 +10,7 @@ OctoDNS supports the following record types: * `CAA` * `CNAME` * `DNAME` +* `LOC` * `MX` * `NAPTR` * `NS` diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 3deca01..8314f38 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -104,7 +104,7 @@ class YamlProvider(BaseProvider): ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'MX', + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'LOC', 'MX', 'NAPTR', 'NS', 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 7beb570..8ee2eaa 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -97,6 +97,7 @@ class Record(EqualityTupleMixin): 'CAA': CaaRecord, 'CNAME': CnameRecord, 'DNAME': DnameRecord, + 'LOC': LocRecord, 'MX': MxRecord, 'NAPTR': NaptrRecord, 'NS': NsRecord, @@ -879,6 +880,195 @@ class DnameRecord(_DynamicMixin, _ValueMixin, Record): _value_type = DnameValue +class LocValue(EqualityTupleMixin): + # TODO: work out how to do defaults per RFC + + @classmethod + def validate(cls, data, _type): + int_keys = [ + 'lat_degrees', + 'lat_minutes', + 'long_degrees', + 'long_minutes', + ] + + float_keys = [ + 'lat_seconds', + 'long_seconds', + 'altitude', + 'size', + 'precision_horz', + 'precision_vert', + ] + + direction_keys = [ + 'lat_direction', + 'long_direction', + ] + + if not isinstance(data, (list, tuple)): + data = (data,) + reasons = [] + for value in data: + for key in int_keys: + try: + int(value[key]) + if ( + ( + key == 'lat_degrees' and + not 0 <= int(value[key]) <= 90 + ) or ( + key == 'long_degrees' and + not 0 <= int(value[key]) <= 180 + ) or ( + key in ['lat_minutes', 'long_minutes'] and + not 0 <= int(value[key]) <= 59 + ) + ): + reasons.append('invalid value for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + except ValueError: + reasons.append('invalid {} "{}"' + .format(key, value[key])) + + for key in float_keys: + try: + float(value[key]) + if ( + ( + key in ['lat_seconds', 'long_seconds'] and + not 0 <= float(value[key]) <= 59.999 + ) or ( + key == 'altitude' and + not -100000.00 <= float(value[key]) <= 42849672.95 + ) or ( + key in ['size', + 'precision_horz', + 'precision_vert'] and + not 0 <= float(value[key]) <= 90000000.00 + ) + ): + reasons.append('invalid value for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + except ValueError: + reasons.append('invalid {} "{}"' + .format(key, value[key])) + + for key in direction_keys: + try: + str(value[key]) + if ( + key == 'lat_direction' and + value[key] not in ['N', 'S'] + ): + reasons.append('invalid direction for {} "{}"' + .format(key, value[key])) + if ( + key == 'long_direction' and + value[key] not in ['E', 'W'] + ): + reasons.append('invalid direction for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + return reasons + + @classmethod + def process(cls, values): + return [LocValue(v) for v in values] + + def __init__(self, value): + self.lat_degrees = int(value['lat_degrees']) + self.lat_minutes = int(value['lat_minutes']) + self.lat_seconds = float(value['lat_seconds']) + self.lat_direction = value['lat_direction'].upper() + self.long_degrees = int(value['long_degrees']) + self.long_minutes = int(value['long_minutes']) + self.long_seconds = float(value['long_seconds']) + self.long_direction = value['long_direction'].upper() + self.altitude = float(value['altitude']) + self.size = float(value['size']) + self.precision_horz = float(value['precision_horz']) + self.precision_vert = float(value['precision_vert']) + + @property + def data(self): + return { + 'lat_degrees': self.lat_degrees, + 'lat_minutes': self.lat_minutes, + 'lat_seconds': self.lat_seconds, + 'lat_direction': self.lat_direction, + 'long_degrees': self.long_degrees, + 'long_minutes': self.long_minutes, + 'long_seconds': self.long_seconds, + 'long_direction': self.long_direction, + 'altitude': self.altitude, + 'size': self.size, + 'precision_horz': self.precision_horz, + 'precision_vert': self.precision_vert, + } + + def __hash__(self): + return hash(( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + )) + + def _equality_tuple(self): + return ( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + ) + + def __repr__(self): + loc_format = "'{0} {1} {2:.3f} {3} " + \ + "{4} {5} {6:.3f} {7} " + \ + "{8:.2f}m {9:.2f}m {10:.2f}m {11:.2f}m'" + return loc_format.format( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + ) + + +class LocRecord(_ValuesMixin, Record): + _type = 'LOC' + _value_type = LocValue + + class MxValue(EqualityTupleMixin): @classmethod diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 7b84ac9..f5cf648 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -77,6 +77,34 @@ included: - test type: CNAME value: unit.tests. +loc: + ttl: 300 + type: LOC + values: + - altitude: 20 + lat_degrees: 31 + lat_direction: S + lat_minutes: 58 + lat_seconds: 52.1 + long_degrees: 115 + long_direction: E + long_minutes: 49 + long_seconds: 11.7 + precision_horz: 10 + precision_vert: 2 + size: 10 + - altitude: 20 + lat_degrees: 53 + lat_direction: N + lat_minutes: 13 + lat_seconds: 10 + long_degrees: 2 + long_direction: W + long_minutes: 18 + long_seconds: 26 + precision_horz: 1000 + precision_vert: 2 + size: 10 mx: ttl: 300 type: MX diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index f757466..3e0b122 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(16, tc) + self.assertEquals(17, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(26, tc) + self.assertEquals(27, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -215,13 +215,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(16, len(changes)) + self.assertEquals(17, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(16, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index bc17b50..7392271 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -132,7 +132,7 @@ class TestConstellixProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 0ad8f72..d1fa208 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -163,7 +163,7 @@ class TestDigitalOceanProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 8 + n = len(self.expected.records) - 9 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 92f32b1..e97751f 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -136,8 +136,8 @@ class TestDnsimpleProvider(TestCase): ] plan = provider.plan(self.expected) - # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 5 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 0ad059d..3c709cf 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 8df0e22..a0f03f9 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -374,7 +374,7 @@ class TestEasyDNSProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 7 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7e1c866..41b0109 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -192,8 +192,8 @@ class TestGandiProvider(TestCase): ] plan = provider.plan(self.expected) - # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + # No root NS, no ignored, no excluded, no LOC + n = len(self.expected.records) - 5 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 33b5e44..5605c5b 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -185,7 +185,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 3 + expected_n = len(expected.records) - 4 self.assertEquals(16, expected_n) # No diffs == no changes @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(19, len(expected.records)) + self.assertEquals(20, len(expected.records)) # A small change to a single record with requests_mock() as mock: diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f255238..08d1df0 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(20, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(17, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -106,6 +106,7 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('naptr')) self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) + self.assertTrue('values' in data.pop('loc')) # these are stored as singular 'value' self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index d55b3b8..ce40b9b 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -9,10 +9,11 @@ from six import text_type from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ - CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, MxRecord, \ - MxValue, NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, \ - SshfpRecord, SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, \ - Update, ValidationError, _Dynamic, _DynamicPool, _DynamicRule + CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, LocRecord, \ + LocValue, MxRecord, MxValue, NaptrRecord, NaptrValue, NsRecord, \ + PtrRecord, Record, SshfpRecord, SshfpValue, SpfRecord, SrvRecord, \ + SrvValue, TxtRecord, Update, ValidationError, _Dynamic, _DynamicPool, \ + _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -379,6 +380,98 @@ class TestRecord(TestCase): self.assertSingleValue(DnameRecord, 'target.foo.com.', 'other.foo.com.') + def test_loc(self): + a_values = [{ + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }] + a_data = {'ttl': 30, 'values': a_values} + a = LocRecord(self.zone, 'a', a_data) + self.assertEquals('a', a.name) + self.assertEquals('a.unit.tests.', a.fqdn) + self.assertEquals(30, a.ttl) + self.assertEquals(a_values[0]['lat_degrees'], a.values[0].lat_degrees) + self.assertEquals(a_values[0]['lat_minutes'], a.values[0].lat_minutes) + self.assertEquals(a_values[0]['lat_seconds'], a.values[0].lat_seconds) + self.assertEquals(a_values[0]['lat_direction'], + a.values[0].lat_direction) + self.assertEquals(a_values[0]['long_degrees'], + a.values[0].long_degrees) + self.assertEquals(a_values[0]['long_minutes'], + a.values[0].long_minutes) + self.assertEquals(a_values[0]['long_seconds'], + a.values[0].long_seconds) + self.assertEquals(a_values[0]['long_direction'], + a.values[0].long_direction) + self.assertEquals(a_values[0]['altitude'], a.values[0].altitude) + self.assertEquals(a_values[0]['size'], a.values[0].size) + self.assertEquals(a_values[0]['precision_horz'], + a.values[0].precision_horz) + self.assertEquals(a_values[0]['precision_vert'], + a.values[0].precision_vert) + + b_value = { + 'lat_degrees': 32, + 'lat_minutes': 7, + 'lat_seconds': 19, + 'lat_direction': 'S', + 'long_degrees': 116, + 'long_minutes': 2, + 'long_seconds': 25, + 'long_direction': 'E', + 'altitude': 10, + 'size': 1, + 'precision_horz': 10000, + 'precision_vert': 10, + } + b_data = {'ttl': 30, 'value': b_value} + b = LocRecord(self.zone, 'b', b_data) + self.assertEquals(b_value['lat_degrees'], b.values[0].lat_degrees) + self.assertEquals(b_value['lat_minutes'], b.values[0].lat_minutes) + self.assertEquals(b_value['lat_seconds'], b.values[0].lat_seconds) + self.assertEquals(b_value['lat_direction'], b.values[0].lat_direction) + self.assertEquals(b_value['long_degrees'], b.values[0].long_degrees) + self.assertEquals(b_value['long_minutes'], b.values[0].long_minutes) + self.assertEquals(b_value['long_seconds'], b.values[0].long_seconds) + self.assertEquals(b_value['long_direction'], + b.values[0].long_direction) + self.assertEquals(b_value['altitude'], b.values[0].altitude) + self.assertEquals(b_value['size'], b.values[0].size) + self.assertEquals(b_value['precision_horz'], + b.values[0].precision_horz) + self.assertEquals(b_value['precision_vert'], + b.values[0].precision_vert) + self.assertEquals(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in lat_direction causes change + other = LocRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].lat_direction = 'N' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in altitude causes change + other.values[0].altitude = a.values[0].altitude + other.values[0].altitude = -10 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # __repr__ doesn't blow up + a.__repr__() + def test_mx(self): a_values = [{ 'preference': 10, @@ -1127,6 +1220,93 @@ class TestRecord(TestCase): self.assertTrue(d >= d) self.assertTrue(d <= d) + def test_loc_value(self): + a = LocValue({ + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }) + b = LocValue({ + 'lat_degrees': 32, + 'lat_minutes': 7, + 'lat_seconds': 19, + 'lat_direction': 'S', + 'long_degrees': 116, + 'long_minutes': 2, + 'long_seconds': 25, + 'long_direction': 'E', + 'altitude': 10, + 'size': 1, + 'precision_horz': 10000, + 'precision_vert': 10, + }) + c = LocValue({ + 'lat_degrees': 53, + 'lat_minutes': 14, + 'lat_seconds': 10, + 'lat_direction': 'N', + 'long_degrees': 2, + 'long_minutes': 18, + 'long_seconds': 26, + 'long_direction': 'W', + 'altitude': 10, + 'size': 1, + 'precision_horz': 1000, + 'precision_vert': 10, + }) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b < c) + + self.assertTrue(c > a) + self.assertTrue(c > b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b <= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c >= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + def test_mx_value(self): a = MxValue({'preference': 0, 'priority': 'a', 'exchange': 'v', 'value': '1'}) @@ -1960,6 +2140,306 @@ class TestRecordValidation(TestCase): self.assertEquals(['DNAME value "foo.bar.com" missing trailing .'], ctx.exception.reasons) + def test_LOC(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + # missing int key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_degrees'], ctx.exception.reasons) + + # missing float key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_seconds'], ctx.exception.reasons) + + # missing text key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_direction'], ctx.exception.reasons) + + # invalid direction + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'U', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid direction for lat_direction "U"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'N', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid direction for long_direction "N"'], + ctx.exception.reasons) + + # invalid degrees + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 360, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_degrees "360"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 'nope', + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid lat_degrees "nope"'], + ctx.exception.reasons) + + # invalid minutes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 60, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_minutes "60"'], + ctx.exception.reasons) + + # invalid seconds + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 60, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_seconds "60"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 'nope', + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid lat_seconds "nope"'], + ctx.exception.reasons) + + # invalid altitude + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': -666666, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for altitude "-666666"'], + ctx.exception.reasons) + + # invalid size + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 99999999.99, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for size "99999999.99"'], + ctx.exception.reasons) + def test_MX(self): # doesn't blow up Record.new(self.zone, '', { diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index 838de88..3a25415 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -32,6 +32,10 @@ mx 300 IN MX 20 smtp-2.unit.tests. mx 300 IN MX 30 smtp-3.unit.tests. mx 300 IN MX 40 smtp-1.unit.tests. +; LOC Records +loc 300 IN LOC 31 58 52.1 S 115 49 11.7 E 20m 10m 10m 2m +loc 300 IN LOC 53 14 10 N 2 18 26 W 20m 10m 1000m 2m + ; A Records @ 300 IN A 1.2.3.4 @ 300 IN A 1.2.3.5 From 3ac8d0fa1c3d6e1a97afd681d1872183ce5aeeae Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 30 Nov 2020 23:44:55 +0800 Subject: [PATCH 10/39] Add LOC record support to AXFR source --- README.md | 2 +- octodns/source/axfr.py | 31 +++++++++++++++++++++++++++++-- tests/test_octodns_source_axfr.py | 6 +++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a65e28f..08933aa 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | | [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [UltraDns](/octodns/provider/ultra.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | | -| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | +| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | | All | Yes | config | diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index e21f29f..7a45155 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -26,8 +26,8 @@ class AxfrBaseSource(BaseSource): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', - 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) def __init__(self, id): super(AxfrBaseSource, self).__init__(id) @@ -58,6 +58,33 @@ class AxfrBaseSource(BaseSource): 'values': values } + def _data_for_LOC(self, _type, records): + values = [] + for record in records: + lat_degrees, lat_minutes, lat_seconds, lat_direction, \ + long_degrees, long_minutes, long_seconds, long_direction, \ + altitude, size, precision_horz, precision_vert = \ + record['value'].replace('m', '').split(' ', 11) + values.append({ + 'lat_degrees': lat_degrees, + 'lat_minutes': lat_minutes, + 'lat_seconds': lat_seconds, + 'lat_direction': lat_direction, + 'long_degrees': long_degrees, + 'long_minutes': long_minutes, + 'long_seconds': long_seconds, + 'long_direction': long_direction, + 'altitude': altitude, + 'size': size, + 'precision_horz': precision_horz, + 'precision_vert': precision_vert, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_MX(self, _type, records): values = [] for record in records: diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 44e04d0..8cf6929 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -36,7 +36,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(12, len(got.records)) + self.assertEquals(13, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -79,12 +79,12 @@ class TestZoneFileSource(TestCase): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(13, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(13, len(again.records)) # bust the cache del self.source._zone_records[valid.name] From 8338e8db5855ff91a40847a5c6a4aaf6dc3945b8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 30 Nov 2020 23:56:04 +0800 Subject: [PATCH 11/39] Add LOC record support to Cloudflare provider --- README.md | 2 +- octodns/provider/cloudflare.py | 48 ++++++- .../cloudflare-dns_records-page-2.json | 56 +------- .../cloudflare-dns_records-page-3.json | 128 ++++++++++++++++++ tests/test_octodns_provider_cloudflare.py | 62 ++++++++- 5 files changed, 234 insertions(+), 62 deletions(-) create mode 100644 tests/fixtures/cloudflare-dns_records-page-3.json diff --git a/README.md b/README.md index 08933aa..5d57821 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ The above command pulled the existing data out of Route53 and placed the results |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index db937e5..05fef20 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -75,8 +75,8 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', - 'SRV', 'SPF', 'TXT')) + SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', + 'PTR', 'SRV', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -133,6 +133,7 @@ class CloudflareProvider(BaseProvider): timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) if resp.status_code == 400: + self.log.debug('_request: data=%s', data) raise CloudflareError(resp.json()) if resp.status_code == 403: raise CloudflareAuthenticationError(resp.json()) @@ -216,6 +217,30 @@ class CloudflareProvider(BaseProvider): _data_for_ALIAS = _data_for_CNAME _data_for_PTR = _data_for_CNAME + def _data_for_LOC(self, _type, records): + values = [] + for record in records: + r = record['data'] + values.append({ + 'lat_degrees': int(r['lat_degrees']), + 'lat_minutes': int(r['lat_minutes']), + 'lat_seconds': float(r['lat_seconds']), + 'lat_direction': r['lat_direction'], + 'long_degrees': int(r['long_degrees']), + 'long_minutes': int(r['long_minutes']), + 'long_seconds': float(r['long_seconds']), + 'long_direction': r['long_direction'], + 'altitude': float(r['altitude']), + 'size': float(r['size']), + 'precision_horz': float(r['precision_horz']), + 'precision_vert': float(r['precision_vert']), + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_MX(self, _type, records): values = [] for r in records: @@ -384,6 +409,25 @@ class CloudflareProvider(BaseProvider): _contents_for_PTR = _contents_for_CNAME + def _contents_for_LOC(self, record): + for value in record.values: + yield { + 'data': { + 'lat_degrees': value.lat_degrees, + 'lat_minutes': value.lat_minutes, + 'lat_seconds': value.lat_seconds, + 'lat_direction': value.lat_direction, + 'long_degrees': value.long_degrees, + 'long_minutes': value.long_minutes, + 'long_seconds': value.long_seconds, + 'long_direction': value.long_direction, + 'altitude': value.altitude, + 'size': value.size, + 'precision_horz': value.precision_horz, + 'precision_vert': value.precision_vert, + } + } + def _contents_for_MX(self, record): for value in record.values: yield { diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index b0bbaef..8075ba5 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -173,64 +173,14 @@ "meta": { "auto_added": false } - }, - { - "id": "fc12ab34cd5611334422ab3322997656", - "type": "SRV", - "name": "_srv._tcp.unit.tests", - "data": { - "service": "_srv", - "proto": "_tcp", - "name": "unit.tests", - "priority": 12, - "weight": 20, - "port": 30, - "target": "foo-2.unit.tests" - }, - "proxiable": true, - "proxied": false, - "ttl": 600, - "locked": false, - "zone_id": "ff12ab34cd5611334422ab3322997650", - "zone_name": "unit.tests", - "modified_on": "2017-03-11T18:01:43.940682Z", - "created_on": "2017-03-11T18:01:43.940682Z", - "meta": { - "auto_added": false - } - }, - { - "id": "fc12ab34cd5611334422ab3322997656", - "type": "SRV", - "name": "_srv._tcp.unit.tests", - "data": { - "service": "_srv", - "proto": "_tcp", - "name": "unit.tests", - "priority": 10, - "weight": 20, - "port": 30, - "target": "foo-1.unit.tests" - }, - "proxiable": true, - "proxied": false, - "ttl": 600, - "locked": false, - "zone_id": "ff12ab34cd5611334422ab3322997650", - "zone_name": "unit.tests", - "modified_on": "2017-03-11T18:01:43.940682Z", - "created_on": "2017-03-11T18:01:43.940682Z", - "meta": { - "auto_added": false - } } ], "result_info": { "page": 2, - "per_page": 11, - "total_pages": 2, + "per_page": 10, + "total_pages": 3, "count": 10, - "total_count": 20 + "total_count": 24 }, "success": true, "errors": [], diff --git a/tests/fixtures/cloudflare-dns_records-page-3.json b/tests/fixtures/cloudflare-dns_records-page-3.json new file mode 100644 index 0000000..0f06ab4 --- /dev/null +++ b/tests/fixtures/cloudflare-dns_records-page-3.json @@ -0,0 +1,128 @@ +{ + "result": [ + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 12, + "weight": 20, + "port": 30, + "target": "foo-2.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 10, + "weight": 20, + "port": 30, + "target": "foo-1.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "372e67954025e0ba6aaa6d586b9e0b59", + "type": "LOC", + "name": "loc.unit.tests", + "content": "IN LOC 31 58 52.1 S 115 49 11.7 E 20m 10m 10m 2m", + "proxiable": true, + "proxied": false, + "ttl": 300, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "created_on": "2020-01-28T05:20:00.12345Z", + "modified_on": "2020-01-28T05:20:00.12345Z", + "data": { + "lat_degrees": 31, + "lat_minutes": 58, + "lat_seconds": 52.1, + "lat_direction": "S", + "long_degrees": 115, + "long_minutes": 49, + "long_seconds": 11.7, + "long_direction": "E", + "altitude": 20, + "size": 10, + "precision_horz": 10, + "precision_vert": 2 + }, + "meta": { + "auto_added": true, + "source": "primary" + } + }, + { + "id": "372e67954025e0ba6aaa6d586b9e0b59", + "type": "LOC", + "name": "loc.unit.tests", + "content": "IN LOC 53 14 10 N 2 18 26 W 20m 10m 1000m 2m", + "proxiable": true, + "proxied": false, + "ttl": 300, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "created_on": "2020-01-28T05:20:00.12345Z", + "modified_on": "2020-01-28T05:20:00.12345Z", + "data": { + "lat_degrees": 53, + "lat_minutes": 13, + "lat_seconds": 10, + "lat_direction": "N", + "long_degrees": 2, + "long_minutes": 18, + "long_seconds": 26, + "long_direction": "W", + "altitude": 20, + "size": 10, + "precision_horz": 1000, + "precision_vert": 2 + }, + "meta": { + "auto_added": true, + "source": "primary" + } + } + ], + "result_info": { + "page": 3, + "per_page": 10, + "total_pages": 3, + "count": 4, + "total_count": 24 + }, + "success": true, + "errors": [], + "messages": [] +} diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 735d95c..3727e20 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -177,10 +177,14 @@ class TestCloudflareProvider(TestCase): 'page-2.json') as fh: mock.get('{}?page=2'.format(base), status_code=200, text=fh.read()) + with open('tests/fixtures/cloudflare-dns_records-' + 'page-3.json') as fh: + mock.get('{}?page=3'.format(base), status_code=200, + text=fh.read()) zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) @@ -189,7 +193,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(14, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -203,12 +207,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 22 # individual record creates + ] + [None] * 24 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(14, len(plan.changes)) + self.assertEquals(14, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -234,7 +238,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(23, provider._request.call_count) + self.assertEquals(25, provider._request.call_count) provider._request.reset_mock() @@ -566,6 +570,52 @@ class TestCloudflareProvider(TestCase): 'content': 'foo.bar.com.' }, list(ptr_record_contents)[0]) + def test_loc(self): + self.maxDiff = None + provider = CloudflareProvider('test', 'email', 'token') + + zone = Zone('unit.tests.', []) + # LOC record + loc_record = Record.new(zone, 'example', { + 'ttl': 300, + 'type': 'LOC', + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + loc_record_contents = provider._gen_data(loc_record) + self.assertEquals({ + 'name': 'example.unit.tests', + 'ttl': 300, + 'type': 'LOC', + 'data': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }, list(loc_record_contents)[0]) + def test_srv(self): provider = CloudflareProvider('test', 'email', 'token') From 5963c8b89453e51e01d2b983d0421cb707fa1568 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 1 Dec 2020 00:02:49 +0800 Subject: [PATCH 12/39] Force order of Delete() -> Create() -> Update() in Cloudflare provider Addresses issues with changing between A, AAAA and CNAME records in both directions with the Cloudflare API See also: github/octodns#507 See also: github/octodns#586 See also: github/octodns#587 --- octodns/provider/cloudflare.py | 10 ++++++++++ tests/test_octodns_provider_cloudflare.py | 10 +++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 05fef20..f9d9fd0 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -143,6 +143,11 @@ class CloudflareProvider(BaseProvider): resp.raise_for_status() return resp.json() + def _change_keyer(self, change): + key = change.__class__.__name__ + order = {'Delete': 0, 'Create': 1, 'Update': 2} + return order[key] + @property def zones(self): if self._zones is None: @@ -660,6 +665,11 @@ class CloudflareProvider(BaseProvider): self.zones[name] = zone_id self._zone_records[name] = {} + # Force the operation order to be Delete() -> Create() -> Update() + # This will help avoid problems in updating a CNAME record into an + # A record and vice-versa + changes.sort(key=self._change_keyer) + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 3727e20..127480b 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -340,6 +340,10 @@ class TestCloudflareProvider(TestCase): self.assertTrue(plan.exists) # creates a the new value and then deletes all the old provider._request.assert_has_calls([ + call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' + 'dns_records/fc12ab34cd5611334422ab3322997653'), + call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' + 'dns_records/fc12ab34cd5611334422ab3322997654'), call('PUT', '/zones/42/dns_records/' 'fc12ab34cd5611334422ab3322997655', data={ 'content': '3.2.3.4', @@ -347,11 +351,7 @@ class TestCloudflareProvider(TestCase): 'name': 'ttl.unit.tests', 'proxied': False, 'ttl': 300 - }), - call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' - 'dns_records/fc12ab34cd5611334422ab3322997653'), - call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' - 'dns_records/fc12ab34cd5611334422ab3322997654') + }) ]) def test_update_add_swap(self): From 5852ae7a2ffdac74e3bff07b44d2a2c0db147024 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 13 Jan 2021 21:46:23 +0800 Subject: [PATCH 13/39] Detect changes to LOC record correctly --- octodns/provider/cloudflare.py | 18 +++++++++++++++++- tests/test_octodns_provider_cloudflare.py | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index f9d9fd0..cc1169d 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -505,7 +505,7 @@ class CloudflareProvider(BaseProvider): # new records cleanly. In general when there are multiple records for a # name & type each will have a distinct/consistent `content` that can # serve as a unique identifier. - # BUT... there are exceptions. MX, CAA, and SRV don't have a simple + # BUT... there are exceptions. MX, CAA, LOC and SRV don't have a simple # content as things are currently implemented so we need to handle # those explicitly and create unique/hashable strings for them. _type = data['type'] @@ -517,6 +517,22 @@ class CloudflareProvider(BaseProvider): elif _type == 'SRV': data = data['data'] return '{port} {priority} {target} {weight}'.format(**data) + elif _type == 'LOC': + data = data['data'] + loc = ( + '{lat_degrees}', + '{lat_minutes}', + '{lat_seconds}', + '{lat_direction}', + '{long_degrees}', + '{long_minutes}', + '{long_seconds}', + '{long_direction}', + '{altitude}', + '{size}', + '{precision_horz}', + '{precision_vert}') + return ' '.join(loc).format(**data) return data['content'] def _apply_Create(self, change): diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 127480b..d4fa74e 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -747,6 +747,23 @@ class TestCloudflareProvider(TestCase): }, 'type': 'SRV', }), + ('31 58 52.1 S 115 49 11.7 E 20 10 10 2', { + 'data': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }, + 'type': 'LOC', + }), ): self.assertEqual(expected, provider._gen_key(data)) From f5c2f3a141ada969b9ef05e74c133abe877a8c10 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sun, 31 Jan 2021 22:45:48 +0800 Subject: [PATCH 14/39] Add LOC record support to PowerDNS provider --- octodns/provider/powerdns.py | 52 ++++++++++++++++++++++++- tests/fixtures/powerdns-full-data.json | 16 ++++++++ tests/test_octodns_provider_powerdns.py | 6 +-- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index de7743c..ee24eab 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -15,8 +15,8 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', - 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'LOC', 'MX', 'NAPTR', + 'NS', 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 def __init__(self, id, host, api_key, port=8081, @@ -102,6 +102,33 @@ class PowerDnsBaseProvider(BaseProvider): _data_for_SPF = _data_for_quoted _data_for_TXT = _data_for_quoted + def _data_for_LOC(self, rrset): + values = [] + for record in rrset['records']: + lat_degrees, lat_minutes, lat_seconds, lat_direction, \ + long_degrees, long_minutes, long_seconds, long_direction, \ + altitude, size, precision_horz, precision_vert = \ + record['content'].replace('m', '').split(' ', 11) + values.append({ + 'lat_degrees': int(lat_degrees), + 'lat_minutes': int(lat_minutes), + 'lat_seconds': float(lat_seconds), + 'lat_direction': lat_direction, + 'long_degrees': int(long_degrees), + 'long_minutes': int(long_minutes), + 'long_seconds': float(long_seconds), + 'long_direction': long_direction, + 'altitude': float(altitude), + 'size': float(size), + 'precision_horz': float(precision_horz), + 'precision_vert': float(precision_vert), + }) + return { + 'ttl': rrset['ttl'], + 'type': rrset['type'], + 'values': values + } + def _data_for_MX(self, rrset): values = [] for record in rrset['records']: @@ -285,6 +312,27 @@ class PowerDnsBaseProvider(BaseProvider): _records_for_SPF = _records_for_quoted _records_for_TXT = _records_for_quoted + def _records_for_LOC(self, record): + return [{ + 'content': + '%d %d %0.3f %s %d %d %.3f %s %0.2fm %0.2fm %0.2fm %0.2fm' % + ( + int(v.lat_degrees), + int(v.lat_minutes), + float(v.lat_seconds), + v.lat_direction, + int(v.long_degrees), + int(v.long_minutes), + float(v.long_seconds), + v.long_direction, + float(v.altitude), + float(v.size), + float(v.precision_horz), + float(v.precision_vert) + ), + 'disabled': False + } for v in record.values] + def _records_for_MX(self, record): return [{ 'content': '{} {}'.format(v.preference, v.exchange), diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 3d445d4..7da8232 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -32,6 +32,22 @@ "ttl": 300, "type": "MX" }, + { + "comments": [], + "name": "loc.unit.tests.", + "records": [ + { + "content": "31 58 52.100 S 115 49 11.700 E 20.00m 10.00m 10.00m 2.00m", + "disabled": false + }, + { + "content": "53 13 10.000 N 2 18 26.000 W 20.00m 10.00m 1000.00m 2.00m", + "disabled": false + } + ], + "ttl": 300, + "type": "LOC" + }, { "comments": [], "name": "sub.unit.tests.", diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 5605c5b..f3f99e2 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -185,8 +185,8 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 4 - self.assertEquals(16, expected_n) + expected_n = len(expected.records) - 3 + self.assertEquals(17, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -194,7 +194,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(17, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) From 9e70caf92c8a44e0328117c3500c1af518e350aa Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 9 Feb 2021 20:47:03 +0800 Subject: [PATCH 15/39] Update test_octodns_source_axfr.py to catch up with upstream changes/tests --- tests/test_octodns_source_axfr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 8cf6929..cd493cb 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -73,7 +73,7 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(13, len(valid.records)) def test_populate(self): # Valid zone file in directory From e991d8dc10d7693e8d4242f14e604fcd6d9df1c5 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Tue, 9 Feb 2021 16:02:52 -0500 Subject: [PATCH 16/39] Removed implementation example. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 84dc033..9f608a0 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,6 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) - [`g0v-network/domains`](https://github.com/g0v-network/domains) - [`jekyll/dns`](https://github.com/jekyll/dns) - - [`parkr/dns`](https://github.com/parkr/dns) - **Custom Sources & Providers.** - [`octodns/octodns-ddns`](https://github.com/octodns/octodns-ddns): A simple Dynamic DNS source. - [`doddo/octodns-lexicon`](https://github.com/doddo/octodns-lexicon): Use [Lexicon](https://github.com/AnalogJ/lexicon) providers as octoDNS providers. From a8366aa02db6fac3d090f0d9a12edf3f7916ab4a Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Tue, 9 Feb 2021 16:06:25 -0500 Subject: [PATCH 17/39] Stopped running CI for doc changes. --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f62f96..337ebb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,8 @@ name: OctoDNS -on: [pull_request] +on: + pull_request: + paths-ignore: + - '**.md' jobs: ci: From 39d86f023e411c0af869a14ce7f12eda41f4d649 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Fri, 12 Feb 2021 00:29:48 +1100 Subject: [PATCH 18/39] powerdns: deletes before replaces --- octodns/provider/powerdns.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index de7743c..a7f150b 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session import logging +from operator import itemgetter from ..record import Create, Record from .base import BaseProvider @@ -381,6 +382,12 @@ class PowerDnsBaseProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ mods.append(getattr(self, '_mod_{}'.format(class_name))(change)) + + # Ensure that any DELETE modifications always occur before any REPLACE + # modifications. This ensures that an A record can be replaced by a + # CNAME record and vice-versa. + mods = sorted(mods, key=itemgetter('changetype')) + self.log.debug('_apply: sending change request') try: From fdf74f9dd36365674473d8ee8d110da92f99b2c8 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Fri, 12 Feb 2021 00:48:42 +1100 Subject: [PATCH 19/39] powerdns: sort in place --- octodns/provider/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index a7f150b..ec30559 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -386,7 +386,7 @@ class PowerDnsBaseProvider(BaseProvider): # Ensure that any DELETE modifications always occur before any REPLACE # modifications. This ensures that an A record can be replaced by a # CNAME record and vice-versa. - mods = sorted(mods, key=itemgetter('changetype')) + mods.sort(key=itemgetter('changetype')) self.log.debug('_apply: sending change request') From 6034e8022f24a83ca80d1c32e1fc0420ecaca606 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 11 Feb 2021 17:05:42 -0800 Subject: [PATCH 20/39] Swap import order --- octodns/provider/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index ec30559..8ffff46 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -6,8 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from requests import HTTPError, Session -import logging from operator import itemgetter +import logging from ..record import Create, Record from .base import BaseProvider From 29c8f3253ed3d17cc8490bfc6e4b8e6f108b1411 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Thu, 11 Feb 2021 20:35:32 -0500 Subject: [PATCH 21/39] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9f608a0..c784c0a 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,7 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - Video: [FOSDEM 2019 - DNS as code with octodns](https://archive.fosdem.org/2019/schedule/event/dns_octodns/) - GitHub Blog: [Enabling DNS split authority with OctoDNS](https://github.blog/2017-04-27-enabling-split-authority-dns-with-octodns/) - Tutorial: [How To Deploy and Manage Your DNS using OctoDNS on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-deploy-and-manage-your-dns-using-octodns-on-ubuntu-18-04) + - Cloudflare Blog: [Improving the Resiliency of Our Infrastructure DNS Zone](https://blog.cloudflare.com/improving-the-resiliency-of-our-infrastructure-dns-zone/) If you know of any other resources, please do let us know! From 45d5da23cfc0940a536dc24102994820ab67f032 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:02:36 +0800 Subject: [PATCH 22/39] Add NULL SRV record examples to unit tests --- tests/config/unit.tests.yaml | 16 ++++++++++++++++ tests/zones/unit.tests.tst | 3 +++ 2 files changed, 19 insertions(+) diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 7b84ac9..03f13a1 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -36,6 +36,22 @@ - flags: 0 tag: issue value: ca.unit.tests +_imap._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 +_pop3._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 _srv._tcp: ttl: 600 type: SRV diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index 838de88..b48b749 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -20,6 +20,9 @@ caa 1800 IN CAA 0 iodef "mailto:admin@unit.tests" ; SRV Records _srv._tcp 600 IN SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 IN SRV 10 20 30 foo-2.unit.tests. +; NULL SRV Records +_pop3._tcp 600 IN SRV 0 0 0 . +_imap._tcp 600 IN SRV 0 0 0 . ; TXT Records txt 600 IN TXT "Bah bah black sheep" From 403be8bb838bb8431700e61df6a4f150fe8b8a07 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:13:17 +0800 Subject: [PATCH 23/39] Fix handling of NULL SRV records in Cloudflare provider --- octodns/provider/cloudflare.py | 8 ++- .../cloudflare-dns_records-page-2.json | 50 +++++++++++++++++++ tests/test_octodns_provider_cloudflare.py | 12 ++--- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index db937e5..0f5c6cf 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -239,11 +239,13 @@ class CloudflareProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for r in records: + target = ('{}.'.format(r['data']['target']) + if r['data']['target'] != "." else ".") values.append({ 'priority': r['data']['priority'], 'weight': r['data']['weight'], 'port': r['data']['port'], - 'target': '{}.'.format(r['data']['target']), + 'target': target, }) return { 'type': _type, @@ -405,6 +407,8 @@ class CloudflareProvider(BaseProvider): name = subdomain for value in record.values: + target = value.target[:-1] if value.target != "." else "." + yield { 'data': { 'service': service, @@ -413,7 +417,7 @@ class CloudflareProvider(BaseProvider): 'priority': value.priority, 'weight': value.weight, 'port': value.port, - 'target': value.target[:-1], + 'target': target, } } diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index b0bbaef..860b6c3 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -174,6 +174,56 @@ "auto_added": false } }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_tcp", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_pop3", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, { "id": "fc12ab34cd5611334422ab3322997656", "type": "SRV", diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 735d95c..94a37f4 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -180,7 +180,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) @@ -189,7 +189,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -203,12 +203,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 22 # individual record creates + ] + [None] * 24 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -234,7 +234,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(23, provider._request.call_count) + self.assertEquals(25, provider._request.call_count) provider._request.reset_mock() From 39412924da534adde67e1c9b11d0d93fd8e67db8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 21:23:14 +0800 Subject: [PATCH 24/39] Fix handling of NULL SRV records in DigitalOcean provider --- octodns/provider/digitalocean.py | 6 +++++- tests/fixtures/digitalocean-page-2.json | 22 +++++++++++++++++++ tests/test_octodns_provider_digitalocean.py | 24 ++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index e192543..6ccee1d 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -186,10 +186,14 @@ class DigitalOceanProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for record in records: + target = ( + '{}.'.format(record['data']) + if record['data'] != "." else "." + ) values.append({ 'port': record['port'], 'priority': record['priority'], - 'target': '{}.'.format(record['data']), + 'target': target, 'weight': record['weight'] }) return { diff --git a/tests/fixtures/digitalocean-page-2.json b/tests/fixtures/digitalocean-page-2.json index 50f17f9..1405527 100644 --- a/tests/fixtures/digitalocean-page-2.json +++ b/tests/fixtures/digitalocean-page-2.json @@ -76,6 +76,28 @@ "weight": null, "flags": null, "tag": null + }, { + "id": 11189896, + "type": "SRV", + "name": "_imap._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null + }, { + "id": 11189897, + "type": "SRV", + "name": "_pop3._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null }], "links": { "pages": { diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 0ad8f72..83fb5c3 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -83,14 +83,14 @@ class TestDigitalOceanProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(12, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -190,6 +190,24 @@ class TestDigitalOceanProvider(TestCase): 'flags': 0, 'name': '@', 'tag': 'issue', 'ttl': 3600, 'type': 'CAA'}), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_imap._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_pop3._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), call('POST', '/domains/unit.tests/records', data={ 'name': '_srv._tcp', 'weight': 20, @@ -200,7 +218,7 @@ class TestDigitalOceanProvider(TestCase): 'port': 30 }), ]) - self.assertEquals(24, provider._client._request.call_count) + self.assertEquals(26, provider._client._request.call_count) provider._client._request.reset_mock() From 876a5b46f34693d7161b773babc95d1a38d02103 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 19 Dec 2020 22:32:23 +0800 Subject: [PATCH 25/39] Update PowerDNS tests and fixtures for NULL SRV records --- tests/fixtures/powerdns-full-data.json | 24 ++++++++++++++++++++++++ tests/test_octodns_provider_powerdns.py | 6 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 3d445d4..a08f028 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -59,6 +59,30 @@ "ttl": 300, "type": "A" }, + { + "comments": [], + "name": "_imap._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, + { + "comments": [], + "name": "_pop3._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, { "comments": [], "name": "_srv._tcp.unit.tests.", diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 33b5e44..7c418ff 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -186,7 +186,7 @@ class TestPowerDnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) expected_n = len(expected.records) - 3 - self.assertEquals(16, expected_n) + self.assertEquals(18, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -194,7 +194,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(19, len(expected.records)) + self.assertEquals(21, len(expected.records)) # A small change to a single record with requests_mock() as mock: From 4105fb7ee798c83bf894ef39150335460536e888 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:06:50 +0800 Subject: [PATCH 26/39] Update Gandi tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/gandi-no-changes.json | 18 ++++++++++++++++++ tests/test_octodns_provider_gandi.py | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json index b018785..a67dc93 100644 --- a/tests/fixtures/gandi-no-changes.json +++ b/tests/fixtures/gandi-no-changes.json @@ -123,6 +123,24 @@ "2.2.3.6" ] }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_imap._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_imap._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_pop3._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_pop3._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, { "rrset_type": "SRV", "rrset_ttl": 600, diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7e1c866..1b0443b 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -117,7 +117,7 @@ class TestGandiProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -284,6 +284,22 @@ class TestGandiProvider(TestCase): '12 20 30 foo-2.unit.tests.' ] }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_pop3._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_imap._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), call('POST', '/livedns/domains/unit.tests/records', data={ 'rrset_name': '@', 'rrset_ttl': 3600, @@ -307,7 +323,7 @@ class TestGandiProvider(TestCase): }) ]) # expected number of total calls - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From ae65311a96e77efea6719027f31b0a435ef7b9d8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:11:22 +0800 Subject: [PATCH 27/39] Update Constellix tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/constellix-records.json | 56 +++++++++++++++++++++++ tests/test_octodns_provider_constellix.py | 6 +-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index 689fd53..282ca62 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -64,6 +64,62 @@ "roundRobinFailover": [], "pools": [], "poolsDetail": [] +}, { + "id": 1898527, + "type": "SRV", + "recordType": "srv", + "name": "_imap._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] +}, { + "id": 1898528, + "type": "SRV", + "recordType": "srv", + "name": "_pop3._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] }, { "id": 1808527, "type": "SRV", diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index bc17b50..8ba4860 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -101,14 +101,14 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(14, len(again.records)) + self.assertEquals(16, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -163,7 +163,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From 909c7ad7e88d54f4adeadbf9d3ad3d189de06360 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 8 Jan 2021 16:51:36 +0800 Subject: [PATCH 28/39] Update EasyDNS tests and fixtures for NULL SRV records Thanks to @actazen for assisting with API tests to confirm support --- tests/fixtures/easydns-records.json | 26 ++++++++++++++++++++++++-- tests/test_octodns_provider_easydns.py | 6 +++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/fixtures/easydns-records.json b/tests/fixtures/easydns-records.json index c3718b5..73ea953 100644 --- a/tests/fixtures/easydns-records.json +++ b/tests/fixtures/easydns-records.json @@ -264,10 +264,32 @@ "rdata": "v=DKIM1;k=rsa;s=email;h=sha256;p=A\/kinda+of\/long\/string+with+numb3rs", "geozone_id": "0", "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340025", + "domain": "unit.tests", + "host": "_imap._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340026", + "domain": "unit.tests", + "host": "_pop3._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" } ], - "count": 24, - "total": 24, + "count": 26, + "total": 26, "start": 0, "max": 1000, "status": 200 diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 8df0e22..2b137a6 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -80,14 +80,14 @@ class TestEasyDNSProvider(TestCase): text=fh.read()) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -379,7 +379,7 @@ class TestEasyDNSProvider(TestCase): self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) - self.assertEquals(23, provider._client._request.call_count) + self.assertEquals(25, provider._client._request.call_count) provider._client._request.reset_mock() From fb197b890e78d6a652fa276bb4148afd9174245b Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 11 Jan 2021 22:07:58 +0800 Subject: [PATCH 29/39] Update Mythic Beasts tests and fixtures for NULL SRV records Thanks to @pwaring for assisting with API tests to confirm support, and reaching out to Mythic Beasts to obtain a fix --- tests/fixtures/mythicbeasts-list.txt | 2 ++ tests/test_octodns_provider_mythicbeasts.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/mythicbeasts-list.txt b/tests/fixtures/mythicbeasts-list.txt index ed4ea4c..006a8ff 100644 --- a/tests/fixtures/mythicbeasts-list.txt +++ b/tests/fixtures/mythicbeasts-list.txt @@ -5,6 +5,8 @@ @ 3600 SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 @ 3600 SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 @ 3600 CAA 0 issue ca.unit.tests +_imap._tcp 600 SRV 0 0 0 . +_pop3._tcp 600 SRV 0 0 0 . _srv._tcp 600 SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 SRV 12 20 30 foo-2.unit.tests. aaaa 600 AAAA 2601:644:500:e210:62f8:1dff:feb8:947a diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index f78cb0b..26af8c1 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -378,8 +378,8 @@ class TestMythicBeastsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) - self.assertEquals(15, len(self.expected.records)) + self.assertEquals(17, len(zone.records)) + self.assertEquals(17, len(self.expected.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -445,7 +445,7 @@ class TestMythicBeastsProvider(TestCase): if isinstance(c, Update)])) self.assertEquals(1, len([c for c in plan.changes if isinstance(c, Delete)])) - self.assertEquals(14, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) - self.assertEquals(16, provider.apply(plan)) + self.assertEquals(18, provider.apply(plan)) self.assertTrue(plan.exists) From e0d79f826f752dc792ad03836e27daebe2c64691 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 15 Jan 2021 18:32:11 +0800 Subject: [PATCH 30/39] Update Edge DNS tests and fixtures for NULL SRV records Thanks to @jdgri for assisting with API tests to confirm support --- tests/fixtures/edgedns-records.json | 20 ++++++++++++++++++-- tests/test_octodns_provider_edgedns.py | 10 +++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/fixtures/edgedns-records.json b/tests/fixtures/edgedns-records.json index 4693eb1..a5ce14f 100644 --- a/tests/fixtures/edgedns-records.json +++ b/tests/fixtures/edgedns-records.json @@ -9,6 +9,22 @@ "name": "_srv._tcp.unit.tests", "ttl": 600 }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "ttl": 600 + }, { "rdata": [ "2601:644:500:e210:62f8:1dff:feb8:947a" @@ -151,7 +167,7 @@ } ], "metadata": { - "totalElements": 16, + "totalElements": 18, "showAll": true } -} \ No newline at end of file +} diff --git a/tests/test_octodns_provider_edgedns.py b/tests/test_octodns_provider_edgedns.py index 20a9a07..694c762 100644 --- a/tests/test_octodns_provider_edgedns.py +++ b/tests/test_octodns_provider_edgedns.py @@ -77,14 +77,14 @@ class TestEdgeDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(16, len(again.records)) + self.assertEquals(18, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -105,7 +105,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(29, changes) + self.assertEquals(31, changes) # Test against a zone that doesn't exist yet with requests_mock() as mock: @@ -118,7 +118,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist yet, but gid not provided with requests_mock() as mock: @@ -132,7 +132,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist, but cid not provided From 2cd5511dc68ccd0fad97dbc665e9cc20f20813d0 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 00:08:24 +0800 Subject: [PATCH 31/39] Warn that NULL SRV records are unsupported in DNSimple provider DNSimple does not handle NULL SRV records correctly (either via their web interface or API). Flag to end user if attempted. Issue noted with DNSimple support 2020-12-09 --- octodns/provider/dnsimple.py | 41 +++++++++++++++++++++++-- tests/test_octodns_provider_dnsimple.py | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index f83098e..647b89c 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -218,12 +218,23 @@ class DnsimpleProvider(BaseProvider): try: weight, port, target = record['content'].split(' ', 2) except ValueError: - # see _data_for_NAPTR's continue + # their api/website will let you create invalid records, this + # essentially handles that by ignoring them for values + # purposes. That will cause updates to happen to delete them if + # they shouldn't exist or update them if they're wrong + self.log.warning( + '_data_for_SRV: unsupported %s record (%s)', + _type, + record['content'] + ) continue + + target = '{}.'.format(target) if target != "." else "." + values.append({ 'port': port, 'priority': record['priority'], - 'target': '{}.'.format(target), + 'target': target, 'weight': weight }) return { @@ -269,7 +280,12 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] + data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: + self.log.warning( + 'populate: skipping unsupported %s record', + _type + ) continue elif _type == 'TXT' and record['content'].startswith('ALIAS for'): # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', @@ -290,6 +306,27 @@ class DnsimpleProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNSimple does not support empty/NULL SRV records + # + # Fails silently and leaves a corrupt record + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 92f32b1..9f1dab3 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -137,7 +137,7 @@ class TestDnsimpleProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + n = len(self.expected.records) - 6 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) From 84a0c089d4c7c28c20fa382d15113bac87f4caa6 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 25 Dec 2020 19:21:41 +0800 Subject: [PATCH 32/39] Warn that NULL SRV records are unsupported in DNS Made Easy provider --- octodns/provider/dnsmadeeasy.py | 24 ++++++++++++++++++++++ tests/test_octodns_provider_dnsmadeeasy.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 0bf05a0..7880280 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -284,6 +284,30 @@ class DnsMadeEasyProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNS Made Easy does not support empty/NULL SRV records + # + # Attempting to sync such a record would generate the following error + # + # octodns.provider.dnsmadeeasy.DnsMadeEasyClientBadRequest: + # - Record value may not be a standalone dot. + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 0ad059d..92aa547 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) From 5d23977bbd26d4b12a96f7187ffc27fda8057bdd Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 15:44:32 +0800 Subject: [PATCH 33/39] Adjust remaining unit tests due to extra records in test zone --- tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_transip.py | 4 ++-- tests/test_octodns_provider_ultra.py | 8 ++++---- tests/test_octodns_provider_yaml.py | 10 ++++++---- tests/test_octodns_source_axfr.py | 8 ++++---- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index f757466..1657f04 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(16, tc) + self.assertEquals(18, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(26, tc) + self.assertEquals(28, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -215,13 +215,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(16, len(changes)) + self.assertEquals(18, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(17, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index f792085..84cfebc 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -222,7 +222,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(14, plan.change_counts['Create']) self.assertEqual(0, plan.change_counts['Update']) self.assertEqual(0, plan.change_counts['Delete']) @@ -235,7 +235,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, len(plan.changes)) + self.assertEqual(14, len(plan.changes)) changes = provider.apply(plan) self.assertEqual(changes, len(plan.changes)) diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index 43eac3c..b6d1017 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -285,12 +285,12 @@ class TestUltraProvider(TestCase): provider._request.side_effect = [ UltraNoZonesExistException('No Zones'), None, # zone create - ] + [None] * 13 # individual record creates + ] + [None] * 15 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -320,7 +320,7 @@ class TestUltraProvider(TestCase): 'p=A/kinda+of/long/string+with+numb3rs']}), ], True) # expected number of total calls - self.assertEquals(15, provider._request.call_count) + self.assertEquals(17, provider._request.call_count) # Create sample rrset payload to attempt to alter page1 = json_load(open('tests/fixtures/ultra-records-page-1.json')) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f255238..d527fb3 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(21, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(18, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -107,6 +107,8 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) # these are stored as singular 'value' + self.assertTrue('value' in data.pop('_imap._tcp')) + self.assertTrue('value' in data.pop('_pop3._tcp')) self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) self.assertTrue('value' in data.pop('dname')) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 44e04d0..f1a8109 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -36,7 +36,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(12, len(got.records)) + self.assertEquals(14, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -73,18 +73,18 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del self.source._zone_records[valid.name] From cec53b2180fc02e9c47c3b2d11b62e00da9a6650 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:11:14 -0800 Subject: [PATCH 34/39] Require different twines based on python version --- requirements-dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 146d673..85a93f5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,5 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==3.2.0 +twine==1.15.0; python_version < '3.2' +twine==3.2.0; python_version >= '3.2' From f64ee57b14ba03363c69405ccdb08aca5bb880d0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:11:46 -0800 Subject: [PATCH 35/39] Actually only install twine on 3.x --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 85a93f5..522f112 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,5 +5,4 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==1.15.0; python_version < '3.2' twine==3.2.0; python_version >= '3.2' From e516e7647b756c9c6d8d71a23ff5127e4d634d20 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:39:52 -0800 Subject: [PATCH 36/39] Remove 2.7 form the version matix --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 337ebb0..b2f48dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,8 +10,7 @@ jobs: strategy: matrix: # Tested versions based on dates in https://devguide.python.org/devcycle/#end-of-life-branches, - # with the addition of 2.7 b/c it's still if pretty wide active use. - python-version: [2.7, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Setup python From 55af09d73cf512b00a03c58786a4f16971e8e915 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 18 Feb 2021 07:11:18 -0800 Subject: [PATCH 37/39] use super for supports base case, remove unused `data_for`. --- octodns/provider/dnsimple.py | 3 +-- octodns/provider/dnsmadeeasy.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 647b89c..599eacb 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -280,7 +280,6 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] - data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: self.log.warning( 'populate: skipping unsupported %s record', @@ -325,7 +324,7 @@ class DnsimpleProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsimpleProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 7880280..b222b5c 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -306,7 +306,7 @@ class DnsMadeEasyProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsMadeEasyProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: From bec4d025dcd1297ea71335339a209b3dc1962443 Mon Sep 17 00:00:00 2001 From: Brian Surowiec Date: Wed, 10 Mar 2021 15:57:50 -0500 Subject: [PATCH 38/39] Update branch name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f3ef4c..9d267d7 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - **GitHub Action:** [OctoDNS-Sync](https://github.com/marketplace/actions/octodns-sync) - **Sample Implementations.** See how others are using it - [`hackclub/dns`](https://github.com/hackclub/dns) - - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) + - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/main/dns) - [`g0v-network/domains`](https://github.com/g0v-network/domains) - [`jekyll/dns`](https://github.com/jekyll/dns) - **Custom Sources & Providers.**