Merge pull request #903 from octodns/idna

Add idna encode/decode helpers

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.9.18 - 2022-??-?? - Internationalization
+
+* Added octodns.idna idna_encode/idna_decode helpers
+
 ## v0.9.17 - 2022-04-02 - Registration required
 
 #### Noteworthy changes

octodns/idna.py

@@ -0,0 +1,32 @@
+#
+#
+#
+
+from idna import decode as _decode, encode as _encode
+
+
+def idna_encode(name):
+    # Based on https://github.com/psf/requests/pull/3695/files
+    # #diff-0debbb2447ce5debf2872cb0e17b18babe3566e9d9900739e8581b355bd513f7R39
+    try:
+        name.encode('ascii')
+        # No utf8 chars, just use as-is
+        return name
+    except UnicodeEncodeError:
+        if name.startswith('*'):
+            # idna.encode doesn't like the *
+            name = _encode(name[2:]).decode('utf-8')
+            return f'*.{name}'
+        return _encode(name).decode('utf-8')
+
+
+def idna_decode(name):
+    pieces = name.lower().split('.')
+    if any(p.startswith('xn--') for p in pieces):
+        # it's idna
+        if name.startswith('*'):
+            # idna.decode doesn't like the *
+            return f'*.{_decode(name[2:])}'
+        return _decode(name)
+    # not idna, just return as-is
+    return name

octodns/record/__init__.py

@@ -104,7 +104,7 @@ class Record(EqualityTupleMixin):
 
     @classmethod
     def new(cls, zone, name, data, source=None, lenient=False):
-        name = str(name)
+        name = str(name).lower()
         fqdn = f'{name}.{zone.name}' if name else zone.name
         try:
             _type = data['type']

requirements-dev.txt

@@ -1,47 +1,46 @@
-Pygments==2.11.2
-attrs==21.4.0
-black==22.3.0
-bleach==4.1.0
-build==0.7.0
-certifi==2021.10.8
-cffi==1.15.0
-charset-normalizer==2.0.12
-click>=8.0.0
+Pygments==2.12.0
+attrs==22.1.0
+black==22.6.0
+bleach==5.0.1
+build==0.8.0
+certifi==2022.6.15
+cffi==1.15.1
+charset-normalizer==2.1.0
+click==8.1.3
 cmarkgfm==0.8.0
-colorama==0.4.4
-coverage==6.3.2
-docutils==0.18.1
-idna==3.3
-importlib-metadata==4.11.2
+commonmark==0.9.1
+coverage==6.4.3
+docutils==0.19
+importlib-metadata==4.12.0
 iniconfig==1.1.1
-keyring==23.5.0
-mypy-extensions>=0.4.3
+keyring==23.8.2
+mypy-extensions==0.4.3
 packaging==21.3
-pathspec>=0.9.0
-pep517==0.12.0
-pkginfo==1.8.2
-platformdirs>=2
+pathspec==0.9.0
+pep517==0.13.0
+pkginfo==1.8.3
+platformdirs==2.5.2
 pluggy==1.0.0
 pprintpp==0.4.0
 py==1.11.0
 pycountry-convert==0.7.2
 pycountry==22.3.5
 pycparser==2.21
-pyflakes==2.4.0
-pyparsing==3.0.7
+pyflakes==2.5.0
+pyparsing==3.0.9
 pytest-cov==3.0.0
-pytest-mock==3.7.0
+pytest-mock==3.8.2
 pytest-network==0.0.1
-pytest==7.0.1
-readme-renderer==33.0
+pytest==7.1.2
+readme-renderer==36.0
 repoze.lru==0.7
 requests-toolbelt==0.9.1
-requests==2.27.1
+requests==2.28.1
 rfc3986==2.0.0
+rich==12.5.1
 tomli==2.0.1
-tqdm==4.63.0
-twine==3.8.0
-typing-extensions>=3.10.0.0
-urllib3==1.26.8
+twine==4.0.1
+typing_extensions==4.3.0
+urllib3==1.26.11
 webencodings==0.5.1
-zipp==3.7.0
+zipp==3.8.1

requirements.txt

@@ -1,6 +1,7 @@
 PyYAML==6.0
 dnspython==2.2.1
 fqdn==1.5.1
+idna==3.3
 natsort==8.1.0
 python-dateutil==2.8.2
 six==1.16.0

setup.py

@@ -84,6 +84,7 @@ setup(
         'PyYaml>=4.2b1',
         'dnspython>=1.15.0',
         'fqdn>=1.5.0',
+        'idna>=3.3',
         'natsort>=5.5.0',
         'python-dateutil>=2.8.1',
     ),

tests/test_octodns_idna.py

@@ -0,0 +1,60 @@
+#
+#
+#
+
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
+
+from unittest import TestCase
+
+from octodns.idna import idna_decode, idna_encode
+
+
+class TestIdna(TestCase):
+    def assertIdna(self, value, expected):
+        got = idna_encode(value)
+        self.assertEqual(expected, got)
+        # round tripped
+        self.assertEqual(value, idna_decode(got))
+
+    def test_noops(self):
+        # empty
+        self.assertIdna('', '')
+
+        # noop
+        self.assertIdna('unit.tests.', 'unit.tests.')
+
+        # wildcard noop
+        self.assertIdna('*.unit.tests.', '*.unit.tests.')
+
+    def test_unicode(self):
+        # encoded
+        self.assertIdna('zajęzyk.pl.', 'xn--zajzyk-y4a.pl.')
+
+        # encoded with wildcard
+        self.assertIdna('*.zajęzyk.pl.', '*.xn--zajzyk-y4a.pl.')
+
+        # encoded with simple name
+        self.assertIdna('noop.zajęzyk.pl.', 'noop.xn--zajzyk-y4a.pl.')
+
+        # encoded with encoded name
+        self.assertIdna(
+            'zajęzyk.zajęzyk.pl.', 'xn--zajzyk-y4a.xn--zajzyk-y4a.pl.'
+        )
+
+        self.assertIdna('déjàvu.com.', 'xn--djvu-1na6c.com.')
+        self.assertIdna('déjà-vu.com.', 'xn--dj-vu-sqa5d.com.')
+
+    def test_underscores(self):
+        # underscores aren't valid in idna names, so these are all ascii
+
+        self.assertIdna('foo_bar.pl.', 'foo_bar.pl.')
+        self.assertIdna('bleep_bloop.foo_bar.pl.', 'bleep_bloop.foo_bar.pl.')
+
+    def test_case_insensitivity(self):
+        # Shouldn't be hit by octoDNS use cases, but checked anyway
+        self.assertEqual('zajęzyk.pl.', idna_decode('XN--ZAJZYK-Y4A.PL.'))