diff --git a/peeringdb_server/ixf.py b/peeringdb_server/ixf.py index 0ceb4dc3..e7f87f36 100644 --- a/peeringdb_server/ixf.py +++ b/peeringdb_server/ixf.py @@ -219,6 +219,65 @@ class Importer: return data + def sanitize_vlans(self, vlans): + """ + Sanitize vlan lists where ip 4 and 6 addresses + for the same vlan (determined by vlan id) exist + in separate entries by combining those + list entries to one + """ + + _vlans = {} + sanitized = [] + + for vlan in vlans: + + # if the vlan_id is not specified we want + # to default to 0 so we can still group based + # on that + + id = vlan.get("vlan_id", 0) + + # neither ipv4 nor ipv6 is specified, there is + # nothing to sanitize here, so skip + + if "ipv4" not in vlan and "ipv6" not in vlan: + continue + + if id not in _vlans: + + # first occurance of vlan id gets appended + # as is + + _vlans[id] = [vlan] + else: + + # additional occurances of vlan id get checked + # on whether or not they will fill in a missing + # ipv4 or ipv6 address, and if so will update + # the existing vlan entry. + # + # otherwise append as a new entry for that vlan id + + current = _vlans[id][-1] + + update = None + + if "ipv4" in vlan and "ipv4" not in current: + update = "ipv4" + elif "ipv6" in vlan and "ipv6" not in current: + update = "ipv6" + + if update: + current[update] = vlan[update] + else: + _vlans[id].append(vlan) + + for vlan_id, entries in _vlans.items(): + sanitized.extend(entries) + + return sanitized + def sanitize(self, data): """ Takes ixf data dict and runs some sanitization on it @@ -238,16 +297,13 @@ class Importer: for member in member_list: asn = member.get("asnum") for conn in member.get("connection_list", []): - vlans = conn.get("vlan_list", []) + + conn["vlan_list"] = self.sanitize_vlans(conn.get("vlan_list", [])) + vlans = conn["vlan_list"] + if not vlans: continue vlan_list_found = True - if len(vlans) == 2: - # if vlans[0].get("vlan_id") == vlans[1].get("vlan_id"): - keys = list(vlans[0].keys()) + list(vlans[1].keys()) - if keys.count("ipv4") == 1 and keys.count("ipv6") == 1: - vlans[0].update(**vlans[1]) - conn["vlan_list"] = [vlans[0]] # de-dupe reoccurring ipv4 / ipv6 addresses @@ -277,6 +333,9 @@ class Importer: data["pdb_error"] = invalid + # set member_list to the sanitized copy + data["member_list"] = member_list + return data def update(self, ixlan, save=True, data=None, timeout=5, asn=None): diff --git a/tests/data/ixf/vlan/test0.expected b/tests/data/ixf/vlan/test0.expected new file mode 100644 index 00000000..22dc14f0 --- /dev/null +++ b/tests/data/ixf/vlan/test0.expected @@ -0,0 +1,13 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test0.input b/tests/data/ixf/vlan/test0.input new file mode 100644 index 00000000..22dc14f0 --- /dev/null +++ b/tests/data/ixf/vlan/test0.input @@ -0,0 +1,13 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test1.expected b/tests/data/ixf/vlan/test1.expected new file mode 100644 index 00000000..22dc14f0 --- /dev/null +++ b/tests/data/ixf/vlan/test1.expected @@ -0,0 +1,13 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test1.input b/tests/data/ixf/vlan/test1.input new file mode 100644 index 00000000..cd9b531c --- /dev/null +++ b/tests/data/ixf/vlan/test1.input @@ -0,0 +1,16 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + } + }, + { + "vlan_id":2, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test2.expected b/tests/data/ixf/vlan/test2.expected new file mode 100644 index 00000000..9f81a7a3 --- /dev/null +++ b/tests/data/ixf/vlan/test2.expected @@ -0,0 +1,12 @@ +{ + "vlan_list": [ + { + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test2.input b/tests/data/ixf/vlan/test2.input new file mode 100644 index 00000000..073305bf --- /dev/null +++ b/tests/data/ixf/vlan/test2.input @@ -0,0 +1,14 @@ +{ + "vlan_list": [ + { + "ipv4":{ + "address":"206.81.81.12" + } + }, + { + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test3.expected b/tests/data/ixf/vlan/test3.expected new file mode 100644 index 00000000..53727abb --- /dev/null +++ b/tests/data/ixf/vlan/test3.expected @@ -0,0 +1,22 @@ +{ + "vlan_list": [ + { + "vlan_id":1, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + }, + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.82.12" + }, + "ipv6":{ + "address":"2001:504:17::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test3.input b/tests/data/ixf/vlan/test3.input new file mode 100644 index 00000000..53727abb --- /dev/null +++ b/tests/data/ixf/vlan/test3.input @@ -0,0 +1,22 @@ +{ + "vlan_list": [ + { + "vlan_id":1, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + }, + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.82.12" + }, + "ipv6":{ + "address":"2001:504:17::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test4.expected b/tests/data/ixf/vlan/test4.expected new file mode 100644 index 00000000..fc07cbda --- /dev/null +++ b/tests/data/ixf/vlan/test4.expected @@ -0,0 +1,16 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + } + }, + { + "vlan_id":3, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test4.input b/tests/data/ixf/vlan/test4.input new file mode 100644 index 00000000..fc07cbda --- /dev/null +++ b/tests/data/ixf/vlan/test4.input @@ -0,0 +1,16 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + } + }, + { + "vlan_id":3, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test5.expected b/tests/data/ixf/vlan/test5.expected new file mode 100644 index 00000000..7a3f2f19 --- /dev/null +++ b/tests/data/ixf/vlan/test5.expected @@ -0,0 +1,20 @@ +{ + "vlan_list": [ + { + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + }, + { + "ipv4":{ + "address":"206.81.81.13" + }, + "ipv6":{ + "address":"2001:504:16::1235" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test5.input b/tests/data/ixf/vlan/test5.input new file mode 100644 index 00000000..17563c03 --- /dev/null +++ b/tests/data/ixf/vlan/test5.input @@ -0,0 +1,22 @@ +{ + "vlan_list": [ + { + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + }, + { + "ipv4":{ + "address":"206.81.81.13" + } + }, + { + "ipv6":{ + "address":"2001:504:16::1235" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test6.expected b/tests/data/ixf/vlan/test6.expected new file mode 100644 index 00000000..43d4d155 --- /dev/null +++ b/tests/data/ixf/vlan/test6.expected @@ -0,0 +1,21 @@ +{ + "vlan_list": [ + { + "ipv4":{ + "address":"206.81.81.11" + }, + "ipv6":{ + "address":"2001:504:16::1233" + } + }, + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test6.input b/tests/data/ixf/vlan/test6.input new file mode 100644 index 00000000..9cdcac18 --- /dev/null +++ b/tests/data/ixf/vlan/test6.input @@ -0,0 +1,23 @@ +{ + "vlan_list": [ + { + "ipv4":{ + "address":"206.81.81.11" + } + }, + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + }, + { + "ipv6":{ + "address":"2001:504:16::1233" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test7.expected b/tests/data/ixf/vlan/test7.expected new file mode 100644 index 00000000..22dc14f0 --- /dev/null +++ b/tests/data/ixf/vlan/test7.expected @@ -0,0 +1,13 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test7.input b/tests/data/ixf/vlan/test7.input new file mode 100644 index 00000000..22dc14f0 --- /dev/null +++ b/tests/data/ixf/vlan/test7.input @@ -0,0 +1,13 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test8.expected b/tests/data/ixf/vlan/test8.expected new file mode 100644 index 00000000..22dc14f0 --- /dev/null +++ b/tests/data/ixf/vlan/test8.expected @@ -0,0 +1,13 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/data/ixf/vlan/test8.input b/tests/data/ixf/vlan/test8.input new file mode 100644 index 00000000..22dc14f0 --- /dev/null +++ b/tests/data/ixf/vlan/test8.input @@ -0,0 +1,13 @@ +{ + "vlan_list": [ + { + "vlan_id":2, + "ipv4":{ + "address":"206.81.81.12" + }, + "ipv6":{ + "address":"2001:504:16::1234" + } + } + ] +} diff --git a/tests/test_ixf_member_import_protocol.py b/tests/test_ixf_member_import_protocol.py index e77ade28..1ecc4fd1 100644 --- a/tests/test_ixf_member_import_protocol.py +++ b/tests/test_ixf_member_import_protocol.py @@ -2191,6 +2191,16 @@ def test_resolve_deskpro_ticket(entities): assert ticket.deskpro_ref in email.subject +def test_vlan_sanitize(data_ixf_vlan): + """ + test that various vlan_list setups are sanitized correctly + """ + importer = ixf.Importer() + sanitized = importer.sanitize_vlans(json.loads(data_ixf_vlan.input)["vlan_list"]) + assert sanitized == data_ixf_vlan.expected["vlan_list"] + + + # FIXTURES @pytest.fixture(params=[True, False]) def save(request):