From e42bf20263eb0e9b5030f77a0d4f180a3cf2ca6a Mon Sep 17 00:00:00 2001 From: Stefan Pratter Date: Fri, 15 Feb 2019 17:46:40 +0000 Subject: [PATCH] IXF preview tool (#408) --- config/facsimile/peeringdb.yaml | 3 + peeringdb_server/import_views.py | 45 ++ peeringdb_server/ixf.py | 415 ++++++++++++++++++ .../commands/pdb_ixf_ixp_member_import.py | 13 +- peeringdb_server/models.py | 244 +--------- peeringdb_server/static/peeringdb.js | 111 +++++ peeringdb_server/static/site.css | 67 +++ .../templates/site/view_exchange_bottom.html | 69 +++ peeringdb_server/urls.py | 6 + .../data/ixf/logs/skip_disabled_networks.json | 64 +++ tests/data/ixf/logs/skip_prefix_mismatch.json | 51 +++ tests/data/ixf/logs/update_01.json | 64 +++ tests/data/json_members_list/members.0.4.json | 18 + tests/data/json_members_list/members.0.5.json | 19 + tests/data/json_members_list/members.0.6.json | 12 + tests/django_init.py | 3 +- tests/test_ixf_member_import.py | 104 +++-- 17 files changed, 1045 insertions(+), 263 deletions(-) create mode 100644 peeringdb_server/import_views.py create mode 100644 peeringdb_server/ixf.py create mode 100644 tests/data/ixf/logs/skip_disabled_networks.json create mode 100644 tests/data/ixf/logs/skip_prefix_mismatch.json create mode 100644 tests/data/ixf/logs/update_01.json diff --git a/config/facsimile/peeringdb.yaml b/config/facsimile/peeringdb.yaml index 25392719..598fe703 100644 --- a/config/facsimile/peeringdb.yaml +++ b/config/facsimile/peeringdb.yaml @@ -22,6 +22,7 @@ misc: view_verify_POST: "2/m" view_username_retrieve_initiate: "2/m" request_translation: "2/m" + view_import_ixlan_ixf_preview: "1/m" api: throtteling: enabled: true @@ -100,6 +101,8 @@ install: - $SRC_DIR$/peeringdb_server/db_router.py - $SRC_DIR$/peeringdb_server/mock.py - $SRC_DIR$/peeringdb_server/maintenance.py + - $SRC_DIR$/peeringdb_server/import_views.py + - $SRC_DIR$/peeringdb_server/ixf.py - $SRC_DIR$/peeringdb_server/management/__init__.py - $SRC_DIR$/peeringdb_server/management/commands/__init__.py - $SRC_DIR$/peeringdb_server/management/commands/pdb_deskpro_publish.py diff --git a/peeringdb_server/import_views.py b/peeringdb_server/import_views.py new file mode 100644 index 00000000..1ffb76a3 --- /dev/null +++ b/peeringdb_server/import_views.py @@ -0,0 +1,45 @@ +import json + +from django.http import JsonResponse, HttpResponse +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ + +from django_namespace_perms.util import has_perms +from ratelimit.decorators import ratelimit, is_ratelimited + +from peeringdb_server import ixf +from peeringdb_server.models import IXLan + +RATELIMITS = settings.RATELIMITS + + +@ratelimit(key="ip", rate=RATELIMITS["view_import_ixlan_ixf_preview"]) +def view_import_ixlan_ixf_preview(request, ixlan_id): + + # check if request was blocked by rate limiting + was_limited = getattr(request, "limited", False) + if was_limited: + return JsonResponse({ + "non_field_errors": [ + _("Please wait a bit before requesting " \ + "another ixf import preview.") + ] + }, status=400) + + try: + ixlan = IXLan.objects.get(id=ixlan_id) + except IXLan.DoesNotExist: + return JsonResponse({ + "non_field_errors": [_("Ixlan not found")] + }, status=404) + + if not has_perms(request.user, ixlan, "update"): + return JsonResponse({ + "non_field_errors": [_("Permission denied")] + }, status=403) + + importer = ixf.Importer() + importer.update(ixlan, save=False) + + return HttpResponse( + json.dumps(importer.log, indent=2), content_type="application/json") diff --git a/peeringdb_server/ixf.py b/peeringdb_server/ixf.py new file mode 100644 index 00000000..45df541b --- /dev/null +++ b/peeringdb_server/ixf.py @@ -0,0 +1,415 @@ +import json + +import requests +import ipaddress + +from django.db import transaction +from django.utils.translation import ugettext_lazy as _ + +import reversion + +from peeringdb_server.models import ( + IXLanIXFMemberImportAttempt, + IXLanIXFMemberImportLog, + Network, + NetworkIXLan, +) + + +class Importer(object): + + allowed_member_types = ["peering", "ixp", "routeserver", "probono"] + allowed_states = ["active", "connected"] + + def __init__(self): + self.reset() + + def reset(self, ixlan=None, save=False): + self.reset_log() + self.netixlans = [] + self.netixlans_deleted = [] + self.ipaddresses = [] + self.ixlan = ixlan + self.save = save + + def fetch(self, url, timeout=5): + """ + Retrieves ixf member export data from the url + + Will do a quick sanity check on the data + + Returns dict containing the parsed data. + + Arguments: + - url + + Keyword arguments: + - timeout : max time to spend on request + """ + + if not url: + return {"pdb_error": _("IXF import url not specified")} + + try: + result = requests.get(url, timeout=timeout) + except Exception as exc: + return {"pdb_error": exc} + + if result.status_code != 200: + # FIXME: log error somewhere + return None + + try: + data = result.json() + except Exception as inst: + data = {"pdb_error": _("No JSON could be parsed")} + return data + + return self.sanitize(data) + + def sanitize(self, data): + """ + Takes ixf data dict and runs some sanitization on it + """ + + invalid = None + vlan_list_found = False + + # This fixes instances where ixps provide two separate entries for + # vlans in vlan_list for ipv4 and ipv6 (AMS-IX for example) + for member in data.get("member_list", []): + asn = member.get("asnum") + for conn in member.get("connection_list", []): + vlans = conn.get("vlan_list", []) + if not vlans: + continue + vlan_list_found = True + if len(vlans) == 2: + # if vlans[0].get("vlan_id") == vlans[1].get("vlan_id"): + keys = vlans[0].keys() + vlans[1].keys() + if keys.count("ipv4") == 1 and keys.count("ipv6") == 1: + vlans[0].update(**vlans[1]) + conn["vlan_list"] = [vlans[0]] + + if not vlan_list_found: + invalid = _("No entries in any of the vlan_list lists, aborting.") + + data["pdb_error"] = invalid + + return data + + def update(self, ixlan, save=True, data=None, timeout=5): + """ + Sync netixlans under this ixlan from ixf member export json data (specs + can be found at https://github.com/euro-ix/json-schemas) + + Arguments: + - ixlan (IXLan): ixlan object to update from ixf + + Keyword Arguments: + - save (bool): commit changes to db + + Returns: + - Tuple(success, netixlans, log) + """ + + self.reset(ixlan=ixlan, save=save) + + if data is None: + data = self.fetch(ixlan.ixf_ixp_member_list_url, timeout=timeout) + + # bail if there has been any errors during sanitize() or fetch() + if data.get("pdb_error"): + self.log_error(data.get("pdb_error"), save=save) + return (False, [], [], self.log) + + # bail if there are no active prefixes on the ixlan + if ixlan.ixpfx_set_active.count() == 0: + self.log_error(_("No prefixes defined on ixlan"), save=save) + return (False, [], [], self.log) + + try: + # parse the ixf data + self.parse(data) + except KeyError as exc: + # any key erros mean that the data is invalid, log the error and + # bail (transactions are atomic and will be rolled back) + self.log_error("Internal Error 'KeyError': {}".format(exc), + save=save) + return (False, self.netixlans, [], self.log) + + # process any netixlans that need to be deleted + self.process_deletions() + + # archive the import so we can roll it back later if needed + self.archive() + + if save: + self.save_log() + + return (True, self.netixlans, self.netixlans_deleted, self.log) + + @reversion.create_revision() + def process_deletions(self): + """ + Cycles all netixlans on the ixlan targeted by the importer and + will remove any that are no longer found in the ixf data by + their ip addresses + + In order for a netixlan to be removed both it's ipv4 and ipv6 address + need to be gone from the ixf data + """ + for netixlan in self.ixlan.netixlan_set_active: + ipv4 = "{}-{}".format(netixlan.asn, netixlan.ipaddr4) + ipv6 = "{}-{}".format(netixlan.asn, netixlan.ipaddr6) + if ipv4 not in self.ipaddresses and ipv6 not in self.ipaddresses: + self.log_peer(netixlan.asn, "delete", + _("Ip addresses no longer in data"), netixlan) + self.netixlans_deleted.append(netixlan) + if self.save: + netixlan.delete() + + @transaction.atomic() + def archive(self): + """ + Create the IXLanIXFMemberImportLog for this import + """ + + if self.save and (self.netixlans or self.netixlans_deleted): + persist_log = IXLanIXFMemberImportLog.objects.create( + ixlan=self.ixlan) + for netixlan in self.netixlans + self.netixlans_deleted: + versions = reversion.models.Version.objects.get_for_object( + netixlan) + if len(versions) == 1: + version_before = None + else: + version_before = versions[1] + version_after = versions[0] + persist_log.entries.create(netixlan=netixlan, + version_before=version_before, + version_after=version_after) + + def parse(self, data): + """ + Parse ixf data + + Arguments: + - data : result from fetch() + """ + with transaction.atomic(): + self.parse_members(data.get("member_list", [])) + + def parse_members(self, member_list): + """ + Parse the `member_list` section of the ixf schema + + Arguments: + - member_list + """ + for member in member_list: + # we only process members of certain types + member_type = member.get("member_type", "peering").lower() + if member_type in self.allowed_member_types: + # check that the as exists in pdb + asn = member["asnum"] + + if Network.objects.filter(asn=asn).exists(): + network = Network.objects.get(asn=asn) + if network.status != "ok": + self.log_peer( + asn, "skip", + _("Network status is '{}'").format(network.status)) + continue + + self.parse_connections( + member.get("connection_list", []), network, member) + else: + self.log_peer(asn, "skip", + _("Network does not exist in peeringdb")) + else: + self.log_peer(asn, "skip", + _("Invalid member type: {}").format(member_type)) + + def parse_connections(self, connection_list, network, member): + """ + Parse the 'connection_list' section of the ixf schema + + Arguments: + - connection_list + - network : pdb network instance + - member : row from ixf member_list + """ + + asn = member["asnum"] + for connection in connection_list: + state = connection.get("state", "active").lower() + if state in self.allowed_states: + + speed = self.parse_speed(connection.get("if_list", [])) + + self.parse_vlans( + connection.get("vlan_list", []), network, member, + connection, speed) + else: + self.log_peer(asn, "skip", + _("Invalid connection state: {}").format(state)) + + def parse_vlans(self, vlan_list, network, member, connection, speed): + """ + Parse the 'vlan_list' section of the ixf_schema + + Arguments: + - vlan_list + - network : pdb network instance + - member : row from ixf member_list + - connection : row from ixf connection_list + - speed : interface speed + """ + + asn = member["asnum"] + for lan in vlan_list: + ipv4_valid = False + ipv6_valid = False + + ipv4 = lan.get("ipv4", {}) + ipv6 = lan.get("ipv6", {}) + + # vlan entry has no ipaddresses set, log and skip + if not ipv4 and not ipv6: + self.log_error(_("Could not find ipv4 or 6 address in " \ + "vlan_list entry for vlan_id {} (AS{})").format( + lan.get("vlan_id"), asn)) + continue + + ipv4_addr = ipv4.get("address") + ipv6_addr = ipv6.get("address") + + # parse and validate the ipaddresses attached to the vlan + # append the ipaddresses to self.ipaddresses so we can + # later check them to see which netixlans need to be + # dropped during `process_deletions` + try: + if ipv4_addr: + self.ipaddresses.append("{}-{}".format( + asn, ipaddress.ip_address(unicode(ipv4_addr)))) + if ipv6_addr: + self.ipaddresses.append("{}-{}".format( + asn, ipaddress.ip_address(unicode(ipv6_addr)))) + except (ipaddress.AddressValueError, ValueError) as exc: + self.log_error( + _("Ip address error '{}' in vlan_list entry for vlan_id {}" + ).format(exc, lan.get("vlan_id"))) + continue + + netixlan_info = NetworkIXLan( + ixlan=self.ixlan, + network=network, + ipaddr4=ipv4_addr, + ipaddr6=ipv6_addr, + speed=speed, + asn=asn, + is_rs_peer=(ipv4.get("routeserver", False) or \ + ipv6.get("routeserver", False)) + ) + + # after this point we either add or modify the netixlan, so + # now is a good time to check if the related network allows + # such updates, bail if not + if not network.allow_ixp_update: + self.log_peer(asn, "skip", + _("Network has disabled ixp updates"), + netixlan_info) + continue + + # add / modify the netixlan + result = self.ixlan.add_netixlan(netixlan_info, save=self.save, + save_others=self.save) + + if result["netixlan"] and result["changed"]: + self.netixlans.append(result["netixlan"]) + if result["created"]: + action = "add" + else: + action = "modify" + + self.log_peer(asn, action, "", result["netixlan"]) + elif result["netixlan"]: + self.log_peer(asn, "noop", "", result["netixlan"]) + elif result["log"]: + self.log_peer(asn, "skip", "\n".join(result["log"]), + netixlan_info) + + def parse_speed(self, if_list): + """ + Parse speed from the 'if_list' section in the ixf data + + Arguments: + - if_list + + Returns: + - speed + """ + speed = 0 + for iface in if_list: + try: + speed += int(iface.get("if_speed", 0)) + except ValueError: + self.log_error( + _("Invalid speed value: {}").format(iface.get("if_speed"))) + return speed + + def save_log(self): + """ + Save the attempt log + """ + IXLanIXFMemberImportAttempt.objects.update_or_create( + ixlan=self.ixlan, + defaults={"info": "\n".join(json.dumps(self.log))}) + + def reset_log(self): + """ + Reset the attempt log + """ + self.log = {"data": [], "errors": []} + + def log_peer(self, asn, action, reason, netixlan=None): + """ + log peer action in attempt log + + Arguments: + - asn + - action : add | modify | delete | noop | skip + - reason + + Keyrword Arguments: + - netixlan : if set, extra data will be added + to the log. + """ + peer = { + "ixlan_id": self.ixlan.id, + "asn": asn, + } + + if netixlan: + peer.update({ + "net_id": netixlan.network_id, + "ipaddr4": u"{}".format(netixlan.ipaddr4 or ""), + "ipaddr6": u"{}".format(netixlan.ipaddr6 or ""), + "speed": netixlan.speed, + "is_rs_peer": netixlan.is_rs_peer, + }) + + self.log["data"].append({ + "peer": peer, + "action": action, + "reason": u"{}".format(reason), + }) + + def log_error(self, error, save=False): + """ + Append error to the attempt log + """ + self.log["errors"].append(u"{}".format(error)) + if save: + self.save_log() diff --git a/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py b/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py index 177fe6dc..eeae475c 100644 --- a/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py +++ b/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py @@ -1,9 +1,12 @@ +import traceback +import json + from django.core.management.base import BaseCommand, CommandError from django.db import transaction from peeringdb_server.models import ( IXLan, ) -import traceback +from peeringdb_server import ixf class Command(BaseCommand): @@ -33,13 +36,11 @@ class Command(BaseCommand): self.log("Fetching data for {} from {}".format( ixlan, ixlan.ixf_ixp_member_list_url)) try: - json_data = ixlan.fetch_ixf_ixp_members_list() + importer = ixf.Importer() self.log("Updating {}".format(ixlan)) with transaction.atomic(): - success, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - json_data, save=self.commit) - for line in log: - self.log(line) + success, netixlans, netixlans_deleted, log = importer.update(ixlan, save=self.commit) + self.log(json.dumps(log)) self.log("Done: {} updated: {} deleted: {}".format( success, len(netixlans), len(netixlans_deleted))) except Exception as inst: diff --git a/peeringdb_server/models.py b/peeringdb_server/models.py index fba95c25..9cda3616 100644 --- a/peeringdb_server/models.py +++ b/peeringdb_server/models.py @@ -1350,59 +1350,6 @@ class IXLan(pdb_models.IXLanBase): def nsp_has_perms_PUT(self, user, request): return validate_PUT_ownership(user, self, request.data, ["ix"]) - def fetch_ixf_ixp_members_list(self): - """ - Retrieves ixf member export data from the url provided in - ixf_ixp_member_list_url. - - Will do a quick sanity check on the data - - Returns dict containing the parsed data. - """ - - if not self.ixf_ixp_member_list_url: - return None - - r = requests.get(self.ixf_ixp_member_list_url, timeout=5) - if r.status_code != 200: - # FIXME: log error somewhere - return None - - try: - o = r.json() - except Exception as inst: - o = {"pdb_sanitize_error": _("No JSON could be parsed")} - return o - - invalid = None - vlan_list_found = False - - # sanitize - - # This fixes instances where ixps provide two separate entries for - # vlans in vlan_list for ipv4 and ipv6 (AMS-IX for example) - for m in o.get("member_list", []): - asn = m.get("asnum") - for c in m.get("connection_list", []): - vlans = c.get("vlan_list", []) - if not vlans: - continue - vlan_list_found = True - if len(vlans) == 2: - # if vlans[0].get("vlan_id") == vlans[1].get("vlan_id"): - keys = vlans[0].keys() + vlans[1].keys() - if keys.count("ipv4") == 1 and keys.count("ipv6") == 1: - vlans[0].update(**vlans[1]) - c["vlan_list"] = [vlans[0]] - print("Sanitized", c["vlan_list"]) - - if not vlan_list_found: - invalid = _("No entries in any of the vlan_list lists, aborting.") - - o["pdb_sanitize_error"] = invalid - - return o - @reversion.create_revision() def add_netixlan(self, netixlan_info, save=True, save_others=True): """ @@ -1427,12 +1374,17 @@ class IXLan(pdb_models.IXLanBase): log = [] changed = False + created = False ipv4 = netixlan_info.ipaddr4 ipv6 = netixlan_info.ipaddr6 asn = netixlan_info.asn ipv4_valid = False ipv6_valid = False + def result(netixlan=None): + return { "netixlan": netixlan, "created": created, + "changed": changed, "log":log} + # check if either of the provided ip addresses are a fit for ANY of # the prefixes in this ixlan for pfx in self.ixpfx_set_active: @@ -1446,24 +1398,24 @@ class IXLan(pdb_models.IXLanBase): if (ipv4 and not ipv4_valid) or (ipv6 and not ipv6_valid): log.append( "Ip addresses ({}, {}) do not match any prefix " \ - "on this ixlan. skipping.".format(ipv4, ipv6)) - return (None, changed, log) + "on this ixlan".format(ipv4, ipv6)) + return result() # Next we check if an active netixlan with the ipaddress exists in ANOTHER lan, and bail # if it does. if ipv4 and \ NetworkIXLan.objects.filter(status="ok", ipaddr4=ipv4).exclude(ixlan=self).count() > 0: log.append( - "Ip address {} already exists in another lan. skipping.". + "Ip address {} already exists in another lan". format(ipv4)) - return (None, changed, log) + return result() if ipv6 and \ NetworkIXLan.objects.filter(status="ok", ipaddr6=ipv6).exclude(ixlan=self).count() > 0: log.append( - "Ip address {} already exists in another lan. skipping.". + "Ip address {} already exists in another lan". format(ipv6)) - return (None, changed, log) + return result() # now we need to figure out if the ipaddresses already exist in this ixlan, # we need to check ipv4 and ipv6 separately as they might exist on different @@ -1506,6 +1458,7 @@ class IXLan(pdb_models.IXLanBase): # neither address exists, create a new netixlan object netixlan = NetworkIXLan(ixlan=self, network=netixlan_info.network, status="ok") + created = True # now we sync the data to our determined netixlan instance @@ -1576,182 +1529,13 @@ class IXLan(pdb_models.IXLanBase): log.append("Validation Failure AS{} {} {}: {}".format( netixlan.network.asn, netixlan.ipaddr4, netixlan.ipaddr6, inst)) - return (None, changed, log) + return result(None) if save and changed: netixlan.status = "ok" netixlan.save() - return (netixlan, changed, log) - - def update_from_ixf_ixp_member_list(self, json_data, save=True): - """ - Sync netixlans under this ixlan from ixf member export json data (specs - can be found at https://github.com/euro-ix/json-schemas) - - Arguments: - - json_data (dict): parsed dict from ixf member export json (schema - described here https://github.com/euro-ix/json-schemas) - - Keyword Arguments: - - save (bool): commit changes to db - - Returns: - - Tuple(success, netixlans, log) - """ - - log = [] - netixlans = [] - ipaddresses = [] - ixlan = self - - def persist_attempt(): - print(log) - if save: - IXLanIXFMemberImportAttempt.objects.update_or_create( - ixlan=ixlan, defaults={ - "info": "\n".join(log) - }) - - if json_data.get("pdb_sanitize_error"): - log.append(json_data.get("pdb_sanitize_error")) - persist_attempt() - return (False, [], [], log) - - if self.ixpfx_set_active.count() == 0: - log.append("No prefixes defined on ixlan, skipping.") - persist_attempt() - return (False, [], [], log) - - with transaction.atomic(): - try: - member_list = json_data["member_list"] - for member in member_list: - # we only process members of certain types - if member.get("member_type", "peering") in \ - ["peering", "ixp", "routeserver", "probono"]: - # check that the as exists in pdb - asn = member["asnum"] - - if Network.objects.filter(asn=asn).exists(): - network = Network.objects.get(asn=asn) - if network.status != "ok": - log.append( - "Network with ASN {} status invalid, skipping..". - format(asn)) - continue - for connection in member.get( - "connection_list", []): - if connection.get("state", "active") in [ - "active", "connected" - ]: - - speed = 0 - for iface in connection.get("if_list", []): - speed += iface.get("if_speed", 0) - - for lan in connection.get("vlan_list", []): - ipv4_valid = False - ipv6_valid = False - - ipv4 = lan.get("ipv4", {}) - ipv6 = lan.get("ipv6", {}) - if not ipv4 and not ipv6: - log.append("Could not find ipv4 or 6 address in " \ - "vlan_list entry for vlan_id {}".format( \ - lan.get("vlan_id"))) - continue - - ipv4_addr = ipv4.get("address") - ipv6_addr = ipv6.get("address") - - try: - if ipv4_addr: - ipaddresses.append( - "{}-{}".format( - asn, - ipaddress.ip_address( - unicode( - ipv4_addr)))) - if ipv6_addr: - ipaddresses.append( - "{}-{}".format( - asn, - ipaddress.ip_address( - unicode( - ipv6_addr)))) - except (ipaddress.AddressValueError, - ValueError) as exc: - log.append( - "Invalid ipaddress '{}' for ASN {} - skipping". - format(exc, asn)) - continue - - if not network.allow_ixp_update: - log.append( - "Network with ASN {} has disabled ixp " \ - "updates, skipping..".format(asn)) - continue - - netixlan, changed, _log = self.add_netixlan( - NetworkIXLan( - ixlan=self, - network=network, - ipaddr4=ipv4_addr, - ipaddr6=ipv6_addr, - speed=(speed or 0), - asn=asn, - is_rs_peer=(ipv4.get("routeserver", False) or \ - ipv6.get("routeserver", False)) - ), - save=save, - save_others=save - ) - - log.extend(_log) - if netixlan and changed: - netixlans.append(netixlan) - - else: - log.append( - "Network with asn {} does not exist in " \ - "PeeringDB, skipping...".format(asn)) - - except KeyError, exc: - log.append("Internal Error 'KeyError': {}".format(exc)) - persist_attempt() - return (False, netixlans, [], log) - - with reversion.create_revision(): - netixlans_deleted = [] - for netixlan in self.netixlan_set_active: - ipv4 = "{}-{}".format(netixlan.asn, netixlan.ipaddr4) - ipv6 = "{}-{}".format(netixlan.asn, netixlan.ipaddr6) - if ipv4 not in ipaddresses and ipv6 not in ipaddresses: - log.append("Removing ASN {} IPV4 {} IPV6 {}".format( - netixlan.asn, netixlan.ipaddr4, netixlan.ipaddr6)) - netixlans_deleted.append(netixlan) - if save: - netixlan.delete() - - with transaction.atomic(): - persist_attempt() - if save and (netixlans or netixlans_deleted): - persist_log = IXLanIXFMemberImportLog.objects.create( - ixlan=self) - for netixlan in netixlans + netixlans_deleted: - versions = reversion.models.Version.objects.get_for_object( - netixlan) - if len(versions) == 1: - version_before = None - else: - version_before = versions[1] - version_after = versions[0] - persist_log.entries.create(netixlan=netixlan, - version_before=version_before, - version_after=version_after) - - return (True, netixlans, netixlans_deleted, log) + return result(netixlan) class IXLanIXFMemberImportAttempt(models.Model): diff --git a/peeringdb_server/static/peeringdb.js b/peeringdb_server/static/peeringdb.js index 0afd410c..39a734c5 100644 --- a/peeringdb_server/static/peeringdb.js +++ b/peeringdb_server/static/peeringdb.js @@ -156,6 +156,117 @@ function moveCursorToEnd(el) { } } +PeeringDB.IXFPreview = { + + /** + * Handle the IX-F import preview request and rendering + * to UI modal + * + * @class IXFPreview + * @namespace PeeringDB + */ + + request : function(ixlanId, renderTo) { + + /** + * request a preview for the ixlan with ixlanId + * + * @method request + * @param {Number} ixlanId + * @param {jQuery} renderTo - render to this element (needs to have + * the appropriate children elements to work, they are not + * created automatically) + */ + + renderTo.find('.ixf-result').empty(). + append($("
").addClass("center").text("... loading ...")); + renderTo.find('.ixf-error-counter').empty(); + $.get('/import/ixlan/'+ixlanId+'/ixf/preview', function(result) { + this.render(result, renderTo); + }.bind(this)).error(function(result) { + if(result.responseJSON) { + this.render(result.responseJSON, renderTo); + } else { + this.render({"non_field_errors": ["HTTP error "+result.status]}); + } + }.bind(this)); + }, + + render : function(result, renderTo) { + + /** + * Render preview result and errors + * + * @method render + * @param {Object} result - result as returned from the preview request + * @param {jQuery} renderTo + * + * Needs to have child divs with the following classes + * + * + * .ixf-errors-list: errors will be rendered to here + * .ixf-result: changes will be rendered to here + * .ixf-error-counter: will be updated with number of errors + * + */ + + renderTo.find('.ixf-errors-list').empty() + renderTo.find('.ixf-result').empty() + this.render_errors((result.errors || []).concat(result.non_field_errors || []), renderTo.find('.ixf-errors-list')); + this.render_data(result.data || [], renderTo.find('.ixf-result')); + }, + + render_errors : function(errors, renderTo) { + /** + * Render the errors, called automatically by `render` + * + * @method render_errors + * @param {Array} errors + * @param {jQuery} renderTo + */ + + var error, i; + + if(!errors.length) + return; + + $('.ixf-error-counter').text("("+errors.length+")"); + + for(i = 0; i < errors.length; i++) { + error = errors[i]; + renderTo.append($('
').addClass("ixf-error").text(error)); + } + }, + + render_data : function(data, renderTo) { + /** + * Renders the changes made by the ix-f import, called automatically + * by `render` + * + * @method render_data + * @param {Array} data + * @param {jQuery} renderTo + */ + + var row, i; + for(i = 0; i < data.length; i++) { + row = data[i]; + renderTo.append( + $('
').addClass("row ixf-row ixf-"+row.action).append( + $('
').addClass("col-sm-1").text(row.action), + $('
').addClass("col-sm-2").text("AS"+row.peer.asn), + $('
').addClass("col-sm-3").text(row.peer.ipaddr4 || "-"), + $('
').addClass("col-sm-3").text(row.peer.ipaddr6 || "-"), + $('
').addClass("col-sm-1").text(PeeringDB.pretty_speed(row.peer.speed)), + $('
').addClass("col-sm-2").text(row.peer.is_rs_peer?"yes":"no"), + $('
').addClass("col-sm-12 ixf-reason").text(row.reason) + ) + ); + } + } + +} + PeeringDB.InlineSearch = { init_search : function() { diff --git a/peeringdb_server/static/site.css b/peeringdb_server/static/site.css index 4fbae790..e4bada4e 100644 --- a/peeringdb_server/static/site.css +++ b/peeringdb_server/static/site.css @@ -83,6 +83,12 @@ a.btn { text-decoration: none; } +.btn-inline { + position: absolute; + right: 5px; + top: 0px; +} + h5 { font-weight: bold; } @@ -1035,3 +1041,64 @@ span.inline.delimited input { div[name=ipaddr6], div[data-edit-name=ipaddr6] { word-break: break-all; } + +.ixf-log { + max-height: 400px; + overflow-y: scroll; + overflow-x: hidden; +} + +.ixf-row { + padding: 2px; + border-bottom: 1px rgba(0,0,0,0.25) solid; +} + +.ixf-skip { + background-color: #ffffd7; +} + +.ixf-add { + background-color: #c5ffcd; +} + +.ixf-modify { + background-color: #c5e9ff; +} + +.ixf-delete { + background-color: #fbdbdb; +} + +.ixf-noop { + background-color: #eaeaea; +} + + +.ixf-reason { + font-size: 11px; +} + +.ixf-error { + color: red; + padding: 2px; +} + +.ixf-errors-list { +} + +.ixf-error-counter:not(:empty) { + color: red; + font-weight: bold; +} + +.ixf-errors-overview { + background-color: #fbdbdb; + color: red; + text-align: center; + font-weight: bold; + cursor: pointer; +} + +.ixf-errors-overview small { + color: #666; +} diff --git a/peeringdb_server/templates/site/view_exchange_bottom.html b/peeringdb_server/templates/site/view_exchange_bottom.html index ff4d9fae..67542294 100644 --- a/peeringdb_server/templates/site/view_exchange_bottom.html +++ b/peeringdb_server/templates/site/view_exchange_bottom.html @@ -82,6 +82,11 @@ data-edit-value="{{ x.ixf_ixp_member_list_url|fallback:"" }}" data-edit-placeholder="{% trans "IXF Member Export URL" %}" data-edit-name="ixf_ixp_member_list_url">{{ x.ixf_ixp_member_list_url|fallback:"" }}
+ {% if permissions.can_write %} + + {% endif %}
+{% if permissions.can_write %} + + + + + +{% endif %} + + diff --git a/peeringdb_server/urls.py b/peeringdb_server/urls.py index 1449bf1a..d096e34b 100644 --- a/peeringdb_server/urls.py +++ b/peeringdb_server/urls.py @@ -19,6 +19,10 @@ from peeringdb_server.export_views import ( AdvancedSearchExportView, ) +from peeringdb_server.import_views import ( + view_import_ixlan_ixf_preview, +) + from django.views.i18n import JavaScriptCatalog # o @@ -119,6 +123,8 @@ urlpatterns = [ view_export_ixf_ixlan_members), url(r'^export/advanced-search/(?P[\w_]+)/(?P[\w_-]+)$', AdvancedSearchExportView.as_view()), + url(r'^import/ixlan/(?P\d+)/ixf/preview$', + view_import_ixlan_ixf_preview), url(r'^$', view_index), url(r'^i18n/', include('django.conf.urls.i18n')), url('jsi18n/', JavaScriptCatalog.as_view(), name='javascript-catalog'), diff --git a/tests/data/ixf/logs/skip_disabled_networks.json b/tests/data/ixf/logs/skip_disabled_networks.json new file mode 100644 index 00000000..93175761 --- /dev/null +++ b/tests/data/ixf/logs/skip_disabled_networks.json @@ -0,0 +1,64 @@ +{ + "errors": [ + "Invalid speed value: fast", + "Could not find ipv4 or 6 address in vlan_list entry for vlan_id 1 (AS2906)", + "Ip address error 'u'195.69.146.error' does not appear to be an IPv4 or IPv6 address' in vlan_list entry for vlan_id 2" + ], + "data": [ + { + "peer": { + "ixlan_id": 1, + "asn": 2906 + }, + "action": "skip", + "reason": "Invalid connection state: disabled" + }, + { + "peer": { + "is_rs_peer": true, + "ipaddr4": "195.69.146.250", + "net_id": 1, + "speed": 10000, + "ixlan_id": 1, + "asn": 2906, + "ipaddr6": "2001:7f8:1::a500:2906:2" + }, + "action": "skip", + "reason": "Network has disabled ixp updates" + }, + { + "peer": { + "is_rs_peer": true, + "ipaddr4": "195.69.147.250", + "net_id": 1, + "speed": 10000, + "ixlan_id": 1, + "asn": 2906, + "ipaddr6": "2001:7f8:1::a500:2906:1" + }, + "action": "skip", + "reason": "Network has disabled ixp updates" + }, + { + "peer": { + "ixlan_id": 1, + "asn": 2906 + }, + "action": "skip", + "reason": "Invalid member type: other" + }, + { + "peer": { + "is_rs_peer": false, + "ipaddr4": "195.69.146.249", + "net_id": 1, + "speed": 10000, + "ixlan_id": 1, + "asn": 2906, + "ipaddr6": "" + }, + "action": "delete", + "reason": "Ip addresses no longer in data" + } + ] +} diff --git a/tests/data/ixf/logs/skip_prefix_mismatch.json b/tests/data/ixf/logs/skip_prefix_mismatch.json new file mode 100644 index 00000000..f2167d09 --- /dev/null +++ b/tests/data/ixf/logs/skip_prefix_mismatch.json @@ -0,0 +1,51 @@ +{ + "errors": [ + "Invalid speed value: fast", + "Could not find ipv4 or 6 address in vlan_list entry for vlan_id 1 (AS2906)", + "Ip address error 'u'195.69.146.error' does not appear to be an IPv4 or IPv6 address' in vlan_list entry for vlan_id 2" + ], + "data": [ + { + "peer": { + "ixlan_id": 2, + "asn": 2906 + }, + "action": "skip", + "reason": "Invalid connection state: disabled" + }, + { + "peer": { + "is_rs_peer": true, + "ipaddr4": "195.69.146.250", + "net_id": 1, + "speed": 10000, + "ixlan_id": 2, + "asn": 2906, + "ipaddr6": "2001:7f8:1::a500:2906:2" + }, + "action": "skip", + "reason": "Ip addresses (195.69.146.250, 2001:7f8:1::a500:2906:2) do not match any prefix on this ixlan" + }, + { + "peer": { + "is_rs_peer": true, + "ipaddr4": "195.69.147.250", + "net_id": 1, + "speed": 10000, + "ixlan_id": 2, + "asn": 2906, + "ipaddr6": "2001:7f8:1::a500:2906:1" + }, + "action": "skip", + "reason": "Ip addresses (195.69.147.250, 2001:7f8:1::a500:2906:1) do not match any prefix on this ixlan" + }, + { + "peer": { + "ixlan_id": 2, + "asn": 2906 + }, + "action": "skip", + "reason": "Invalid member type: other" + } + ] +} diff --git a/tests/data/ixf/logs/update_01.json b/tests/data/ixf/logs/update_01.json new file mode 100644 index 00000000..05d49a2b --- /dev/null +++ b/tests/data/ixf/logs/update_01.json @@ -0,0 +1,64 @@ +{ + "errors": [ + "Invalid speed value: fast", + "Could not find ipv4 or 6 address in vlan_list entry for vlan_id 1 (AS2906)", + "Ip address error 'u'195.69.146.error' does not appear to be an IPv4 or IPv6 address' in vlan_list entry for vlan_id 2" + ], + "data": [ + { + "peer": { + "ixlan_id": 1, + "asn": 2906 + }, + "action": "skip", + "reason": "Invalid connection state: disabled" + }, + { + "peer": { + "is_rs_peer": true, + "ipaddr4": "195.69.146.250", + "net_id": 1, + "speed": 10000, + "ixlan_id": 1, + "asn": 2906, + "ipaddr6": "2001:7f8:1::a500:2906:2" + }, + "action": "add", + "reason": "" + }, + { + "peer": { + "is_rs_peer": true, + "ipaddr4": "195.69.147.250", + "net_id": 1, + "speed": 10000, + "ixlan_id": 1, + "asn": 2906, + "ipaddr6": "2001:7f8:1::a500:2906:1" + }, + "action": "add", + "reason": "" + }, + { + "peer": { + "ixlan_id": 1, + "asn": 2906 + }, + "action": "skip", + "reason": "Invalid member type: other" + }, + { + "peer": { + "is_rs_peer": false, + "ipaddr4": "195.69.146.249", + "net_id": 1, + "speed": 10000, + "ixlan_id": 1, + "asn": 2906, + "ipaddr6": "" + }, + "action": "delete", + "reason": "Ip addresses no longer in data" + } + ] +} diff --git a/tests/data/json_members_list/members.0.4.json b/tests/data/json_members_list/members.0.4.json index e419c35b..6a8cedaf 100644 --- a/tests/data/json_members_list/members.0.4.json +++ b/tests/data/json_members_list/members.0.4.json @@ -19,6 +19,9 @@ "peering_policy_url": "https://www.netflix.com/openconnect/", "member_since": "2009-02-04T00:00:00Z", "connection_list": [ + { + "state" : "disabled" + }, { "ixp_id": 42, "connected_since": "2009-02-04T00:00:00Z", @@ -28,6 +31,9 @@ "switch_id": 0, "if_speed": 10000, "if_type": "LR4" + }, + { + "if_speed": "fast" } ], "vlan_list": [ @@ -46,6 +52,15 @@ "max_prefix": 42, "as_macro": "AS-NFLX-V6" } + }, + { + "vlan_id" : 1 + }, + { + "vlan_id" : 2, + "ipv4": { + "address": "195.69.146.error" + } } ] }, @@ -80,6 +95,9 @@ ] } ] + }, + { + "member_type": "other" } ] } diff --git a/tests/data/json_members_list/members.0.5.json b/tests/data/json_members_list/members.0.5.json index aaca8771..ef6d021c 100644 --- a/tests/data/json_members_list/members.0.5.json +++ b/tests/data/json_members_list/members.0.5.json @@ -19,6 +19,9 @@ "peering_policy_url": "https://www.netflix.com/openconnect/", "member_since": "2009-02-04T00:00:00Z", "connection_list": [ + { + "state" : "disabled" + }, { "ixp_id": 42, "connected_since": "2009-02-04T00:00:00Z", @@ -28,6 +31,9 @@ "switch_id": 0, "if_speed": 10000, "if_type": "LR4" + }, + { + "if_speed": "fast" } ], "vlan_list": [ @@ -76,10 +82,23 @@ "max_prefix": 42, "as_macro": "AS-NFLX-V6" } + }, + { + "vlan_id": 1 + }, + { + "vlan_id": 2, + "ipv4": { + "address": "195.69.146.error" + } } ] } ] + }, + { + "member_type": "other" } + ] } diff --git a/tests/data/json_members_list/members.0.6.json b/tests/data/json_members_list/members.0.6.json index d68bff51..8a46f32e 100644 --- a/tests/data/json_members_list/members.0.6.json +++ b/tests/data/json_members_list/members.0.6.json @@ -31,6 +31,9 @@ "switch_id": 0, "if_speed": 10000, "if_type": "LR4" + }, + { + "if_speed": "fast" } ], "vlan_list": [ @@ -54,6 +57,15 @@ "00:0a:95:9d:68:16" ] } + }, + { + "vlan_id": 1 + }, + { + "vlan_id": 2, + "ipv4": { + "address": "195.69.146.error" + } } ] }, diff --git a/tests/django_init.py b/tests/django_init.py index 32247bc1..428eb54e 100644 --- a/tests/django_init.py +++ b/tests/django_init.py @@ -163,5 +163,6 @@ settings.configure( "view_request_ownership_POST": "3/m", "request_login_POST": "10/m", "view_verify_POST": "2/m", - "request_translation": "10/m" + "request_translation": "10/m", + "view_import_ixlan_ixf_preview": "1/m" }) diff --git a/tests/test_ixf_member_import.py b/tests/test_ixf_member_import.py index 40560fe6..64810492 100644 --- a/tests/test_ixf_member_import.py +++ b/tests/test_ixf_member_import.py @@ -9,7 +9,13 @@ from django.test import TestCase, Client, RequestFactory from peeringdb_server.models import ( Organization, Network, NetworkIXLan, IXLan, IXLanPrefix, InternetExchange, IXLanIXFMemberImportAttempt, IXLanIXFMemberImportLog, - IXLanIXFMemberImportLogEntry) + IXLanIXFMemberImportLogEntry, User) +from peeringdb_server.import_views import ( + view_import_ixlan_ixf_preview, + ) +from peeringdb_server import ixf + +from util import ClientCase class JsonMembersListTestCase(TestCase): @@ -104,6 +110,14 @@ class JsonMembersListTestCase(TestCase): ipaddr4="195.69.146.249", ipaddr6=None, status="ok"), ] + def setUp(self): + self.ixf_importer = ixf.Importer() + + def assertLog(self, log, expected): + path = os.path.join(os.path.dirname(__file__), "data", "ixf", "logs", "{}.json".format(expected)) + with open(path, "r") as fh: + self.assertEqual(log, json.load(fh)) + def test_update_from_ixf_ixp_member_list(self): ixlan = self.entities["ixlan"][0] n_deleted = self.entities["netixlan"][0] @@ -112,11 +126,9 @@ class JsonMembersListTestCase(TestCase): self.assertEqual( unicode(n_deleted2.ipaddr6), u'2001:7f8:1::a500:2906:1') self.assertEqual(ixlan.netixlan_set_active.count(), 1) - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) - print(log) - self.assertEqual(len(log), 1) + self.assertLog(log, "update_01") self.assertEqual(len(netixlans), 2) self.assertEqual(len(netixlans_deleted), 1) @@ -152,10 +164,9 @@ class JsonMembersListTestCase(TestCase): against any of the prefixes that exist on the ixlan get skipped """ ixlan = self.entities["ixlan"][1] - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) - self.assertEqual(len(log), 2) + self.assertLog(log, "skip_prefix_mismatch") self.assertEqual(len(netixlans), 0) def test_update_from_ixf_ixp_member_list_skip_missing_prefixes(self): @@ -164,12 +175,11 @@ class JsonMembersListTestCase(TestCase): ixlan that does not have any prefixes """ ixlan = self.entities["ixlan"][2] - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) self.assertEqual(len(netixlans), 0) self.assertEqual(len(netixlans_deleted), 0) - self.assertEqual(log, [u'No prefixes defined on ixlan, skipping.']) + self.assertEqual(log["errors"], [u'No prefixes defined on ixlan']) def test_update_from_ixf_ixp_member_list_skip_disabled_networks(self): """ @@ -180,10 +190,9 @@ class JsonMembersListTestCase(TestCase): network = self.entities["net"][0] network.allow_ixp_update = False network.save() - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) - self.assertEqual(len(log), 3) + self.assertLog(log, "skip_disabled_networks") self.assertEqual(len(netixlans), 0) for netixlan in network.netixlan_set_active.all(): @@ -192,8 +201,7 @@ class JsonMembersListTestCase(TestCase): def test_update_from_ixf_ixp_member_list_logs(self): ixlan = self.entities["ixlan"][0] - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) attempt_dt_1 = ixlan.ixf_import_attempt.updated @@ -221,8 +229,7 @@ class JsonMembersListTestCase(TestCase): time.sleep(0.1) - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) ixlan.ixf_import_attempt.refresh_from_db() attempt_dt_2 = ixlan.ixf_import_attempt.updated @@ -243,8 +250,7 @@ class JsonMembersListTestCase(TestCase): def test_rollback(self): ixlan = self.entities["ixlan"][0] - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) for entry in ixlan.ixf_import_log_set.last().entries.all(): self.assertEqual(entry.rollback_status(), 0) @@ -262,8 +268,7 @@ class JsonMembersListTestCase(TestCase): def test_rollback_avoid_ipaddress_conflict(self): ixlan = self.entities["ixlan"][0] - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) self.assertEqual(len(netixlans_deleted), 1) @@ -294,8 +299,7 @@ class JsonMembersListTestCase(TestCase): # import the data ixlan = self.entities["ixlan"][0] - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) # request the view and compare it agaisnt expected data c = Client() @@ -325,8 +329,7 @@ class JsonMembersListTestCase(TestCase): # import the data ixlan = self.entities["ixlan"][0] - r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list( - self.json_data) + r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data) # request the view and compare it agaisnt expected data c = Client() @@ -354,3 +357,52 @@ class JsonMembersListTestCase_V05(JsonMembersListTestCase): class JsonMembersListTestCase_V04(JsonMembersListTestCase): version = "0.4" + + +class TestImportPreview(ClientCase): + + """ + Test the ixf import preview + """ + + @classmethod + def setUpTestData(cls): + super(TestImportPreview, cls).setUpTestData() + cls.org = Organization.objects.create(name="Test Org", status="ok") + cls.ix = InternetExchange.objects.create(name="Test IX", status="ok", org=cls.org) + cls.ixlan = IXLan.objects.create(status="ok", ix=cls.ix) + cls.admin_user = User.objects.create_user("admin","admin@localhost","admin") + + cls.org.admin_usergroup.user_set.add(cls.admin_user) + + + def test_import_preview(self): + request = RequestFactory().get("/import/ixlan/{}/ixf/preview/".format(self.ixlan.id)) + request.user = self.admin_user + + response = view_import_ixlan_ixf_preview(request, self.ixlan.id) + + assert response.status_code == 200 + assert json.loads(response.content)["errors"] == ["IXF import url not specified"] + + + def test_import_preview_fail_ratelimit(self): + request = RequestFactory().get("/import/ixlan/{}/ixf/preview/".format(self.ixlan.id)) + request.user = self.admin_user + + response = view_import_ixlan_ixf_preview(request, self.ixlan.id) + assert response.status_code == 200 + + response = view_import_ixlan_ixf_preview(request, self.ixlan.id) + assert response.status_code == 400 + + + def test_import_preview_fail_permission(self): + request = RequestFactory().get("/import/ixlan/{}/ixf/preview/".format(self.ixlan.id)) + request.user = self.guest_user + + response = view_import_ixlan_ixf_preview(request, self.ixlan.id) + assert response.status_code == 403 + + +