Merge pull request #458 from peeringdb/pr_408_ixf_preview

IXF preview tool (#408)

peeringdb_server/import_views.py

@@ -0,0 +1,45 @@
+import json
+
+from django.http import JsonResponse, HttpResponse
+from django.conf import settings
+from django.utils.translation import ugettext_lazy as _
+
+from django_namespace_perms.util import has_perms
+from ratelimit.decorators import ratelimit, is_ratelimited
+
+from peeringdb_server import ixf
+from peeringdb_server.models import IXLan
+
+RATELIMITS = settings.RATELIMITS
+
+
+@ratelimit(key="ip", rate=RATELIMITS["view_import_ixlan_ixf_preview"])
+def view_import_ixlan_ixf_preview(request, ixlan_id):
+
+    # check if request was blocked by rate limiting
+    was_limited = getattr(request, "limited", False)
+    if was_limited:
+        return JsonResponse({
+            "non_field_errors": [
+                _("Please wait a bit before requesting " \
+                  "another ixf import preview.")
+            ]
+        }, status=400)
+
+    try:
+        ixlan = IXLan.objects.get(id=ixlan_id)
+    except IXLan.DoesNotExist:
+        return JsonResponse({
+            "non_field_errors": [_("Ixlan not found")]
+        }, status=404)
+
+    if not has_perms(request.user, ixlan, "update"):
+        return JsonResponse({
+            "non_field_errors": [_("Permission denied")]
+        }, status=403)
+
+    importer = ixf.Importer()
+    importer.update(ixlan, save=False)
+
+    return HttpResponse(
+        json.dumps(importer.log, indent=2), content_type="application/json")

peeringdb_server/ixf.py

@@ -0,0 +1,415 @@
+import json
+
+import requests
+import ipaddress
+
+from django.db import transaction
+from django.utils.translation import ugettext_lazy as _
+
+import reversion
+
+from peeringdb_server.models import (
+    IXLanIXFMemberImportAttempt,
+    IXLanIXFMemberImportLog,
+    Network,
+    NetworkIXLan,
+)
+
+
+class Importer(object):
+
+    allowed_member_types = ["peering", "ixp", "routeserver", "probono"]
+    allowed_states = ["active", "connected"]
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self, ixlan=None, save=False):
+        self.reset_log()
+        self.netixlans = []
+        self.netixlans_deleted = []
+        self.ipaddresses = []
+        self.ixlan = ixlan
+        self.save = save
+
+    def fetch(self, url, timeout=5):
+        """
+        Retrieves ixf member export data from the url
+
+        Will do a quick sanity check on the data
+
+        Returns dict containing the parsed data.
+
+        Arguments:
+            - url <str>
+
+        Keyword arguments:
+            - timeout <float>: max time to spend on request
+        """
+
+        if not url:
+            return {"pdb_error": _("IXF import url not specified")}
+
+        try:
+            result = requests.get(url, timeout=timeout)
+        except Exception as exc:
+            return {"pdb_error": exc}
+
+        if result.status_code != 200:
+            # FIXME: log error somewhere
+            return None
+
+        try:
+            data = result.json()
+        except Exception as inst:
+            data = {"pdb_error": _("No JSON could be parsed")}
+            return data
+
+        return self.sanitize(data)
+
+    def sanitize(self, data):
+        """
+        Takes ixf data dict and runs some sanitization on it
+        """
+
+        invalid = None
+        vlan_list_found = False
+
+        # This fixes instances where ixps provide two separate entries for
+        # vlans in vlan_list for ipv4 and ipv6 (AMS-IX for example)
+        for member in data.get("member_list", []):
+            asn = member.get("asnum")
+            for conn in member.get("connection_list", []):
+                vlans = conn.get("vlan_list", [])
+                if not vlans:
+                    continue
+                vlan_list_found = True
+                if len(vlans) == 2:
+                    # if vlans[0].get("vlan_id") == vlans[1].get("vlan_id"):
+                    keys = vlans[0].keys() + vlans[1].keys()
+                    if keys.count("ipv4") == 1 and keys.count("ipv6") == 1:
+                        vlans[0].update(**vlans[1])
+                        conn["vlan_list"] = [vlans[0]]
+
+        if not vlan_list_found:
+            invalid = _("No entries in any of the vlan_list lists, aborting.")
+
+        data["pdb_error"] = invalid
+
+        return data
+
+    def update(self, ixlan, save=True, data=None, timeout=5):
+        """
+        Sync netixlans under this ixlan from ixf member export json data (specs
+        can be found at https://github.com/euro-ix/json-schemas)
+
+        Arguments:
+            - ixlan (IXLan): ixlan object to update from ixf
+
+        Keyword Arguments:
+            - save (bool): commit changes to db
+
+        Returns:
+            - Tuple(success<bool>, netixlans<list>, log<list>)
+        """
+
+        self.reset(ixlan=ixlan, save=save)
+
+        if data is None:
+            data = self.fetch(ixlan.ixf_ixp_member_list_url, timeout=timeout)
+
+        # bail if there has been any errors during sanitize() or fetch()
+        if data.get("pdb_error"):
+            self.log_error(data.get("pdb_error"), save=save)
+            return (False, [], [], self.log)
+
+        # bail if there are no active prefixes on the ixlan
+        if ixlan.ixpfx_set_active.count() == 0:
+            self.log_error(_("No prefixes defined on ixlan"), save=save)
+            return (False, [], [], self.log)
+
+        try:
+            # parse the ixf data
+            self.parse(data)
+        except KeyError as exc:
+            # any key erros mean that the data is invalid, log the error and
+            # bail (transactions are atomic and will be rolled back)
+            self.log_error("Internal Error 'KeyError': {}".format(exc),
+                           save=save)
+            return (False, self.netixlans, [], self.log)
+
+        # process any netixlans that need to be deleted
+        self.process_deletions()
+
+        # archive the import so we can roll it back later if needed
+        self.archive()
+
+        if save:
+            self.save_log()
+
+        return (True, self.netixlans, self.netixlans_deleted, self.log)
+
+    @reversion.create_revision()
+    def process_deletions(self):
+        """
+        Cycles all netixlans on the ixlan targeted by the importer and
+        will remove any that are no longer found in the ixf data by
+        their ip addresses
+
+        In order for a netixlan to be removed both it's ipv4 and ipv6 address
+        need to be gone from the ixf data
+        """
+        for netixlan in self.ixlan.netixlan_set_active:
+            ipv4 = "{}-{}".format(netixlan.asn, netixlan.ipaddr4)
+            ipv6 = "{}-{}".format(netixlan.asn, netixlan.ipaddr6)
+            if ipv4 not in self.ipaddresses and ipv6 not in self.ipaddresses:
+                self.log_peer(netixlan.asn, "delete",
+                              _("Ip addresses no longer in data"), netixlan)
+                self.netixlans_deleted.append(netixlan)
+                if self.save:
+                    netixlan.delete()
+
+    @transaction.atomic()
+    def archive(self):
+        """
+        Create the IXLanIXFMemberImportLog for this import
+        """
+
+        if self.save and (self.netixlans or self.netixlans_deleted):
+            persist_log = IXLanIXFMemberImportLog.objects.create(
+                ixlan=self.ixlan)
+            for netixlan in self.netixlans + self.netixlans_deleted:
+                versions = reversion.models.Version.objects.get_for_object(
+                    netixlan)
+                if len(versions) == 1:
+                    version_before = None
+                else:
+                    version_before = versions[1]
+                version_after = versions[0]
+                persist_log.entries.create(netixlan=netixlan,
+                                           version_before=version_before,
+                                           version_after=version_after)
+
+    def parse(self, data):
+        """
+        Parse ixf data
+
+        Arguments:
+            - data <dict>: result from fetch()
+        """
+        with transaction.atomic():
+            self.parse_members(data.get("member_list", []))
+
+    def parse_members(self, member_list):
+        """
+        Parse the `member_list` section of the ixf schema
+
+        Arguments:
+            - member_list <list>
+        """
+        for member in member_list:
+            # we only process members of certain types
+            member_type = member.get("member_type", "peering").lower()
+            if member_type in self.allowed_member_types:
+                # check that the as exists in pdb
+                asn = member["asnum"]
+
+                if Network.objects.filter(asn=asn).exists():
+                    network = Network.objects.get(asn=asn)
+                    if network.status != "ok":
+                        self.log_peer(
+                            asn, "skip",
+                            _("Network status is '{}'").format(network.status))
+                        continue
+
+                    self.parse_connections(
+                        member.get("connection_list", []), network, member)
+                else:
+                    self.log_peer(asn, "skip",
+                                  _("Network does not exist in peeringdb"))
+            else:
+                self.log_peer(asn, "skip",
+                              _("Invalid member type: {}").format(member_type))
+
+    def parse_connections(self, connection_list, network, member):
+        """
+        Parse the 'connection_list' section of the ixf schema
+
+        Arguments:
+            - connection_list <list>
+            - network <Network>: pdb network instance
+            - member <dict>: row from ixf member_list
+        """
+
+        asn = member["asnum"]
+        for connection in connection_list:
+            state = connection.get("state", "active").lower()
+            if state in self.allowed_states:
+
+                speed = self.parse_speed(connection.get("if_list", []))
+
+                self.parse_vlans(
+                    connection.get("vlan_list", []), network, member,
+                    connection, speed)
+            else:
+                self.log_peer(asn, "skip",
+                              _("Invalid connection state: {}").format(state))
+
+    def parse_vlans(self, vlan_list, network, member, connection, speed):
+        """
+        Parse the 'vlan_list' section of the ixf_schema
+
+        Arguments:
+            - vlan_list <list>
+            - network <Network>: pdb network instance
+            - member <dict>: row from ixf member_list
+            - connection <dict>: row from ixf connection_list
+            - speed <int>: interface speed
+        """
+
+        asn = member["asnum"]
+        for lan in vlan_list:
+            ipv4_valid = False
+            ipv6_valid = False
+
+            ipv4 = lan.get("ipv4", {})
+            ipv6 = lan.get("ipv6", {})
+
+            # vlan entry has no ipaddresses set, log and skip
+            if not ipv4 and not ipv6:
+                self.log_error(_("Could not find ipv4 or 6 address in " \
+                              "vlan_list entry for vlan_id {} (AS{})").format(
+                              lan.get("vlan_id"), asn))
+                continue
+
+            ipv4_addr = ipv4.get("address")
+            ipv6_addr = ipv6.get("address")
+
+            # parse and validate the ipaddresses attached to the vlan
+            # append the ipaddresses to self.ipaddresses so we can
+            # later check them to see which netixlans need to be
+            # dropped during `process_deletions`
+            try:
+                if ipv4_addr:
+                    self.ipaddresses.append("{}-{}".format(
+                        asn, ipaddress.ip_address(unicode(ipv4_addr))))
+                if ipv6_addr:
+                    self.ipaddresses.append("{}-{}".format(
+                        asn, ipaddress.ip_address(unicode(ipv6_addr))))
+            except (ipaddress.AddressValueError, ValueError) as exc:
+                self.log_error(
+                    _("Ip address error '{}' in vlan_list entry for vlan_id {}"
+                      ).format(exc, lan.get("vlan_id")))
+                continue
+
+            netixlan_info = NetworkIXLan(
+                    ixlan=self.ixlan,
+                    network=network,
+                    ipaddr4=ipv4_addr,
+                    ipaddr6=ipv6_addr,
+                    speed=speed,
+                    asn=asn,
+                    is_rs_peer=(ipv4.get("routeserver", False) or \
+                        ipv6.get("routeserver", False))
+            )
+
+            # after this point we either add or modify the netixlan, so
+            # now is a good time to check if the related network allows
+            # such updates, bail if not
+            if not network.allow_ixp_update:
+                self.log_peer(asn, "skip",
+                              _("Network has disabled ixp updates"),
+                              netixlan_info)
+                continue
+
+            # add / modify the netixlan
+            result = self.ixlan.add_netixlan(netixlan_info, save=self.save,
+                                             save_others=self.save)
+
+            if result["netixlan"] and result["changed"]:
+                self.netixlans.append(result["netixlan"])
+                if result["created"]:
+                    action = "add"
+                else:
+                    action = "modify"
+
+                self.log_peer(asn, action, "", result["netixlan"])
+            elif result["netixlan"]:
+                self.log_peer(asn, "noop", "", result["netixlan"])
+            elif result["log"]:
+                self.log_peer(asn, "skip", "\n".join(result["log"]),
+                              netixlan_info)
+
+    def parse_speed(self, if_list):
+        """
+        Parse speed from the 'if_list' section in the ixf data
+
+        Arguments:
+            - if_list <list>
+
+        Returns:
+            - speed <int>
+        """
+        speed = 0
+        for iface in if_list:
+            try:
+                speed += int(iface.get("if_speed", 0))
+            except ValueError:
+                self.log_error(
+                    _("Invalid speed value: {}").format(iface.get("if_speed")))
+        return speed
+
+    def save_log(self):
+        """
+        Save the attempt log
+        """
+        IXLanIXFMemberImportAttempt.objects.update_or_create(
+            ixlan=self.ixlan,
+            defaults={"info": "\n".join(json.dumps(self.log))})
+
+    def reset_log(self):
+        """
+        Reset the attempt log
+        """
+        self.log = {"data": [], "errors": []}
+
+    def log_peer(self, asn, action, reason, netixlan=None):
+        """
+        log peer action in attempt log
+
+        Arguments:
+            - asn <int>
+            - action <str>: add | modify | delete | noop | skip
+            - reason <str>
+
+        Keyrword Arguments:
+            - netixlan <Netixlan>: if set, extra data will be added
+                to the log.
+        """
+        peer = {
+            "ixlan_id": self.ixlan.id,
+            "asn": asn,
+        }
+
+        if netixlan:
+            peer.update({
+                "net_id": netixlan.network_id,
+                "ipaddr4": u"{}".format(netixlan.ipaddr4 or ""),
+                "ipaddr6": u"{}".format(netixlan.ipaddr6 or ""),
+                "speed": netixlan.speed,
+                "is_rs_peer": netixlan.is_rs_peer,
+            })
+
+        self.log["data"].append({
+            "peer": peer,
+            "action": action,
+            "reason": u"{}".format(reason),
+        })
+
+    def log_error(self, error, save=False):
+        """
+        Append error to the attempt log
+        """
+        self.log["errors"].append(u"{}".format(error))
+        if save:
+            self.save_log()

peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py

@@ -1,9 +1,12 @@
+import traceback
+import json
+
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 
 from peeringdb_server.models import (
     IXLan, )
-import traceback
+from peeringdb_server import ixf
 
 
 class Command(BaseCommand):
@@ -33,13 +36,11 @@ class Command(BaseCommand):
             self.log("Fetching data for {} from {}".format(
                 ixlan, ixlan.ixf_ixp_member_list_url))
             try:
-                json_data = ixlan.fetch_ixf_ixp_members_list()
+                importer = ixf.Importer()
                 self.log("Updating {}".format(ixlan))
                 with transaction.atomic():
-                    success, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
-                        json_data, save=self.commit)
-                for line in log:
-                    self.log(line)
+                    success, netixlans, netixlans_deleted, log = importer.update(ixlan, save=self.commit)
+                self.log(json.dumps(log))
                 self.log("Done: {} updated: {} deleted: {}".format(
                     success, len(netixlans), len(netixlans_deleted)))
             except Exception as inst:

peeringdb_server/models.py

@@ -1378,59 +1378,6 @@ class IXLan(pdb_models.IXLanBase):
     def nsp_has_perms_PUT(self, user, request):
         return validate_PUT_ownership(user, self, request.data, ["ix"])
 
-    def fetch_ixf_ixp_members_list(self):
-        """
-        Retrieves ixf member export data from the url provided in
-        ixf_ixp_member_list_url.
-
-        Will do a quick sanity check on the data
-
-        Returns dict containing the parsed data.
-        """
-
-        if not self.ixf_ixp_member_list_url:
-            return None
-
-        r = requests.get(self.ixf_ixp_member_list_url, timeout=5)
-        if r.status_code != 200:
-            # FIXME: log error somewhere
-            return None
-
-        try:
-            o = r.json()
-        except Exception as inst:
-            o = {"pdb_sanitize_error": _("No JSON could be parsed")}
-            return o
-
-        invalid = None
-        vlan_list_found = False
-
-        # sanitize
-
-        # This fixes instances where ixps provide two separate entries for
-        # vlans in vlan_list for ipv4 and ipv6 (AMS-IX for example)
-        for m in o.get("member_list", []):
-            asn = m.get("asnum")
-            for c in m.get("connection_list", []):
-                vlans = c.get("vlan_list", [])
-                if not vlans:
-                    continue
-                vlan_list_found = True
-                if len(vlans) == 2:
-                    # if vlans[0].get("vlan_id") == vlans[1].get("vlan_id"):
-                    keys = vlans[0].keys() + vlans[1].keys()
-                    if keys.count("ipv4") == 1 and keys.count("ipv6") == 1:
-                        vlans[0].update(**vlans[1])
-                        c["vlan_list"] = [vlans[0]]
-                        print("Sanitized", c["vlan_list"])
-
-        if not vlan_list_found:
-            invalid = _("No entries in any of the vlan_list lists, aborting.")
-
-        o["pdb_sanitize_error"] = invalid
-
-        return o
-
     @reversion.create_revision()
     def add_netixlan(self, netixlan_info, save=True, save_others=True):
         """
@@ -1455,12 +1402,17 @@ class IXLan(pdb_models.IXLanBase):
 
         log = []
         changed = False
+        created = False
         ipv4 = netixlan_info.ipaddr4
         ipv6 = netixlan_info.ipaddr6
         asn = netixlan_info.asn
         ipv4_valid = False
         ipv6_valid = False
 
+        def result(netixlan=None):
+            return { "netixlan": netixlan, "created": created,
+                     "changed": changed, "log":log}
+
         # check if either of the provided ip addresses are a fit for ANY of
         # the prefixes in this ixlan
         for pfx in self.ixpfx_set_active:
@@ -1474,24 +1426,24 @@ class IXLan(pdb_models.IXLanBase):
         if (ipv4 and not ipv4_valid) or (ipv6 and not ipv6_valid):
             log.append(
                 "Ip addresses ({}, {}) do not match any prefix " \
-                "on this ixlan. skipping.".format(ipv4, ipv6))
-            return (None, changed, log)
+                "on this ixlan".format(ipv4, ipv6))
+            return result()
 
         # Next we check if an active netixlan with the ipaddress exists in ANOTHER lan, and bail
         # if it does.
         if ipv4 and \
             NetworkIXLan.objects.filter(status="ok", ipaddr4=ipv4).exclude(ixlan=self).count() > 0:
             log.append(
-                "Ip address {} already exists in another lan. skipping.".
+                "Ip address {} already exists in another lan".
                 format(ipv4))
-            return (None, changed, log)
+            return result()
 
         if ipv6 and \
             NetworkIXLan.objects.filter(status="ok", ipaddr6=ipv6).exclude(ixlan=self).count() > 0:
             log.append(
-                "Ip address {} already exists in another lan. skipping.".
+                "Ip address {} already exists in another lan".
                 format(ipv6))
-            return (None, changed, log)
+            return result()
 
         # now we need to figure out if the ipaddresses already exist in this ixlan,
         # we need to check ipv4 and ipv6 separately as they might exist on different
@@ -1534,6 +1486,7 @@ class IXLan(pdb_models.IXLanBase):
             # neither address exists, create a new netixlan object
             netixlan = NetworkIXLan(ixlan=self, network=netixlan_info.network,
                                     status="ok")
+            created = True
 
         # now we sync the data to our determined netixlan instance
 
@@ -1604,182 +1557,13 @@ class IXLan(pdb_models.IXLanBase):
             log.append("Validation Failure AS{} {} {}: {}".format(
                 netixlan.network.asn, netixlan.ipaddr4, netixlan.ipaddr6,
                 inst))
-            return (None, changed, log)
+            return result(None)
 
         if save and changed:
             netixlan.status = "ok"
             netixlan.save()
 
-        return (netixlan, changed, log)
-
-    def update_from_ixf_ixp_member_list(self, json_data, save=True):
-        """
-        Sync netixlans under this ixlan from ixf member export json data (specs
-        can be found at https://github.com/euro-ix/json-schemas)
-
-        Arguments:
-            - json_data (dict): parsed dict from ixf member export json (schema
-                described here https://github.com/euro-ix/json-schemas)
-
-        Keyword Arguments:
-            - save (bool): commit changes to db
-
-        Returns:
-            - Tuple(success<bool>, netixlans<list>, log<list>)
-        """
-
-        log = []
-        netixlans = []
-        ipaddresses = []
-        ixlan = self
-
-        def persist_attempt():
-            print(log)
-            if save:
-                IXLanIXFMemberImportAttempt.objects.update_or_create(
-                    ixlan=ixlan, defaults={
-                        "info": "\n".join(log)
-                    })
-
-        if json_data.get("pdb_sanitize_error"):
-            log.append(json_data.get("pdb_sanitize_error"))
-            persist_attempt()
-            return (False, [], [], log)
-
-        if self.ixpfx_set_active.count() == 0:
-            log.append("No prefixes defined on ixlan, skipping.")
-            persist_attempt()
-            return (False, [], [], log)
-
-        with transaction.atomic():
-            try:
-                member_list = json_data["member_list"]
-                for member in member_list:
-                    # we only process members of certain types
-                    if member.get("member_type", "peering") in \
-                    ["peering", "ixp", "routeserver", "probono"]:
-                        # check that the as exists in pdb
-                        asn = member["asnum"]
-
-                        if Network.objects.filter(asn=asn).exists():
-                            network = Network.objects.get(asn=asn)
-                            if network.status != "ok":
-                                log.append(
-                                    "Network with ASN {} status invalid, skipping..".
-                                    format(asn))
-                                continue
-                            for connection in member.get(
-                                    "connection_list", []):
-                                if connection.get("state", "active") in [
-                                        "active", "connected"
-                                ]:
-
-                                    speed = 0
-                                    for iface in connection.get("if_list", []):
-                                        speed += iface.get("if_speed", 0)
-
-                                    for lan in connection.get("vlan_list", []):
-                                        ipv4_valid = False
-                                        ipv6_valid = False
-
-                                        ipv4 = lan.get("ipv4", {})
-                                        ipv6 = lan.get("ipv6", {})
-                                        if not ipv4 and not ipv6:
-                                            log.append("Could not find ipv4 or 6 address in " \
-                                                "vlan_list entry for vlan_id {}".format( \
-                                                lan.get("vlan_id")))
-                                            continue
-
-                                        ipv4_addr = ipv4.get("address")
-                                        ipv6_addr = ipv6.get("address")
-
-                                        try:
-                                            if ipv4_addr:
-                                                ipaddresses.append(
-                                                    "{}-{}".format(
-                                                        asn,
-                                                        ipaddress.ip_address(
-                                                            unicode(
-                                                                ipv4_addr))))
-                                            if ipv6_addr:
-                                                ipaddresses.append(
-                                                    "{}-{}".format(
-                                                        asn,
-                                                        ipaddress.ip_address(
-                                                            unicode(
-                                                                ipv6_addr))))
-                                        except (ipaddress.AddressValueError,
-                                                ValueError) as exc:
-                                            log.append(
-                                                "Invalid ipaddress '{}' for ASN {} - skipping".
-                                                format(exc, asn))
-                                            continue
-
-                                        if not network.allow_ixp_update:
-                                            log.append(
-                                                "Network with ASN {} has disabled ixp " \
-                                                "updates, skipping..".format(asn))
-                                            continue
-
-                                        netixlan, changed, _log = self.add_netixlan(
-                                            NetworkIXLan(
-                                                ixlan=self,
-                                                network=network,
-                                                ipaddr4=ipv4_addr,
-                                                ipaddr6=ipv6_addr,
-                                                speed=(speed or 0),
-                                                asn=asn,
-                                                is_rs_peer=(ipv4.get("routeserver", False) or \
-                                                    ipv6.get("routeserver", False))
-                                            ),
-                                            save=save,
-                                            save_others=save
-                                        )
-
-                                        log.extend(_log)
-                                        if netixlan and changed:
-                                            netixlans.append(netixlan)
-
-                        else:
-                            log.append(
-                                "Network with asn {} does not exist in " \
-                                "PeeringDB, skipping...".format(asn))
-
-            except KeyError, exc:
-                log.append("Internal Error 'KeyError': {}".format(exc))
-                persist_attempt()
-                return (False, netixlans, [], log)
-
-            with reversion.create_revision():
-                netixlans_deleted = []
-                for netixlan in self.netixlan_set_active:
-                    ipv4 = "{}-{}".format(netixlan.asn, netixlan.ipaddr4)
-                    ipv6 = "{}-{}".format(netixlan.asn, netixlan.ipaddr6)
-                    if ipv4 not in ipaddresses and ipv6 not in ipaddresses:
-                        log.append("Removing ASN {} IPV4 {} IPV6 {}".format(
-                            netixlan.asn, netixlan.ipaddr4, netixlan.ipaddr6))
-                        netixlans_deleted.append(netixlan)
-                        if save:
-                            netixlan.delete()
-
-        with transaction.atomic():
-            persist_attempt()
-            if save and (netixlans or netixlans_deleted):
-                persist_log = IXLanIXFMemberImportLog.objects.create(
-                    ixlan=self)
-                for netixlan in netixlans + netixlans_deleted:
-                    versions = reversion.models.Version.objects.get_for_object(
-                        netixlan)
-                    if len(versions) == 1:
-                        version_before = None
-                    else:
-                        version_before = versions[1]
-                    version_after = versions[0]
-                    persist_log.entries.create(netixlan=netixlan,
-                                               version_before=version_before,
-                                               version_after=version_after)
-
-        return (True, netixlans, netixlans_deleted, log)
+        return result(netixlan)
 
 
 class IXLanIXFMemberImportAttempt(models.Model):

peeringdb_server/static/peeringdb.js

@@ -156,6 +156,117 @@ function moveCursorToEnd(el) {
     }
 }
 
+PeeringDB.IXFPreview = {
+
+  /**
+   * Handle the IX-F import preview request and rendering
+   * to UI modal
+   *
+   * @class IXFPreview
+   * @namespace PeeringDB
+   */
+
+  request : function(ixlanId, renderTo) {
+
+    /**
+     * request a preview for the ixlan with ixlanId
+     *
+     * @method request
+     * @param {Number} ixlanId
+     * @param {jQuery} renderTo - render to this element (needs to have
+     *    the appropriate children elements to work, they are not
+     *    created automatically)
+     */
+
+    renderTo.find('.ixf-result').empty().
+      append($("<div>").addClass("center").text("... loading ..."));
+    renderTo.find('.ixf-error-counter').empty();
+    $.get('/import/ixlan/'+ixlanId+'/ixf/preview', function(result) {
+      this.render(result, renderTo);
+    }.bind(this)).error(function(result) {
+      if(result.responseJSON) {
+        this.render(result.responseJSON, renderTo);
+      } else {
+        this.render({"non_field_errors": ["HTTP error "+result.status]});
+      }
+    }.bind(this));
+  },
+
+  render : function(result, renderTo) {
+
+    /**
+     * Render preview result and errors
+     *
+     * @method render
+     * @param {Object} result - result as returned from the preview request
+     * @param {jQuery} renderTo
+     *
+     *    Needs to have child divs with the following classes
+     *
+     *
+     *    .ixf-errors-list: errors will be rendered to here
+     *    .ixf-result: changes will be rendered to here
+     *    .ixf-error-counter: will be updated with number of errors
+     *
+     */
+
+    renderTo.find('.ixf-errors-list').empty()
+    renderTo.find('.ixf-result').empty()
+    this.render_errors((result.errors || []).concat(result.non_field_errors || []), renderTo.find('.ixf-errors-list'));
+    this.render_data(result.data || [], renderTo.find('.ixf-result'));
+  },
+
+  render_errors : function(errors, renderTo) {
+    /**
+     * Render the errors, called automatically by `render`
+     *
+     * @method render_errors
+     * @param {Array} errors
+     * @param {jQuery} renderTo
+     */
+
+    var error, i;
+
+    if(!errors.length)
+      return;
+
+    $('.ixf-error-counter').text("("+errors.length+")");
+
+    for(i = 0; i < errors.length; i++) {
+      error = errors[i];
+      renderTo.append($('<div>').addClass("ixf-error").text(error));
+    }
+  },
+
+  render_data : function(data, renderTo) {
+    /**
+     * Renders the changes made by the ix-f import, called automatically
+     * by `render`
+     *
+     * @method render_data
+     * @param {Array} data
+     * @param {jQuery} renderTo
+     */
+
+    var row, i;
+    for(i = 0; i < data.length; i++) {
+      row = data[i];
+      renderTo.append(
+        $('<div>').addClass("row ixf-row ixf-"+row.action).append(
+          $('<div>').addClass("col-sm-1").text(row.action),
+          $('<div>').addClass("col-sm-2").text("AS"+row.peer.asn),
+          $('<div>').addClass("col-sm-3").text(row.peer.ipaddr4 || "-"),
+          $('<div>').addClass("col-sm-3").text(row.peer.ipaddr6 || "-"),
+          $('<div>').addClass("col-sm-1").text(PeeringDB.pretty_speed(row.peer.speed)),
+          $('<div>').addClass("col-sm-2").text(row.peer.is_rs_peer?"yes":"no"),
+          $('<div>').addClass("col-sm-12 ixf-reason").text(row.reason)
+        )
+      );
+    }
+  }
+
+}
+
 PeeringDB.InlineSearch = {
 
   init_search : function() {

peeringdb_server/static/site.css

@@ -115,6 +115,12 @@ a.btn {
   text-decoration: none;
 }
 
+.btn-inline {
+  position: absolute;
+  right: 5px;
+  top: 0px;
+}
+
 h5 {
   font-weight: bold;
 }
@@ -1183,3 +1189,64 @@ span.inline.delimited input {
 div[name=ipaddr6],
 div[data-edit-name=ipaddr6]
 { word-break: break-all; }
+
+.ixf-log {
+  max-height: 400px;
+  overflow-y: scroll;
+  overflow-x: hidden;
+}
+
+.ixf-row {
+  padding: 2px;
+  border-bottom: 1px rgba(0,0,0,0.25) solid;
+}
+
+.ixf-skip {
+  background-color: #ffffd7;
+}
+
+.ixf-add {
+  background-color: #c5ffcd;
+}
+
+.ixf-modify {
+  background-color: #c5e9ff;
+}
+
+.ixf-delete {
+  background-color: #fbdbdb;
+}
+
+.ixf-noop {
+  background-color: #eaeaea;
+}
+
+
+.ixf-reason {
+  font-size: 11px;
+}
+
+.ixf-error {
+  color: red;
+  padding: 2px;
+}
+
+.ixf-errors-list {
+}
+
+.ixf-error-counter:not(:empty) {
+  color: red;
+  font-weight: bold;
+}
+
+.ixf-errors-overview {
+  background-color: #fbdbdb;
+  color: red;
+  text-align: center;
+  font-weight: bold;
+  cursor: pointer;
+}
+
+.ixf-errors-overview small {
+  color: #666;
+}

peeringdb_server/templates/site/view_exchange_bottom.html

@@ -82,6 +82,11 @@
              data-edit-value="{{ x.ixf_ixp_member_list_url|fallback:"" }}"
              data-edit-placeholder="{% trans "IXF Member Export URL" %}"
              data-edit-name="ixf_ixp_member_list_url">{{ x.ixf_ixp_member_list_url|fallback:"" }}</div>
+        {% if permissions.can_write %}
+        <button class="btn btn-xs btn-inline btn-primary"
+                data-edit-toggled="view"
+                data-ixf-preview="{{ x.id }}">{% trans "Preview" %}</button>
+        {% endif %}
       </div>
       <div class="marg-top-60 pad-side-15"
            data-edit-module="api_listing"
@@ -310,3 +315,67 @@
 
 </div>
 
+{% if permissions.can_write %}
+
+<div class="modal fade" id="ixf-preview-modal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-lg" role="document">
+    <div class="modal-content">
+      <div class="modal-header">
+        <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+        <h4 class="modal-title">{% trans "IX-F Import Preview" %}</h4>
+      </div>
+      <div class="modal-body">
+        <p>{% blocktrans %}The actual import will run once per day at 00:00Z - use this to preview the changes that will be done{% endblocktrans %}</p>
+        <div class="ixf-log" id="ixf-log">
+          <ul class="nav nav-tabs" role="tablist">
+            <li role="presentation" class="active">
+              <a href="#ixf-changes" aria-controls="add_fac" role="tab" data-toggle="tab">
+                {% trans "Changes" %}
+              </a>
+            </li>
+            <li role="presentation">
+              <a href="#ixf-errors" aria-controls="add_net" role="tab" data-toggle="tab">
+                {% trans "Errors" %}<span class="ixf-error-counter"></span>
+              </a>
+            </li>
+          </ul>
+          <div class="tab-content">
+            <div role="tabpanel" class="tab-pane active" id="ixf-changes">
+              <div class="ixf-headers row">
+                <div class="col-sm-1"><strong>action</strong></div>
+                <div class="col-sm-2"><strong>asn</strong></div>
+                <div class="col-sm-3"><strong>ipv4</strong></div>
+                <div class="col-sm-3"><strong>ipv6</strong></div>
+                <div class="col-sm-1"><strong>{% trans "speed" %}</strong></div>
+                <div class="col-sm-2"><strong>{% trans "routeserver" %}</strong></div>
+              </div>
+              <div class="ixf-result"></div>
+            </div>
+            <div role="tabpanel" class="tab-pane active ixf-errors" id="ixf-errors">
+              <div>{% blocktrans %}Sometimes we encounter some errors when parsing ix-f data, these will usually affect the import, you can view any of those errors below{% endblocktrans %}</div>
+              <div class="ixf-errors-list"></div>
+            </div>
+          </div>
+
+        </div>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+      </div>
+    </div><!-- /.modal-content -->
+  </div><!-- /.modal-dialog -->
+</div><!-- /.modal -->
+
+<script language="javascript" type="text/javascript">
+$(document).ready(function() {
+  $('button[data-ixf-preview]').click(function() {
+    var ixlanId = $(this).data("ixf-preview")
+    $('#ixf-preview-modal').modal("show");
+    PeeringDB.IXFPreview.request(ixlanId, $("#ixf-log"));
+  });
+});
+</script>
+
+{% endif %}
+
+

peeringdb_server/urls.py

@@ -19,6 +19,10 @@ from peeringdb_server.export_views import (
     AdvancedSearchExportView,
 )
 
+from peeringdb_server.import_views import (
+    view_import_ixlan_ixf_preview,
+)
+
 from django.views.i18n import JavaScriptCatalog
 
 # o
@@ -121,6 +125,8 @@ urlpatterns = [
         view_export_ixf_ixlan_members),
     url(r'^export/advanced-search/(?P<tag>[\w_]+)/(?P<fmt>[\w_-]+)$',
         AdvancedSearchExportView.as_view()),
+    url(r'^import/ixlan/(?P<ixlan_id>\d+)/ixf/preview$',
+        view_import_ixlan_ixf_preview),
     url(r'^$', view_index),
     url(r'^i18n/', include('django.conf.urls.i18n')),
     url('jsi18n/', JavaScriptCatalog.as_view(), name='javascript-catalog'),