mirror/peeringdb-peeringdb
1
0
Fork 0
mirror of https://github.com/peeringdb/peeringdb.git synced 2024-05-11 05:55:09 +00:00
Code Releases Activity

Gh 408 Review

Browse Source
This commit is contained in:
Stefan Pratter
2019-02-15 17:46:40 +00:00
parent b4ded90fa9
commit 3bf31a6de7
17 changed files with 1043 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config/facsimile/peeringdb.yaml
View File

@@ -22,6 +22,7 @@ misc:
view_verify_POST: "2/m"
view_username_retrieve_initiate: "2/m"
request_translation: "2/m"
view_import_ixlan_ixf_preview: "1/m"
api:
throtteling:
enabled: true

45
peeringdb_server/import_views.py Normal file
View File

@@ -0,0 +1,45 @@
import json
from django.http import JsonResponse, HttpResponse
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django_namespace_perms.util import has_perms
from ratelimit.decorators import ratelimit, is_ratelimited
from peeringdb_server import ixf
from peeringdb_server.models import IXLan
RATELIMITS = settings.RATELIMITS
@ratelimit(key="ip", rate=RATELIMITS["view_import_ixlan_ixf_preview"])
def view_import_ixlan_ixf_preview(request, ixlan_id):
# check if request was blocked by rate limiting
was_limited = getattr(request, "limited", False)
if was_limited:
return JsonResponse({
"non_field_errors": [
_("Please wait a bit before requesting " \
"another ixf import preview.")
]
}, status=400)
try:
ixlan = IXLan.objects.get(id=ixlan_id)
except IXLan.DoesNotExist:
return JsonResponse({
"non_field_errors": [_("Ixlan not found")]
}, status=404)
if not has_perms(request.user, ixlan, "update"):
return JsonResponse({
"non_field_errors": [_("Permission denied")]
}, status=403)
importer = ixf.Importer()
importer.update(ixlan, save=False)
return HttpResponse(
json.dumps(importer.log, indent=2), content_type="application/json")

415
peeringdb_server/ixf.py Normal file
View File

@@ -0,0 +1,415 @@
import json
import requests
import ipaddress
from django.db import transaction
from django.utils.translation import ugettext_lazy as _
import reversion
from peeringdb_server.models import (
IXLanIXFMemberImportAttempt,
IXLanIXFMemberImportLog,
Network,
NetworkIXLan,
)
class Importer(object):
allowed_member_types = ["peering", "ixp", "routeserver", "probono"]
allowed_states = ["active", "connected"]
def __init__(self):
self.reset()
def reset(self, ixlan=None, save=False):
self.reset_log()
self.netixlans = []
self.netixlans_deleted = []
self.ipaddresses = []
self.ixlan = ixlan
self.save = save
def fetch(self, url, timeout=5):
"""
Retrieves ixf member export data from the url
Will do a quick sanity check on the data
Returns dict containing the parsed data.
Arguments:
- url <str>
Keyword arguments:
- timeout <float>: max time to spend on request
"""
if not url:
return {"pdb_error": _("IXF import url not specified")}
try:
result = requests.get(url, timeout=timeout)
except Exception as exc:
return {"pdb_error": exc}
if result.status_code != 200:
# FIXME: log error somewhere
return None
try:
data = result.json()
except Exception as inst:
data = {"pdb_error": _("No JSON could be parsed")}
return data
return self.sanitize(data)
def sanitize(self, data):
"""
Takes ixf data dict and runs some sanitization on it
"""
invalid = None
vlan_list_found = False
# This fixes instances where ixps provide two separate entries for
# vlans in vlan_list for ipv4 and ipv6 (AMS-IX for example)
for member in data.get("member_list", []):
asn = member.get("asnum")
for conn in member.get("connection_list", []):
vlans = conn.get("vlan_list", [])
if not vlans:
continue
vlan_list_found = True
if len(vlans) == 2:
# if vlans[0].get("vlan_id") == vlans[1].get("vlan_id"):
keys = vlans[0].keys() + vlans[1].keys()
if keys.count("ipv4") == 1 and keys.count("ipv6") == 1:
vlans[0].update(**vlans[1])
conn["vlan_list"] = [vlans[0]]
if not vlan_list_found:
invalid = _("No entries in any of the vlan_list lists, aborting.")
data["pdb_error"] = invalid
return data
def update(self, ixlan, save=True, data=None, timeout=5):
"""
Sync netixlans under this ixlan from ixf member export json data (specs
can be found at https://github.com/euro-ix/json-schemas)
Arguments:
- ixlan (IXLan): ixlan object to update from ixf
Keyword Arguments:
- save (bool): commit changes to db
Returns:
- Tuple(success<bool>, netixlans<list>, log<list>)
"""
self.reset(ixlan=ixlan, save=save)
if data is None:
data = self.fetch(ixlan.ixf_ixp_member_list_url, timeout=timeout)
# bail if there has been any errors during sanitize() or fetch()
if data.get("pdb_error"):
self.log_error(data.get("pdb_error"), save=save)
return (False, [], [], self.log)
# bail if there are no active prefixes on the ixlan
if ixlan.ixpfx_set_active.count() == 0:
self.log_error(_("No prefixes defined on ixlan"), save=save)
return (False, [], [], self.log)
try:
# parse the ixf data
self.parse(data)
except KeyError as exc:
# any key erros mean that the data is invalid, log the error and
# bail (transactions are atomic and will be rolled back)
self.log_error("Internal Error 'KeyError': {}".format(exc),
save=save)
return (False, self.netixlans, [], self.log)
# process any netixlans that need to be deleted
self.process_deletions()
# archive the import so we can roll it back later if needed
self.archive()
if save:
self.save_log()
return (True, self.netixlans, self.netixlans_deleted, self.log)
@reversion.create_revision()
def process_deletions(self):
"""
Cycles all netixlans on the ixlan targeted by the importer and
will remove any that are no longer found in the ixf data by
their ip addresses
In order for a netixlan to be removed both it's ipv4 and ipv6 address
need to be gone from the ixf data
"""
for netixlan in self.ixlan.netixlan_set_active:
ipv4 = "{}-{}".format(netixlan.asn, netixlan.ipaddr4)
ipv6 = "{}-{}".format(netixlan.asn, netixlan.ipaddr6)
if ipv4 not in self.ipaddresses and ipv6 not in self.ipaddresses:
self.log_peer(netixlan.asn, "delete",
_("Ip addresses no longer in data"), netixlan)
self.netixlans_deleted.append(netixlan)
if self.save:
netixlan.delete()
@transaction.atomic()
def archive(self):
"""
Create the IXLanIXFMemberImportLog for this import
"""
if self.save and (self.netixlans or self.netixlans_deleted):
persist_log = IXLanIXFMemberImportLog.objects.create(
ixlan=self.ixlan)
for netixlan in self.netixlans + self.netixlans_deleted:
versions = reversion.models.Version.objects.get_for_object(
netixlan)
if len(versions) == 1:
version_before = None
else:
version_before = versions[1]
version_after = versions[0]
persist_log.entries.create(netixlan=netixlan,
version_before=version_before,
version_after=version_after)
def parse(self, data):
"""
Parse ixf data
Arguments:
- data <dict>: result from fetch()
"""
with transaction.atomic():
self.parse_members(data.get("member_list", []))
def parse_members(self, member_list):
"""
Parse the `member_list` section of the ixf schema
Arguments:
- member_list <list>
"""
for member in member_list:
# we only process members of certain types
member_type = member.get("member_type", "peering").lower()
if member_type in self.allowed_member_types:
# check that the as exists in pdb
asn = member["asnum"]
if Network.objects.filter(asn=asn).exists():
network = Network.objects.get(asn=asn)
if network.status != "ok":
self.log_peer(
asn, "skip",
_("Network status is '{}'").format(network.status))
continue
self.parse_connections(
member.get("connection_list", []), network, member)
else:
self.log_peer(asn, "skip",
_("Network does not exist in peeringdb"))
else:
self.log_peer(asn, "skip",
_("Invalid member type: {}").format(member_type))
def parse_connections(self, connection_list, network, member):
"""
Parse the 'connection_list' section of the ixf schema
Arguments:
- connection_list <list>
- network <Network>: pdb network instance
- member <dict>: row from ixf member_list
"""
asn = member["asnum"]
for connection in connection_list:
state = connection.get("state", "active").lower()
if state in self.allowed_states:
speed = self.parse_speed(connection.get("if_list", []))
self.parse_vlans(
connection.get("vlan_list", []), network, member,
connection, speed)
else:
self.log_peer(asn, "skip",
_("Invalid connection state: {}").format(state))
def parse_vlans(self, vlan_list, network, member, connection, speed):
"""
Parse the 'vlan_list' section of the ixf_schema
Arguments:
- vlan_list <list>
- network <Network>: pdb network instance
- member <dict>: row from ixf member_list
- connection <dict>: row from ixf connection_list
- speed <int>: interface speed
"""
asn = member["asnum"]
for lan in vlan_list:
ipv4_valid = False
ipv6_valid = False
ipv4 = lan.get("ipv4", {})
ipv6 = lan.get("ipv6", {})
# vlan entry has no ipaddresses set, log and skip
if not ipv4 and not ipv6:
self.log_error(_("Could not find ipv4 or 6 address in " \
"vlan_list entry for vlan_id {} (AS{})").format(
lan.get("vlan_id"), asn))
continue
ipv4_addr = ipv4.get("address")
ipv6_addr = ipv6.get("address")
# parse and validate the ipaddresses attached to the vlan
# append the ipaddresses to self.ipaddresses so we can
# later check them to see which netixlans need to be
# dropped during `process_deletions`
try:
if ipv4_addr:
self.ipaddresses.append("{}-{}".format(
asn, ipaddress.ip_address(unicode(ipv4_addr))))
if ipv6_addr:
self.ipaddresses.append("{}-{}".format(
asn, ipaddress.ip_address(unicode(ipv6_addr))))
except (ipaddress.AddressValueError, ValueError) as exc:
self.log_error(
_("Ip address error '{}' in vlan_list entry for vlan_id {}"
).format(exc, lan.get("vlan_id")))
continue
netixlan_info = NetworkIXLan(
ixlan=self.ixlan,
network=network,
ipaddr4=ipv4_addr,
ipaddr6=ipv6_addr,
speed=speed,
asn=asn,
is_rs_peer=(ipv4.get("routeserver", False) or \
ipv6.get("routeserver", False))
)
# after this point we either add or modify the netixlan, so
# now is a good time to check if the related network allows
# such updates, bail if not
if not network.allow_ixp_update:
self.log_peer(asn, "skip",
_("Network has disabled ixp updates"),
netixlan_info)
continue
# add / modify the netixlan
result = self.ixlan.add_netixlan(netixlan_info, save=self.save,
save_others=self.save)
if result["netixlan"] and result["changed"]:
self.netixlans.append(result["netixlan"])
if result["created"]:
action = "add"
else:
action = "modify"
self.log_peer(asn, action, "", result["netixlan"])
elif result["netixlan"]:
self.log_peer(asn, "noop", "", result["netixlan"])
elif result["log"]:
self.log_peer(asn, "skip", "\n".join(result["log"]),
netixlan_info)
def parse_speed(self, if_list):
"""
Parse speed from the 'if_list' section in the ixf data
Arguments:
- if_list <list>
Returns:
- speed <int>
"""
speed = 0
for iface in if_list:
try:
speed += int(iface.get("if_speed", 0))
except ValueError:
self.log_error(
_("Invalid speed value: {}").format(iface.get("if_speed")))
return speed
def save_log(self):
"""
Save the attempt log
"""
IXLanIXFMemberImportAttempt.objects.update_or_create(
ixlan=self.ixlan,
defaults={"info": "\n".join(json.dumps(self.log))})
def reset_log(self):
"""
Reset the attempt log
"""
self.log = {"data": [], "errors": []}
def log_peer(self, asn, action, reason, netixlan=None):
"""
log peer action in attempt log
Arguments:
- asn <int>
- action <str>: add | modify | delete | noop | skip
- reason <str>
Keyrword Arguments:
- netixlan <Netixlan>: if set, extra data will be added
to the log.
"""
peer = {
"ixlan_id": self.ixlan.id,
"asn": asn,
}
if netixlan:
peer.update({
"net_id": netixlan.network_id,
"ipaddr4": u"{}".format(netixlan.ipaddr4 or ""),
"ipaddr6": u"{}".format(netixlan.ipaddr6 or ""),
"speed": netixlan.speed,
"is_rs_peer": netixlan.is_rs_peer,
})
self.log["data"].append({
"peer": peer,
"action": action,
"reason": u"{}".format(reason),
})
def log_error(self, error, save=False):
"""
Append error to the attempt log
"""
self.log["errors"].append(u"{}".format(error))
if save:
self.save_log()

13
peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py
View File

@@ -1,9 +1,12 @@
import traceback
import json
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from peeringdb_server.models import (
IXLan, )
import traceback
from peeringdb_server import ixf
class Command(BaseCommand):
@@ -33,13 +36,11 @@ class Command(BaseCommand):
self.log("Fetching data for {} from {}".format(
ixlan, ixlan.ixf_ixp_member_list_url))
try:
json_data = ixlan.fetch_ixf_ixp_members_list()
importer = ixf.Importer()
self.log("Updating {}".format(ixlan))
with transaction.atomic():
success, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
json_data, save=self.commit)
for line in log:
self.log(line)
success, netixlans, netixlans_deleted, log = importer.update(ixlan, save=self.commit)
self.log(json.dumps(log))
self.log("Done: {} updated: {} deleted: {}".format(
success, len(netixlans), len(netixlans_deleted)))
except Exception as inst:

244
peeringdb_server/models.py
View File

@@ -1350,59 +1350,6 @@ class IXLan(pdb_models.IXLanBase):
def nsp_has_perms_PUT(self, user, request):
return validate_PUT_ownership(user, self, request.data, ["ix"])
def fetch_ixf_ixp_members_list(self):
"""
Retrieves ixf member export data from the url provided in
ixf_ixp_member_list_url.
Will do a quick sanity check on the data
Returns dict containing the parsed data.
"""
if not self.ixf_ixp_member_list_url:
return None
r = requests.get(self.ixf_ixp_member_list_url, timeout=5)
if r.status_code != 200:
# FIXME: log error somewhere
return None
try:
o = r.json()
except Exception as inst:
o = {"pdb_sanitize_error": _("No JSON could be parsed")}
return o
invalid = None
vlan_list_found = False
# sanitize
# This fixes instances where ixps provide two separate entries for
# vlans in vlan_list for ipv4 and ipv6 (AMS-IX for example)
for m in o.get("member_list", []):
asn = m.get("asnum")
for c in m.get("connection_list", []):
vlans = c.get("vlan_list", [])
if not vlans:
continue
vlan_list_found = True
if len(vlans) == 2:
# if vlans[0].get("vlan_id") == vlans[1].get("vlan_id"):
keys = vlans[0].keys() + vlans[1].keys()
if keys.count("ipv4") == 1 and keys.count("ipv6") == 1:
vlans[0].update(**vlans[1])
c["vlan_list"] = [vlans[0]]
print("Sanitized", c["vlan_list"])
if not vlan_list_found:
invalid = _("No entries in any of the vlan_list lists, aborting.")
o["pdb_sanitize_error"] = invalid
return o
@reversion.create_revision()
def add_netixlan(self, netixlan_info, save=True, save_others=True):
"""
@@ -1427,12 +1374,17 @@ class IXLan(pdb_models.IXLanBase):
log = []
changed = False
created = False
ipv4 = netixlan_info.ipaddr4
ipv6 = netixlan_info.ipaddr6
asn = netixlan_info.asn
ipv4_valid = False
ipv6_valid = False
def result(netixlan=None):
return { "netixlan": netixlan, "created": created,
"changed": changed, "log":log}
# check if either of the provided ip addresses are a fit for ANY of
# the prefixes in this ixlan
for pfx in self.ixpfx_set_active:
@@ -1446,24 +1398,24 @@ class IXLan(pdb_models.IXLanBase):
if (ipv4 and not ipv4_valid) or (ipv6 and not ipv6_valid):
log.append(
"Ip addresses ({}, {}) do not match any prefix " \
"on this ixlan. skipping.".format(ipv4, ipv6))
return (None, changed, log)
"on this ixlan".format(ipv4, ipv6))
return result()
# Next we check if an active netixlan with the ipaddress exists in ANOTHER lan, and bail
# if it does.
if ipv4 and \
NetworkIXLan.objects.filter(status="ok", ipaddr4=ipv4).exclude(ixlan=self).count() > 0:
log.append(
"Ip address {} already exists in another lan. skipping.".
"Ip address {} already exists in another lan".
format(ipv4))
return (None, changed, log)
return result()
if ipv6 and \
NetworkIXLan.objects.filter(status="ok", ipaddr6=ipv6).exclude(ixlan=self).count() > 0:
log.append(
"Ip address {} already exists in another lan. skipping.".
"Ip address {} already exists in another lan".
format(ipv6))
return (None, changed, log)
return result()
# now we need to figure out if the ipaddresses already exist in this ixlan,
# we need to check ipv4 and ipv6 separately as they might exist on different
@@ -1506,6 +1458,7 @@ class IXLan(pdb_models.IXLanBase):
# neither address exists, create a new netixlan object
netixlan = NetworkIXLan(ixlan=self, network=netixlan_info.network,
status="ok")
created = True
# now we sync the data to our determined netixlan instance
@@ -1576,182 +1529,13 @@ class IXLan(pdb_models.IXLanBase):
log.append("Validation Failure AS{} {} {}: {}".format(
netixlan.network.asn, netixlan.ipaddr4, netixlan.ipaddr6,
inst))
return (None, changed, log)
return result(None)
if save and changed:
netixlan.status = "ok"
netixlan.save()
return (netixlan, changed, log)
def update_from_ixf_ixp_member_list(self, json_data, save=True):
"""
Sync netixlans under this ixlan from ixf member export json data (specs
can be found at https://github.com/euro-ix/json-schemas)
Arguments:
- json_data (dict): parsed dict from ixf member export json (schema
described here https://github.com/euro-ix/json-schemas)
Keyword Arguments:
- save (bool): commit changes to db
Returns:
- Tuple(success<bool>, netixlans<list>, log<list>)
"""
log = []
netixlans = []
ipaddresses = []
ixlan = self
def persist_attempt():
print(log)
if save:
IXLanIXFMemberImportAttempt.objects.update_or_create(
ixlan=ixlan, defaults={
"info": "\n".join(log)
})
if json_data.get("pdb_sanitize_error"):
log.append(json_data.get("pdb_sanitize_error"))
persist_attempt()
return (False, [], [], log)
if self.ixpfx_set_active.count() == 0:
log.append("No prefixes defined on ixlan, skipping.")
persist_attempt()
return (False, [], [], log)
with transaction.atomic():
try:
member_list = json_data["member_list"]
for member in member_list:
# we only process members of certain types
if member.get("member_type", "peering") in \
["peering", "ixp", "routeserver", "probono"]:
# check that the as exists in pdb
asn = member["asnum"]
if Network.objects.filter(asn=asn).exists():
network = Network.objects.get(asn=asn)
if network.status != "ok":
log.append(
"Network with ASN {} status invalid, skipping..".
format(asn))
continue
for connection in member.get(
"connection_list", []):
if connection.get("state", "active") in [
"active", "connected"
]:
speed = 0
for iface in connection.get("if_list", []):
speed += iface.get("if_speed", 0)
for lan in connection.get("vlan_list", []):
ipv4_valid = False
ipv6_valid = False
ipv4 = lan.get("ipv4", {})
ipv6 = lan.get("ipv6", {})
if not ipv4 and not ipv6:
log.append("Could not find ipv4 or 6 address in " \
"vlan_list entry for vlan_id {}".format( \
lan.get("vlan_id")))
continue
ipv4_addr = ipv4.get("address")
ipv6_addr = ipv6.get("address")
try:
if ipv4_addr:
ipaddresses.append(
"{}-{}".format(
asn,
ipaddress.ip_address(
unicode(
ipv4_addr))))
if ipv6_addr:
ipaddresses.append(
"{}-{}".format(
asn,
ipaddress.ip_address(
unicode(
ipv6_addr))))
except (ipaddress.AddressValueError,
ValueError) as exc:
log.append(
"Invalid ipaddress '{}' for ASN {} - skipping".
format(exc, asn))
continue
if not network.allow_ixp_update:
log.append(
"Network with ASN {} has disabled ixp " \
"updates, skipping..".format(asn))
continue
netixlan, changed, _log = self.add_netixlan(
NetworkIXLan(
ixlan=self,
network=network,
ipaddr4=ipv4_addr,
ipaddr6=ipv6_addr,
speed=(speed or 0),
asn=asn,
is_rs_peer=(ipv4.get("routeserver", False) or \
ipv6.get("routeserver", False))
),
save=save,
save_others=save
)
log.extend(_log)
if netixlan and changed:
netixlans.append(netixlan)
else:
log.append(
"Network with asn {} does not exist in " \
"PeeringDB, skipping...".format(asn))
except KeyError, exc:
log.append("Internal Error 'KeyError': {}".format(exc))
persist_attempt()
return (False, netixlans, [], log)
with reversion.create_revision():
netixlans_deleted = []
for netixlan in self.netixlan_set_active:
ipv4 = "{}-{}".format(netixlan.asn, netixlan.ipaddr4)
ipv6 = "{}-{}".format(netixlan.asn, netixlan.ipaddr6)
if ipv4 not in ipaddresses and ipv6 not in ipaddresses:
log.append("Removing ASN {} IPV4 {} IPV6 {}".format(
netixlan.asn, netixlan.ipaddr4, netixlan.ipaddr6))
netixlans_deleted.append(netixlan)
if save:
netixlan.delete()
with transaction.atomic():
persist_attempt()
if save and (netixlans or netixlans_deleted):
persist_log = IXLanIXFMemberImportLog.objects.create(
ixlan=self)
for netixlan in netixlans + netixlans_deleted:
versions = reversion.models.Version.objects.get_for_object(
netixlan)
if len(versions) == 1:
version_before = None
else:
version_before = versions[1]
version_after = versions[0]
persist_log.entries.create(netixlan=netixlan,
version_before=version_before,
version_after=version_after)
return (True, netixlans, netixlans_deleted, log)
return result(netixlan)
class IXLanIXFMemberImportAttempt(models.Model):

111
peeringdb_server/static/peeringdb.js
View File

@@ -156,6 +156,117 @@ function moveCursorToEnd(el) {
}
}
PeeringDB.IXFPreview = {
/**
* Handle the IX-F import preview request and rendering
* to UI modal
*
* @class IXFPreview
* @namespace PeeringDB
*/
request : function(ixlanId, renderTo) {
/**
* request a preview for the ixlan with ixlanId
*
* @method request
* @param {Number} ixlanId
* @param {jQuery} renderTo - render to this element (needs to have
* the appropriate children elements to work, they are not
* created automatically)
*/
renderTo.find('.ixf-result').empty().
append($("<div>").addClass("center").text("... loading ..."));
renderTo.find('.ixf-error-counter').empty();
$.get('/import/ixlan/'+ixlanId+'/ixf/preview', function(result) {
this.render(result, renderTo);
}.bind(this)).error(function(result) {
if(result.responseJSON) {
this.render(result.responseJSON, renderTo);
} else {
this.render({"non_field_errors": ["HTTP error "+result.status]});
}
}.bind(this));
},
render : function(result, renderTo) {
/**
* Render preview result and errors
*
* @method render
* @param {Object} result - result as returned from the preview request
* @param {jQuery} renderTo
*
* Needs to have child divs with the following classes
*
*
* .ixf-errors-list: errors will be rendered to here
* .ixf-result: changes will be rendered to here
* .ixf-error-counter: will be updated with number of errors
*
*/
renderTo.find('.ixf-errors-list').empty()
renderTo.find('.ixf-result').empty()
this.render_errors((result.errors || []).concat(result.non_field_errors || []), renderTo.find('.ixf-errors-list'));
this.render_data(result.data || [], renderTo.find('.ixf-result'));
},
render_errors : function(errors, renderTo) {
/**
* Render the errors, called automatically by `render`
*
* @method render_errors
* @param {Array} errors
* @param {jQuery} renderTo
*/
var error, i;
if(!errors.length)
return;
$('.ixf-error-counter').text("("+errors.length+")");
for(i = 0; i < errors.length; i++) {
error = errors[i];
renderTo.append($('<div>').addClass("ixf-error").text(error));
}
},
render_data : function(data, renderTo) {
/**
* Renders the changes made by the ix-f import, called automatically
* by `render`
*
* @method render_data
* @param {Array} data
* @param {jQuery} renderTo
*/
var row, i;
for(i = 0; i < data.length; i++) {
row = data[i];
renderTo.append(
$('<div>').addClass("row ixf-row ixf-"+row.action).append(
$('<div>').addClass("col-sm-1").text(row.action),
$('<div>').addClass("col-sm-2").text("AS"+row.peer.asn),
$('<div>').addClass("col-sm-3").text(row.peer.ipaddr4 || "-"),
$('<div>').addClass("col-sm-3").text(row.peer.ipaddr6 || "-"),
$('<div>').addClass("col-sm-1").text(PeeringDB.pretty_speed(row.peer.speed)),
$('<div>').addClass("col-sm-2").text(row.peer.is_rs_peer?"yes":"no"),
$('<div>').addClass("col-sm-12 ixf-reason").text(row.reason)
)
);
}
}
}
PeeringDB.InlineSearch = {
init_search : function() {

67
peeringdb_server/static/site.css
View File

@@ -83,6 +83,12 @@ a.btn {
text-decoration: none;
}
.btn-inline {
position: absolute;
right: 5px;
top: 0px;
}
h5 {
font-weight: bold;
}
@@ -1035,3 +1041,64 @@ span.inline.delimited input {
div[name=ipaddr6],
div[data-edit-name=ipaddr6]
{ word-break: break-all; }
.ixf-log {
max-height: 400px;
overflow-y: scroll;
overflow-x: hidden;
}
.ixf-row {
padding: 2px;
border-bottom: 1px rgba(0,0,0,0.25) solid;
}
.ixf-skip {
background-color: #ffffd7;
}
.ixf-add {
background-color: #c5ffcd;
}
.ixf-modify {
background-color: #c5e9ff;
}
.ixf-delete {
background-color: #fbdbdb;
}
.ixf-noop {
background-color: #eaeaea;
}
.ixf-reason {
font-size: 11px;
}
.ixf-error {
color: red;
padding: 2px;
}
.ixf-errors-list {
}
.ixf-error-counter:not(:empty) {
color: red;
font-weight: bold;
}
.ixf-errors-overview {
background-color: #fbdbdb;
color: red;
text-align: center;
font-weight: bold;
cursor: pointer;
}
.ixf-errors-overview small {
color: #666;
}

69
peeringdb_server/templates/site/view_exchange_bottom.html
View File

@@ -82,6 +82,11 @@
data-edit-value="{{ x.ixf_ixp_member_list_url|fallback:"" }}"
data-edit-placeholder="{% trans "IXF Member Export URL" %}"
data-edit-name="ixf_ixp_member_list_url">{{ x.ixf_ixp_member_list_url|fallback:"" }}</div>
{% if permissions.can_write %}
<button class="btn btn-xs btn-inline btn-primary"
data-edit-toggled="view"
data-ixf-preview="{{ x.id }}">{% trans "Preview" %}</button>
{% endif %}
</div>
<div class="marg-top-60 pad-side-15"
data-edit-module="api_listing"
@@ -310,3 +315,67 @@
</div>
{% if permissions.can_write %}
<div class="modal fade" id="ixf-preview-modal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-lg" role="document">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
<h4 class="modal-title">{% trans "IX-F Import Preview" %}</h4>
</div>
<div class="modal-body">
<p>{% blocktrans %}The actual import will run once per day at 00:00Z - use this to preview the changes that will be done{% endblocktrans %}</p>
<div class="ixf-log" id="ixf-log">
<ul class="nav nav-tabs" role="tablist">
<li role="presentation" class="active">
<a href="#ixf-changes" aria-controls="add_fac" role="tab" data-toggle="tab">
{% trans "Changes" %}
</a>
</li>
<li role="presentation">
<a href="#ixf-errors" aria-controls="add_net" role="tab" data-toggle="tab">
{% trans "Errors" %}<span class="ixf-error-counter"></span>
</a>
</li>
</ul>
<div class="tab-content">
<div role="tabpanel" class="tab-pane active" id="ixf-changes">
<div class="ixf-headers row">
<div class="col-sm-1"><strong>action</strong></div>
<div class="col-sm-2"><strong>asn</strong></div>
<div class="col-sm-3"><strong>ipv4</strong></div>
<div class="col-sm-3"><strong>ipv6</strong></div>
<div class="col-sm-1"><strong>{% trans "speed" %}</strong></div>
<div class="col-sm-2"><strong>{% trans "routeserver" %}</strong></div>
</div>
<div class="ixf-result"></div>
</div>
<div role="tabpanel" class="tab-pane active ixf-errors" id="ixf-errors">
<div>{% blocktrans %}Sometimes we encounter some errors when parsing ix-f data, these will usually affect the import, you can view any of those errors below{% endblocktrans %}</div>
<div class="ixf-errors-list"></div>
</div>
</div>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div><!-- /.modal-content -->
</div><!-- /.modal-dialog -->
</div><!-- /.modal -->
<script language="javascript" type="text/javascript">
$(document).ready(function() {
$('button[data-ixf-preview]').click(function() {
var ixlanId = $(this).data("ixf-preview")
$('#ixf-preview-modal').modal("show");
PeeringDB.IXFPreview.request(ixlanId, $("#ixf-log"));
});
});
</script>
{% endif %}

6
peeringdb_server/urls.py
View File

@@ -19,6 +19,10 @@ from peeringdb_server.export_views import (
AdvancedSearchExportView,
)
from peeringdb_server.import_views import (
view_import_ixlan_ixf_preview,
)
from django.views.i18n import JavaScriptCatalog
# o
@@ -119,6 +123,8 @@ urlpatterns = [
view_export_ixf_ixlan_members),
url(r'^export/advanced-search/(?P<tag>[\w_]+)/(?P<fmt>[\w_-]+)$',
AdvancedSearchExportView.as_view()),
url(r'^import/ixlan/(?P<ixlan_id>\d+)/ixf/preview$',
view_import_ixlan_ixf_preview),
url(r'^$', view_index),
url(r'^i18n/', include('django.conf.urls.i18n')),
url('jsi18n/', JavaScriptCatalog.as_view(), name='javascript-catalog'),

64
tests/data/ixf/logs/skip_disabled_networks.json Normal file
View File

@@ -0,0 +1,64 @@
{
"errors": [
"Invalid speed value: fast",
"Could not find ipv4 or 6 address in vlan_list entry for vlan_id 1 (AS2906)",
"Ip address error 'u'195.69.146.error' does not appear to be an IPv4 or IPv6 address' in vlan_list entry for vlan_id 2"
],
"data": [
{
"peer": {
"ixlan_id": 1,
"asn": 2906
},
"action": "skip",
"reason": "Invalid connection state: disabled"
},
{
"peer": {
"is_rs_peer": true,
"ipaddr4": "195.69.146.250",
"net_id": 1,
"speed": 10000,
"ixlan_id": 1,
"asn": 2906,
"ipaddr6": "2001:7f8:1::a500:2906:2"
},
"action": "skip",
"reason": "Network has disabled ixp updates"
},
{
"peer": {
"is_rs_peer": true,
"ipaddr4": "195.69.147.250",
"net_id": 1,
"speed": 10000,
"ixlan_id": 1,
"asn": 2906,
"ipaddr6": "2001:7f8:1::a500:2906:1"
},
"action": "skip",
"reason": "Network has disabled ixp updates"
},
{
"peer": {
"ixlan_id": 1,
"asn": 2906
},
"action": "skip",
"reason": "Invalid member type: other"
},
{
"peer": {
"is_rs_peer": false,
"ipaddr4": "195.69.146.249",
"net_id": 1,
"speed": 10000,
"ixlan_id": 1,
"asn": 2906,
"ipaddr6": ""
},
"action": "delete",
"reason": "Ip addresses no longer in data"
}
]
}

51
tests/data/ixf/logs/skip_prefix_mismatch.json Normal file
View File

@@ -0,0 +1,51 @@
{
"errors": [
"Invalid speed value: fast",
"Could not find ipv4 or 6 address in vlan_list entry for vlan_id 1 (AS2906)",
"Ip address error 'u'195.69.146.error' does not appear to be an IPv4 or IPv6 address' in vlan_list entry for vlan_id 2"
],
"data": [
{
"peer": {
"ixlan_id": 2,
"asn": 2906
},
"action": "skip",
"reason": "Invalid connection state: disabled"
},
{
"peer": {
"is_rs_peer": true,
"ipaddr4": "195.69.146.250",
"net_id": 1,
"speed": 10000,
"ixlan_id": 2,
"asn": 2906,
"ipaddr6": "2001:7f8:1::a500:2906:2"
},
"action": "skip",
"reason": "Ip addresses (195.69.146.250, 2001:7f8:1::a500:2906:2) do not match any prefix on this ixlan"
},
{
"peer": {
"is_rs_peer": true,
"ipaddr4": "195.69.147.250",
"net_id": 1,
"speed": 10000,
"ixlan_id": 2,
"asn": 2906,
"ipaddr6": "2001:7f8:1::a500:2906:1"
},
"action": "skip",
"reason": "Ip addresses (195.69.147.250, 2001:7f8:1::a500:2906:1) do not match any prefix on this ixlan"
},
{
"peer": {
"ixlan_id": 2,
"asn": 2906
},
"action": "skip",
"reason": "Invalid member type: other"
}
]
}

64
tests/data/ixf/logs/update_01.json Normal file
View File

@@ -0,0 +1,64 @@
{
"errors": [
"Invalid speed value: fast",
"Could not find ipv4 or 6 address in vlan_list entry for vlan_id 1 (AS2906)",
"Ip address error 'u'195.69.146.error' does not appear to be an IPv4 or IPv6 address' in vlan_list entry for vlan_id 2"
],
"data": [
{
"peer": {
"ixlan_id": 1,
"asn": 2906
},
"action": "skip",
"reason": "Invalid connection state: disabled"
},
{
"peer": {
"is_rs_peer": true,
"ipaddr4": "195.69.146.250",
"net_id": 1,
"speed": 10000,
"ixlan_id": 1,
"asn": 2906,
"ipaddr6": "2001:7f8:1::a500:2906:2"
},
"action": "add",
"reason": ""
},
{
"peer": {
"is_rs_peer": true,
"ipaddr4": "195.69.147.250",
"net_id": 1,
"speed": 10000,
"ixlan_id": 1,
"asn": 2906,
"ipaddr6": "2001:7f8:1::a500:2906:1"
},
"action": "add",
"reason": ""
},
{
"peer": {
"ixlan_id": 1,
"asn": 2906
},
"action": "skip",
"reason": "Invalid member type: other"
},
{
"peer": {
"is_rs_peer": false,
"ipaddr4": "195.69.146.249",
"net_id": 1,
"speed": 10000,
"ixlan_id": 1,
"asn": 2906,
"ipaddr6": ""
},
"action": "delete",
"reason": "Ip addresses no longer in data"
}
]
}

18
tests/data/json_members_list/members.0.4.json
View File

@@ -19,6 +19,9 @@
"peering_policy_url": "https://www.netflix.com/openconnect/",
"member_since": "2009-02-04T00:00:00Z",
"connection_list": [
{
"state" : "disabled"
},
{
"ixp_id": 42,
"connected_since": "2009-02-04T00:00:00Z",
@@ -28,6 +31,9 @@
"switch_id": 0,
"if_speed": 10000,
"if_type": "LR4"
},
{
"if_speed": "fast"
}
],
"vlan_list": [
@@ -46,6 +52,15 @@
"max_prefix": 42,
"as_macro": "AS-NFLX-V6"
}
},
{
"vlan_id" : 1
},
{
"vlan_id" : 2,
"ipv4": {
"address": "195.69.146.error"
}
}
]
},
@@ -80,6 +95,9 @@
]
}
]
},
{
"member_type": "other"
}
]
}

19
tests/data/json_members_list/members.0.5.json
View File

@@ -19,6 +19,9 @@
"peering_policy_url": "https://www.netflix.com/openconnect/",
"member_since": "2009-02-04T00:00:00Z",
"connection_list": [
{
"state" : "disabled"
},
{
"ixp_id": 42,
"connected_since": "2009-02-04T00:00:00Z",
@@ -28,6 +31,9 @@
"switch_id": 0,
"if_speed": 10000,
"if_type": "LR4"
},
{
"if_speed": "fast"
}
],
"vlan_list": [
@@ -76,10 +82,23 @@
"max_prefix": 42,
"as_macro": "AS-NFLX-V6"
}
},
{
"vlan_id": 1
},
{
"vlan_id": 2,
"ipv4": {
"address": "195.69.146.error"
}
}
]
}
]
},
{
"member_type": "other"
}
]
}

12
tests/data/json_members_list/members.0.6.json
View File

@@ -31,6 +31,9 @@
"switch_id": 0,
"if_speed": 10000,
"if_type": "LR4"
},
{
"if_speed": "fast"
}
],
"vlan_list": [
@@ -54,6 +57,15 @@
"00:0a:95:9d:68:16"
]
}
},
{
"vlan_id": 1
},
{
"vlan_id": 2,
"ipv4": {
"address": "195.69.146.error"
}
}
]
},

3
tests/django_init.py
View File

@@ -163,5 +163,6 @@ settings.configure(
"view_request_ownership_POST": "3/m",
"request_login_POST": "10/m",
"view_verify_POST": "2/m",
"request_translation": "10/m"
"request_translation": "10/m",
"view_import_ixlan_ixf_preview": "1/m"
})

104
tests/test_ixf_member_import.py
View File

@@ -9,7 +9,13 @@ from django.test import TestCase, Client, RequestFactory
from peeringdb_server.models import (
Organization, Network, NetworkIXLan, IXLan, IXLanPrefix, InternetExchange,
IXLanIXFMemberImportAttempt, IXLanIXFMemberImportLog,
IXLanIXFMemberImportLogEntry)
IXLanIXFMemberImportLogEntry, User)
from peeringdb_server.import_views import (
view_import_ixlan_ixf_preview,
)
from peeringdb_server import ixf
from util import ClientCase
class JsonMembersListTestCase(TestCase):
@@ -104,6 +110,14 @@ class JsonMembersListTestCase(TestCase):
ipaddr4="195.69.146.249", ipaddr6=None, status="ok"),
]
def setUp(self):
self.ixf_importer = ixf.Importer()
def assertLog(self, log, expected):
path = os.path.join(os.path.dirname(__file__), "data", "ixf", "logs", "{}.json".format(expected))
with open(path, "r") as fh:
self.assertEqual(log, json.load(fh))
def test_update_from_ixf_ixp_member_list(self):
ixlan = self.entities["ixlan"][0]
n_deleted = self.entities["netixlan"][0]
@@ -112,11 +126,9 @@ class JsonMembersListTestCase(TestCase):
self.assertEqual(
unicode(n_deleted2.ipaddr6), u'2001:7f8:1::a500:2906:1')
self.assertEqual(ixlan.netixlan_set_active.count(), 1)
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
print(log)
self.assertEqual(len(log), 1)
self.assertLog(log, "update_01")
self.assertEqual(len(netixlans), 2)
self.assertEqual(len(netixlans_deleted), 1)
@@ -152,10 +164,9 @@ class JsonMembersListTestCase(TestCase):
against any of the prefixes that exist on the ixlan get skipped
"""
ixlan = self.entities["ixlan"][1]
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
self.assertEqual(len(log), 2)
self.assertLog(log, "skip_prefix_mismatch")
self.assertEqual(len(netixlans), 0)
def test_update_from_ixf_ixp_member_list_skip_missing_prefixes(self):
@@ -164,12 +175,11 @@ class JsonMembersListTestCase(TestCase):
ixlan that does not have any prefixes
"""
ixlan = self.entities["ixlan"][2]
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
self.assertEqual(len(netixlans), 0)
self.assertEqual(len(netixlans_deleted), 0)
self.assertEqual(log, [u'No prefixes defined on ixlan, skipping.'])
self.assertEqual(log["errors"], [u'No prefixes defined on ixlan'])
def test_update_from_ixf_ixp_member_list_skip_disabled_networks(self):
"""
@@ -180,10 +190,9 @@ class JsonMembersListTestCase(TestCase):
network = self.entities["net"][0]
network.allow_ixp_update = False
network.save()
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
self.assertEqual(len(log), 3)
self.assertLog(log, "skip_disabled_networks")
self.assertEqual(len(netixlans), 0)
for netixlan in network.netixlan_set_active.all():
@@ -192,8 +201,7 @@ class JsonMembersListTestCase(TestCase):
def test_update_from_ixf_ixp_member_list_logs(self):
ixlan = self.entities["ixlan"][0]
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
attempt_dt_1 = ixlan.ixf_import_attempt.updated
@@ -221,8 +229,7 @@ class JsonMembersListTestCase(TestCase):
time.sleep(0.1)
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
ixlan.ixf_import_attempt.refresh_from_db()
attempt_dt_2 = ixlan.ixf_import_attempt.updated
@@ -243,8 +250,7 @@ class JsonMembersListTestCase(TestCase):
def test_rollback(self):
ixlan = self.entities["ixlan"][0]
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
for entry in ixlan.ixf_import_log_set.last().entries.all():
self.assertEqual(entry.rollback_status(), 0)
@@ -262,8 +268,7 @@ class JsonMembersListTestCase(TestCase):
def test_rollback_avoid_ipaddress_conflict(self):
ixlan = self.entities["ixlan"][0]
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
self.assertEqual(len(netixlans_deleted), 1)
@@ -294,8 +299,7 @@ class JsonMembersListTestCase(TestCase):
# import the data
ixlan = self.entities["ixlan"][0]
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
# request the view and compare it agaisnt expected data
c = Client()
@@ -325,8 +329,7 @@ class JsonMembersListTestCase(TestCase):
# import the data
ixlan = self.entities["ixlan"][0]
r, netixlans, netixlans_deleted, log = ixlan.update_from_ixf_ixp_member_list(
self.json_data)
r, netixlans, netixlans_deleted, log = self.ixf_importer.update(ixlan, data=self.json_data)
# request the view and compare it agaisnt expected data
c = Client()
@@ -354,3 +357,52 @@ class JsonMembersListTestCase_V05(JsonMembersListTestCase):
class JsonMembersListTestCase_V04(JsonMembersListTestCase):
version = "0.4"
class TestImportPreview(ClientCase):
"""
Test the ixf import preview
"""
@classmethod
def setUpTestData(cls):
super(TestImportPreview, cls).setUpTestData()
cls.org = Organization.objects.create(name="Test Org", status="ok")
cls.ix = InternetExchange.objects.create(name="Test IX", status="ok", org=cls.org)
cls.ixlan = IXLan.objects.create(status="ok", ix=cls.ix)
cls.admin_user = User.objects.create_user("admin","admin@localhost","admin")
cls.org.admin_usergroup.user_set.add(cls.admin_user)
def test_import_preview(self):
request = RequestFactory().get("/import/ixlan/{}/ixf/preview/".format(self.ixlan.id))
request.user = self.admin_user
response = view_import_ixlan_ixf_preview(request, self.ixlan.id)
assert response.status_code == 200
assert json.loads(response.content)["errors"] == ["IXF import url not specified"]
def test_import_preview_fail_ratelimit(self):
request = RequestFactory().get("/import/ixlan/{}/ixf/preview/".format(self.ixlan.id))
request.user = self.admin_user
response = view_import_ixlan_ixf_preview(request, self.ixlan.id)
assert response.status_code == 200
response = view_import_ixlan_ixf_preview(request, self.ixlan.id)
assert response.status_code == 400
def test_import_preview_fail_permission(self):
request = RequestFactory().get("/import/ixlan/{}/ixf/preview/".format(self.ixlan.id))
request.user = self.guest_user
response = view_import_ixlan_ixf_preview(request, self.ixlan.id)
assert response.status_code == 403