1
0
mirror of https://github.com/peeringdb/peeringdb.git synced 2024-05-11 05:55:09 +00:00

July updates (#762)

* Change label from primary ASN to ASN

* Raise validation error when trying to update ASN

* first steps for dotf importer procotol (#697)

* migrations (#697)

* Add translation to error meessage

* Make ASN readonly in table

* Add test now that ASN should not be able to update

* Set fac.rencode to '' for all entries and make it readonly in serializer

* Add unique constraints to network ixlan ip addresses

* Add migration to null out duplicate ipaddresses for deleted netixlans

* Add unique constraints to network ixlan ip addresses

* Add migration to null out duplicate ipaddresses for deleted netixlans

* remove old migrations (#697)

* fix netixlan ipaddr dedupe migration (#268)
add netixlan ipaddr unique constraint migration (#268)

* ixf_member_data migrations (#697)

* fix table name (#697)

* importer protocol (#697)

* fix netixlan ipaddr dedupe migration (#268)
add netixlan ipaddr unique constraint migration (#268)

* ixf proposed changes notifications (#697)

* Delete repeated query

* Add a test to show rencode is readonly

* Blank out rencode when mocking data

* Remove validator now that constraint exists

* Add back unique field validator w Check Deleted true

* conflict resolving (#697)

* UniqueFieldValidator raise error with code "unique" (#268)

* conflict resolution (#697)

* Add fixme comment to tests

* conflict resolution (#697)

* Remove now invalid undelete tests

* UniqueFieldValidator raise error with code "unique" (#268)

* delete admin tools for duplicate ip addresses

* Make migration to delete duplicateipnetworkixlan

* Add ixlan-ixpfx status matching validation, add corresponding test

* delete redundant checking in test

* resolve conflict ui (#697)

* fix migrations hierarchy

* squash migrations for ixf member data

* clean up preview and post-mortem tools

* remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling

* only include the ix-f data url in notifications to admincom (#697)

* resolve on --skip-import (#697)

* ac conflict resolution (#697)

* Define more accurately the incompatible statuses for ixlan and ixpfx

* Add another status test

* Preventing disrupting changes (#697)

* fix tests (#697)

* Stop allow_ixp_update from being write only and add a global stat for automated networks

* Add tests for global stats that appear in footer

* Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass

* test for protected entities (#697)

* admincom conflict resolution refine readonly fields (#697)
network notifications only if the problem is actually actionable by the network (#697)

* ixp / ac notifcation when ix-f source cannot be parsed (#697)
fix issue with ixlan prefix protection (#697)

* migrations (#697)

* code documentation (#697)

* ux tweaks (#697)

* UX tweaks (#697)

* Fix typo

* fix netixlan returned in IXFMemberData.apply when adding a new one (#697)

* fix import log incosistencies (#697)

* Add IXFMemberData to test

* Update test data

* Add protocol tests

* Add tests for views

* always persist changes to remote data on set_conflict (#697)

* More tests

* always persist changes to remote data on set_conflict (#697)

* suggest-add test

* net_present_at_ix should check status (#697)

* Add more protocol tests

* Edit language of some tests

* django-peeringdb to 2.1.1
relock pipfile, pin django-ratelimit to <3 as it breaks stuff

* Add net_count_ixf field to ix object (#683)

* Add the IX-F Member Export URL to the ixlan API endpoint (#249)

* Lock some objects from being deleted by the owner (#696)

* regenerate api docs (#249)

* always persist changes to remote data on set_add and set_update (#697)

* IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field

* always persist changes to remote data on set_add and set_update (#697)

* Fix suggest-add tests

* IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field

* IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field

* fix issue with deletion when ixfmemberdata for entry existed previously (#697)

* fix test_suggest_delete_local_ixf_no_flag (#697 tests)

* fix issue with deletion when ixfmemberdata for entry existed previously (#697)

* invalid ips get logged and notified to the ix via notify_error (#697)

* Fix more tests

* issue with previous_data when running without save (#697)
properly track speed errors (#697)

* reset errors on ixfmemberdata that go into pending_save (#697)

* add remote_data to admin view (#697)

* fix error reset inconsistency (#697)

* Refine invalid data tests

* remove debug output

* for notifications to ac include contact points for net and ix in the message (#697)

* settings to toggle ix-f tickets / emails (#697)

* allow turning off ix-f notifications for net and ix separately (#697)

* add jsonschema test

* Add idempotent tests to updater

* remove old ixf member tests

* Invalid data tests when ixp_updates are enabled

* fix speed error validation (#697)

* fix issue with rollback (#697)

* fix migration hierarchy

* fix ixfmemberdata _email

* django-peeringdb to 2.2 and relock

* add ixf rollback tests

* ixf email notifications off by default

* black formatted

* pyupgrade

Co-authored-by: egfrank <egfrank@20c.com>
Co-authored-by: Stefan Pratter <stefan@20c.com>
This commit is contained in:
Matt Griswold
2020-07-15 02:07:01 -05:00
committed by GitHub
parent 462c92a288
commit ea55c4dc38
108 changed files with 8908 additions and 2541 deletions

View File

@@ -1,5 +1,6 @@
import json
import re
import datetime
import requests
import ipaddress
@@ -7,6 +8,8 @@ import ipaddress
from django.db import transaction
from django.core.cache import cache
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ValidationError
from django.conf import settings
import reversion
@@ -16,10 +19,25 @@ from peeringdb_server.models import (
IXLanIXFMemberImportLogEntry,
Network,
NetworkIXLan,
IXFMemberData,
)
REASON_ENTRY_GONE_FROM_REMOTE = _(
"The entry for (asn and IPv4 and IPv6) does not exist "
"in the exchange's IX-F data as a singular member connection"
)
REASON_NEW_ENTRY = _(
"The entry for (asn and IPv4 and IPv6) does not exist "
"in PeeringDB as a singular network -> ix connection"
)
REASON_VALUES_CHANGED = _(
"Data differences between PeeringDB and the exchange's IX-F data"
)
class Importer(object):
class Importer:
allowed_member_types = [
"peering",
@@ -43,14 +61,20 @@ class Importer(object):
def reset(self, ixlan=None, save=False, asn=None):
self.reset_log()
self.netixlans = []
self.netixlans_deleted = []
self.ipaddresses = []
self.ixf_ids = []
self.actions_taken = {
"add": [],
"delete": [],
"modify": [],
"noop": [],
}
self.pending_save = []
self.asns = []
self.archive_info = {}
self.ixlan = ixlan
self.save = save
self.asn = asn
self.now = datetime.datetime.now(datetime.timezone.utc)
self.invalid_ip_errors = []
def fetch(self, url, timeout=5):
"""
@@ -76,7 +100,7 @@ class Importer(object):
return {"pdb_error": exc}
if result.status_code != 200:
return {"pdb_error": "Got HTTP status {}".format(result.status_code)}
return {"pdb_error": f"Got HTTP status {result.status_code}"}
try:
data = result.json()
@@ -102,7 +126,7 @@ class Importer(object):
url <str>
"""
return "IXF-CACHE-{}".format(url)
return f"IXF-CACHE-{url}"
def fetch_cached(self, url):
"""
@@ -184,16 +208,23 @@ class Importer(object):
# bail if there has been any errors during sanitize() or fetch()
if data.get("pdb_error"):
self.notify_error(data.get("pdb_error"))
self.log_error(data.get("pdb_error"), save=save)
return (False, [], [], self.log)
return False
# null ix-f error note on ixlan if it had error'd before
if self.ixlan.ixf_ixp_import_error:
self.ixlan.ixf_ixp_import_error = None
self.ixlan.save()
# bail if there are no active prefixes on the ixlan
if ixlan.ixpfx_set_active.count() == 0:
self.log_error(_("No prefixes defined on ixlan"), save=save)
return (False, [], [], self.log)
return False
if self.skip_import:
return (True, [], [], self.log)
self.cleanup_ixf_member_data()
return True
try:
# parse the ixf data
@@ -201,19 +232,74 @@ class Importer(object):
except KeyError as exc:
# any key erros mean that the data is invalid, log the error and
# bail (transactions are atomic and will be rolled back)
self.log_error("Internal Error 'KeyError': {}".format(exc), save=save)
return (False, self.netixlans, [], self.log)
self.log_error(f"Internal Error 'KeyError': {exc}", save=save)
return False
# process any netixlans that need to be deleted
self.process_deletions()
# process creation of new netixlans and updates
# of existing netixlans. This needs to happen
# after process_deletions in order to avoid potential
# ip conflicts
self.process_saves()
self.cleanup_ixf_member_data()
# archive the import so we can roll it back later if needed
self.archive()
if self.invalid_ip_errors:
self.notify_error("\n".join(self.invalid_ip_errors))
if save:
# update exchange's ixf fields
self.update_ix()
self.save_log()
return (True, self.netixlans, self.netixlans_deleted, self.log)
return True
@reversion.create_revision()
def update_ix(self):
"""
Will see if any data was changed during this import
and update the exchange's ixf_last_import timestamp
if so
Also will set the ixf_net_count value if it has changed
from before
"""
ix = self.ixlan.ix
save_ix = False
ixf_member_data_changed = IXFMemberData.objects.filter(
updated__gte=self.now, ixlan=self.ixlan
).exists()
netixlan_data_changed = NetworkIXLan.objects.filter(
updated__gte=self.now, ixlan=self.ixlan
).exists()
if ixf_member_data_changed or netixlan_data_changed:
ix.ixf_last_import = self.now
save_ix = True
ixf_net_count = len(self.pending_save)
if ixf_net_count != ix.ixf_net_count:
ix.ixf_net_count = ixf_net_count
save_ix = True
if save_ix:
ix.save()
@reversion.create_revision()
def process_saves(self):
for ixf_member in self.pending_save:
self.apply_add_or_update(ixf_member)
@reversion.create_revision()
def process_deletions(self):
@@ -235,46 +321,50 @@ class Importer(object):
netixlan_qset = netixlan_qset.filter(asn=self.asn)
for netixlan in netixlan_qset:
ipv4 = "{}-{}".format(netixlan.asn, netixlan.ipaddr4)
ipv6 = "{}-{}".format(netixlan.asn, netixlan.ipaddr6)
if netixlan.asn not in self.asns:
self.log_peer(
netixlan.asn, "delete", _("ASN no longer in data"), netixlan
)
self.netixlans_deleted.append(netixlan)
if self.save:
netixlan.delete()
elif ipv4 not in self.ipaddresses and ipv6 not in self.ipaddresses:
self.log_peer(
if netixlan.ixf_id not in self.ixf_ids:
ixf_member_data = IXFMemberData.instantiate(
netixlan.asn,
"delete",
_(
"Ip addresses no longer exist in validated data or are "
"no longer with this asn"
),
netixlan,
netixlan.ipaddr4,
netixlan.ipaddr6,
netixlan.ixlan,
data={},
)
self.netixlans_deleted.append(netixlan)
if self.save:
netixlan.delete()
elif (netixlan.ipaddr4 and ipv4 not in self.ipaddresses) or (
netixlan.ipaddr6 and ipv6 not in self.ipaddresses
):
if not netixlan.network.allow_ixp_update:
self.log_peer(
netixlan.asn,
"delete",
_(
"At least one ipaddress mismatched and "
"network has disabled updates"
),
netixlan,
if netixlan.network.allow_ixp_update:
self.log_apply(
ixf_member_data.apply(save=self.save),
reason=REASON_ENTRY_GONE_FROM_REMOTE,
)
self.netixlans_deleted.append(netixlan)
if self.save:
netixlan.delete()
else:
ixf_member_data.set_remove(
save=self.save, reason=REASON_ENTRY_GONE_FROM_REMOTE
)
self.log_ixf_member_data(ixf_member_data)
def cleanup_ixf_member_data(self):
# clean up old ix-f memeber data objects
for ixf_member in IXFMemberData.objects.filter(ixlan=self.ixlan):
# proposed deletion got fulfilled
if ixf_member.action == "delete":
if ixf_member.netixlan.status == "deleted":
ixf_member.set_resolved()
# noop means the ask has been fulfilled but the
# ixf member data entry has not been set to resolved yet
elif ixf_member.action == "noop":
ixf_member.set_resolved()
# proposed change / addition is now gone from
# ix-f data
elif not self.skip_import and ixf_member.ixf_id not in self.ixf_ids:
if ixf_member.action in ["add", "modify"]:
ixf_member.set_resolved()
@transaction.atomic()
def archive(self):
@@ -282,20 +372,31 @@ class Importer(object):
Create the IXLanIXFMemberImportLog for this import
"""
if self.save and (self.netixlans or self.netixlans_deleted):
persist_log = IXLanIXFMemberImportLog.objects.create(ixlan=self.ixlan)
for netixlan in self.netixlans + self.netixlans_deleted:
if not self.save:
return
persist_log = IXLanIXFMemberImportLog.objects.create(ixlan=self.ixlan)
for action in ["delete", "modify", "add"]:
for info in self.actions_taken[action]:
netixlan = info["netixlan"]
version_before = info["version"]
versions = reversion.models.Version.objects.get_for_object(netixlan)
if len(versions) == 1:
version_before = None
if version_before:
versions = versions.filter(id__gt=version_before.id)
version_after = versions.last()
else:
version_before = versions[1]
version_after = versions[0]
info = self.archive_info.get(netixlan.id, {})
version_after = versions.first()
if not version_after:
continue
persist_log.entries.create(
netixlan=netixlan,
version_before=version_before,
action=info.get("action"),
action=action,
reason=info.get("reason"),
version_after=version_after,
)
@@ -367,6 +468,7 @@ class Importer(object):
asn = member["asnum"]
for connection in connection_list:
self.connection_errors = []
state = connection.get("state", "active").lower()
if state in self.allowed_states:
@@ -414,23 +516,30 @@ class Importer(object):
ipv6_addr = ipv6.get("address")
# parse and validate the ipaddresses attached to the vlan
# append the ipaddresses to self.ipaddresses so we can
# later check them to see which netixlans need to be
# append a unqiue ixf identifier to self.ixf_ids
#
# identifier is a tuple of (asn, ip4, ip6)
#
# we will later check them to see which netixlans need to be
# dropped during `process_deletions`
try:
ixf_id = [asn]
if ipv4_addr:
self.ipaddresses.append(
"{}-{}".format(
asn, ipaddress.ip_address(u"{}".format(ipv4_addr))
)
)
ixf_id.append(ipaddress.ip_address(f"{ipv4_addr}"))
else:
ixf_id.append(None)
if ipv6_addr:
self.ipaddresses.append(
"{}-{}".format(
asn, ipaddress.ip_address(u"{}".format(ipv6_addr))
)
)
ixf_id.append(ipaddress.ip_address(f"{ipv6_addr}"))
else:
ixf_id.append(None)
ixf_id = tuple(ixf_id)
self.ixf_ids.append(ixf_id)
except (ipaddress.AddressValueError, ValueError) as exc:
self.invalid_ip_errors.append(f"{exc}")
self.log_error(
_("Ip address error '{}' in vlan_list entry for vlan_id {}").format(
exc, lan.get("vlan_id")
@@ -438,18 +547,6 @@ class Importer(object):
)
continue
netixlan_info = NetworkIXLan(
ixlan=self.ixlan,
network=network,
ipaddr4=ipv4_addr,
ipaddr6=ipv6_addr,
speed=speed,
asn=asn,
is_rs_peer=(
ipv4.get("routeserver", False) or ipv6.get("routeserver", False)
),
)
if not self.save and (
not self.ixlan.test_ipv4_address(ipv4_addr)
and not self.ixlan.test_ipv6_address(ipv6_addr)
@@ -458,46 +555,33 @@ class Importer(object):
# not at the ixlan if they dont match the prefix
continue
# if connection state is inactive we won't create or update
if connection.get("state", "active") == "inactive":
self.log_peer(
asn,
"noop",
_("Connection is currently marked as inactive"),
netixlan_info,
)
continue
operational = False
else:
operational = True
# after this point we either add or modify the netixlan, so
# now is a good time to check if the related network allows
# such updates, bail if not
if not network.allow_ixp_update:
self.log_peer(
asn, "noop", _("Network has disabled ixp updates"), netixlan_info
)
continue
# add / modify the netixlan
result = self.ixlan.add_netixlan(
netixlan_info, save=self.save, save_others=self.save
is_rs_peer = ipv4.get("routeserver", False) or ipv6.get(
"routeserver", False
)
if result["netixlan"] and result["changed"]:
self.netixlans.append(result["netixlan"])
if result["created"]:
action = "add"
reason = _("New ip-address")
else:
action = "modify"
reason = _("Fields changed: {}").format(
", ".join(result.get("changed"))
)
ixf_member_data = IXFMemberData.instantiate(
asn,
ipv4_addr,
ipv6_addr,
speed=speed,
operational=operational,
is_rs_peer=is_rs_peer,
data=json.dumps(member),
ixlan=self.ixlan,
save=self.save,
)
self.log_peer(asn, action, reason, result["netixlan"])
elif result["netixlan"]:
self.log_peer(asn, "noop", _("No changes"), result["netixlan"])
elif result["log"]:
self.log_peer(asn, "ignore", "\n".join(result["log"]), netixlan_info)
if self.connection_errors:
ixf_member_data.error = "\n".join(self.connection_errors)
else:
ixf_member_data.error = ixf_member_data.previous_error
self.pending_save.append(ixf_member_data)
def parse_speed(self, if_list):
"""
@@ -514,11 +598,68 @@ class Importer(object):
try:
speed += int(iface.get("if_speed", 0))
except ValueError:
self.log_error(
_("Invalid speed value: {}").format(iface.get("if_speed"))
)
log_msg = _("Invalid speed value: {}").format(iface.get("if_speed"))
self.log_error(log_msg)
self.connection_errors.append(log_msg)
return speed
def apply_add_or_update(self, ixf_member_data):
if ixf_member_data.netixlan_exists:
# importer-protocol: netixlan exists
if not ixf_member_data.changes:
# importer-protocol: no changes
self.resolve(ixf_member_data)
else:
# importer-protocol: data changes
self.apply_update(ixf_member_data)
else:
# importer-protocol: netixlan does not exist
self.apply_add(ixf_member_data)
def resolve(self, ixf_member_data):
ixf_member_data.set_resolved(save=self.save)
def apply_update(self, ixf_member_data):
changed_fields = ", ".join(ixf_member_data.changes.keys())
reason = f"{REASON_VALUES_CHANGED}: {changed_fields}"
if ixf_member_data.net.allow_ixp_update:
try:
self.log_apply(ixf_member_data.apply(save=self.save), reason=reason)
except ValidationError as exc:
ixf_member_data.set_conflict(error=exc, save=self.save)
else:
ixf_member_data.set_update(
save=self.save, reason=reason,
)
self.log_ixf_member_data(ixf_member_data)
def apply_add(self, ixf_member_data):
if ixf_member_data.net.allow_ixp_update:
try:
self.log_apply(
ixf_member_data.apply(save=self.save), reason=REASON_NEW_ENTRY
)
except ValidationError as exc:
ixf_member_data.set_conflict(error=exc, save=self.save)
else:
ixf_member_data.set_add(save=self.save, reason=REASON_NEW_ENTRY)
self.log_ixf_member_data(ixf_member_data)
def save_log(self):
"""
Save the attempt log
@@ -533,6 +674,31 @@ class Importer(object):
"""
self.log = {"data": [], "errors": []}
def log_apply(self, apply_result, reason=""):
netixlan = apply_result["netixlan"]
self.actions_taken[apply_result["action"]].append(
{
"netixlan": netixlan,
"version": reversion.models.Version.objects.get_for_object(
netixlan
).first(),
"reason": reason,
}
)
return self.log_peer(
netixlan.asn, apply_result["action"], reason, netixlan=netixlan
)
def log_ixf_member_data(self, ixf_member_data):
return self.log_peer(
ixf_member_data.net.asn,
f"suggest-{ixf_member_data.action}",
ixf_member_data.reason,
netixlan=ixf_member_data,
)
def log_peer(self, asn, action, reason, netixlan=None):
"""
log peer action in attempt log
@@ -554,36 +720,61 @@ class Importer(object):
}
if netixlan:
if hasattr(netixlan, "network_id"):
net_id = netixlan.network_id
else:
net_id = netixlan.net.id
peer.update(
{
"net_id": netixlan.network_id,
"net_id": net_id,
"ipaddr4": "{}".format(netixlan.ipaddr4 or ""),
"ipaddr6": "{}".format(netixlan.ipaddr6 or ""),
"speed": netixlan.speed,
"is_rs_peer": netixlan.is_rs_peer,
"operational": netixlan.operational,
}
)
if netixlan.id:
self.archive_info[netixlan.id] = {
"action": action,
"reason": "{}".format(reason),
}
self.log["data"].append(
{"peer": peer, "action": action, "reason": "{}".format(reason),}
{"peer": peer, "action": action, "reason": f"{reason}",}
)
def notify_error(self, error):
now = datetime.datetime.now(datetime.timezone.utc)
notified = self.ixlan.ixf_ixp_import_error_notified
prev_error = self.ixlan.ixf_ixp_import_error
if notified:
diff = (now - notified).total_seconds() / 3600
if diff < settings.IXF_PARSE_ERROR_NOTIFICATION_PERIOD:
return
self.ixlan.ixf_ixp_import_error_notified = now
self.ixlan.ixf_ixp_import_error = error
self.ixlan.save()
ixf_member_data = IXFMemberData(ixlan=self.ixlan, asn=0)
ixf_member_data._notify(
"email/notify-ixf-source-error.txt",
"Could not process IX-F Data",
context={"error": error, "dt": now},
save=False,
ix=True,
ac=True,
)
def log_error(self, error, save=False):
"""
Append error to the attempt log
"""
self.log["errors"].append("{}".format(error))
self.log["errors"].append(f"{error}")
if save:
self.save_log()
class PostMortem(object):
class PostMortem:
"""
Generate postmortem report for ix-f import
@@ -649,7 +840,7 @@ class PostMortem(object):
qset = IXLanIXFMemberImportLogEntry.objects.filter(netixlan__asn=self.asn)
qset = qset.exclude(action__isnull=True)
qset = qset.order_by("-log__created")
qset = qset.order_by("-log__created", "-id")
qset = qset.select_related("log", "netixlan", "log__ixlan", "log__ixlan__ix")
for entry in qset[:limit]: