From 87d5df3c22d57c8940a6450f00b20bc7702d6891 Mon Sep 17 00:00:00 2001 From: Stefan Pratter Date: Mon, 22 Jan 2024 21:20:13 +0200 Subject: [PATCH] Support 202311 fixes 3 (#1510) * 1280 fixes * cleanup and fixes for rir_status update, also add --reset * linting * comment * fix tests and some adjustments * fix mock according to new rir_status values * more rir_status update fixes and safety checks fix tests --- peeringdb_server/deskpro.py | 31 ++- peeringdb_server/inet.py | 5 +- .../management/commands/pdb_api_test.py | 17 +- .../management/commands/pdb_rir_status.py | 185 ++++++++++++++++-- peeringdb_server/mock.py | 3 +- peeringdb_server/serializers.py | 14 +- peeringdb_server/signals.py | 13 +- .../email/notify-pdb-admin-rir-status.txt | 6 +- peeringdb_server/verified_update/const.py | 1 + peeringdb_server/verified_update/views.py | 6 + tests/test_generate_test_data.py | 2 + tests/test_rir_status.py | 9 +- tests/test_verified_update.py | 6 +- 13 files changed, 242 insertions(+), 56 deletions(-) diff --git a/peeringdb_server/deskpro.py b/peeringdb_server/deskpro.py index fa3485f0..d99ba212 100644 --- a/peeringdb_server/deskpro.py +++ b/peeringdb_server/deskpro.py @@ -114,16 +114,35 @@ def ticket_queue_prefixauto_approve(user, ix, prefix): ) -def ticket_queue_rir_status_update(net): +def ticket_queue_rir_status_updates(networks: list, threshold: int, date: datetime): """ - Queue deskro ticket creation for prefix automation action: create. + Queue a single deskpro ticket creation for multiple network RIR status + updates and raise an exception if the threshold is exceeded. + + :param networks: List of network objects that have updated RIR status. + :param threshold: Threshold number for network count to raise exception. + :param date: Date of RIR status update. """ + if not threshold: + threshold = 100 + + if len(networks) > threshold: + raise Exception( + f"RIR status update threshold of {threshold} exceeded. Manual review required." + ) + + ticket_body = loader.get_template("email/notify-pdb-admin-rir-status.txt").render( + { + "networks": networks, + "date": date, + "days_until_deletion": settings.KEEP_RIR_STATUS, + } + ) + ticket_queue_email_only( - f"[RIR_STATUS] RIR status updated on Network '{net.name}' in '{net.rir_status_updated}'", - loader.get_template("email/notify-pdb-admin-rir-status.txt").render( - {"ix": net} - ), + "[RIR_STATUS] RIR status degradation updates", + ticket_body, None, ) diff --git a/peeringdb_server/inet.py b/peeringdb_server/inet.py index 410caf11..45cc35ff 100644 --- a/peeringdb_server/inet.py +++ b/peeringdb_server/inet.py @@ -153,10 +153,11 @@ def rir_status_is_ok(rir_status: str) -> bool: """ return rir_status in [ # actual rir statuses returned via rdap rir assigment check + # that we consider 'ok' "assigned", "allocated", - # status peeringdb sets on creation of network, indicating rir status - # is pending + # peeringdb initial status (after creation or undeletion) + # should be treated as `ok` "pending", ] diff --git a/peeringdb_server/management/commands/pdb_api_test.py b/peeringdb_server/management/commands/pdb_api_test.py index 557da1de..7d96b391 100644 --- a/peeringdb_server/management/commands/pdb_api_test.py +++ b/peeringdb_server/management/commands/pdb_api_test.py @@ -2121,7 +2121,7 @@ class TestJSON(unittest.TestCase): "net", data, ) - assert r_data.get("rir_status") == "pending" + assert r_data.get("rir_status") == "ok" ########################################################################## @@ -2197,24 +2197,19 @@ class TestJSON(unittest.TestCase): ########################################################################## - def test_org_admin_002_PUT_net_rir_status(self): + def test_org_admin_002_GET_net_rir_status(self): net = SHARED["net_rw_ok"] now = timezone.now() - net.rir_status = "ok" + net.rir_status = "assigned" net.rir_status_updated = now net.save() - self.assert_update( - self.db_org_admin, - "net", - SHARED["net_rw_ok"].id, - {"name": self.make_name("TesT")}, - ) + api_data = self.db_org_admin.get("net", SHARED["net_rw_ok"].id) net.refresh_from_db() - - assert net.rir_status == "ok" + assert api_data[0]["rir_status"] == "ok" + assert net.rir_status == "assigned" assert net.rir_status_updated == now ########################################################################## diff --git a/peeringdb_server/management/commands/pdb_rir_status.py b/peeringdb_server/management/commands/pdb_rir_status.py index 56e83acb..9b68f16e 100644 --- a/peeringdb_server/management/commands/pdb_rir_status.py +++ b/peeringdb_server/management/commands/pdb_rir_status.py @@ -1,11 +1,11 @@ import reversion from django.conf import settings as pdb_settings -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.db import transaction from django.utils import timezone from rdap.assignment import RIRAssignmentLookup -from peeringdb_server.deskpro import ticket_queue_rir_status_update +from peeringdb_server.deskpro import ticket_queue_rir_status_updates from peeringdb_server.inet import rir_status_is_ok from peeringdb_server.models import Network @@ -22,6 +22,21 @@ class Command(BaseCommand): type=int, help="Only check networks with a RIR status older than this age", ) + parser.add_argument( + "--reset", action="store_true", help="Reset all RIR status." + ) + parser.add_argument( + "-o", + "--output", + help="Output file for --reset, will contain all networks with bad RIR status", + ) + parser.add_argument( + "-M", + "--max-networks-from-good-to-bad", + type=int, + default=100, + help="Maximum amount of networks going from good to bad. If exceeded, script will exit with error and a human should look at. This is to help prevent mass flagging of networks because of bad RIR data. Default to 100.", + ) def log(self, msg): if self.commit: @@ -29,17 +44,77 @@ class Command(BaseCommand): else: self.stdout.write(f"[pretend] {msg}") + def reset(self): + """ + Reset RIR status for all networks, setting their + rir_status to the value read from the RIR allocation data. + + This will also set the rir_status_updated field to now. + + Running this essentially resets the rir status state, resetting + timelines for stale network deletion. + + This will NOT send any deskpro notifications. + + If the --output option is provided, all networks with a bad + RIR status will be written to the file. + """ + + # reset all rir status + rir = RIRAssignmentLookup() + rir.load_data( + pdb_settings.RIR_ALLOCATION_DATA_PATH, + pdb_settings.RIR_ALLOCATION_DATA_CACHE_DAYS, + ) + self.log("Resetting all RIR status") + + qset = Network.objects.filter(status="ok") + now = timezone.now() + + bad_networks = [] + + batch_save = [] + for net in qset: + net.rir_status = rir.get_status(net.asn) + net.rir_status_updated = now + + if not rir_status_is_ok(net.rir_status): + bad_networks.append(net) + + batch_save.append(net) + + self.log(f"Saving {len(batch_save)} networks") + if self.commit: + Network.objects.bulk_update( + batch_save, ["rir_status", "rir_status_updated"] + ) + + if self.output: + with open(self.output, "w") as f: + for net in bad_networks: + f.write(f"AS{net.asn} {net.rir_status}\n") + self.log(f"{len(bad_networks)} bad networks written to {self.output}") + @transaction.atomic() @reversion.create_revision() def handle(self, *args, **options): + # dont update network `updated` field on rir status changes + + Network._meta.get_field("updated").auto_now = False + self.commit = options.get("commit") self.asn = options.get("asn") self.max_age = options.get("max_age") self.limit = options.get("limit") + self.output = options.get("output") + self.max_networks_from_good_to_bad = options.get( + "max_networks_from_good_to_bad" + ) - # dont update network `updated` field on rir status changes - - Network._meta.get_field("updated").auto_now = False + reset = options.get("reset") + if reset: + self.reset() + return now = timezone.now() networks = None @@ -70,22 +145,60 @@ class Command(BaseCommand): reversion.set_comment("pdb_rir_status script") + batch_save = [] + + # tracks networks going from ok rir status to not ok + # rir status, networks in this list will be notified to the AC via + # deskpro API + networks_from_good_to_bad = [] + for net in networks: new_rir_status = rir.get_status(net.asn) old_rir_status = net.rir_status - if rir_status_is_ok(old_rir_status) and old_rir_status != new_rir_status: - self.log(f"{net.name} ({net.asn}) RIR status: {new_rir_status}") - if self.commit: - net.rir_status = new_rir_status - net.rir_status_updated = now - net.save(update_fields=["rir_status", "rir_status_updated"]) - elif not rir_status_is_ok(old_rir_status): - if rir_status_is_ok(new_rir_status): + if not new_rir_status: + + # missing from rir data, we use None to indicate + # never checked, so we set this to missing to + # indicate that we have checked and it is missing + + new_rir_status = "missing" + + if rir_status_is_ok(old_rir_status): + + # old status was ok (assigned) or never set + + if not rir_status_is_ok(new_rir_status): + + # new status is not ok (!assigned) or old status was never set + self.log(f"{net.name} ({net.asn}) RIR status: {new_rir_status}") + net.rir_status_updated = now + net.rir_status = new_rir_status + networks_from_good_to_bad.append(net) + if self.commit: - ticket_queue_rir_status_update(net) - else: + batch_save.append(net) + + elif old_rir_status != new_rir_status: + + # both old and new status are ok (assigned), but they are different + net.rir_status_updated = now + net.rir_status = new_rir_status + + if self.commit: + batch_save.append(net) + + elif not rir_status_is_ok(new_rir_status): + + # new status is not ok + + if not rir_status_is_ok(old_rir_status): + + # old status was not ok (!assigned) + # check if we should delete the network, because + # it has been unassigned for too long + notok_since = now - net.rir_status_updated if ( notok_since.total_seconds() @@ -95,4 +208,44 @@ class Command(BaseCommand): f"{net.name} ({net.asn}) has been RIR unassigned for too long, deleting" ) if self.commit: - net.delete() + # call with force so delete isn't blocked by DOTF protection + net.delete(force=True) + else: + days = pdb_settings.KEEP_RIR_STATUS - notok_since.days + self.log( + f"Network still unassigned, {days} days left to deletion" + ) + + if networks_from_good_to_bad: + + # if we have too many networks going from good to bad + # we exit with an error to prevent mass flagging of networks due to bad + # RIR data + + num_networks_from_good_to_bad = len(networks_from_good_to_bad) + self.log( + f"Found {num_networks_from_good_to_bad} networks going from good to bad" + ) + if num_networks_from_good_to_bad > self.max_networks_from_good_to_bad: + raise CommandError( + f"Too many networks going from good to bad ({num_networks_from_good_to_bad}), exiting to prevent mass flagging of networks. Please check manually. You can specify a threshold for this via the -M option." + ) + + # batch update + + if self.commit: + + if networks_from_good_to_bad: + + # notify admin comittee on networks changed from ok RIR status to not ok RIR status + + ticket_queue_rir_status_updates( + networks_from_good_to_bad, + self.max_networks_from_good_to_bad, + now, + ) + + self.log(f"Applying rir status updates for {len(batch_save)} networks") + Network.objects.bulk_update( + batch_save, ["rir_status", "rir_status_updated"] + ) diff --git a/peeringdb_server/mock.py b/peeringdb_server/mock.py index 555af7fe..adc73f57 100644 --- a/peeringdb_server/mock.py +++ b/peeringdb_server/mock.py @@ -150,6 +150,7 @@ class Mock: # with the same name as the field name else: data[field.name] = getattr(self, field.name)(data, reftag=reftag) + obj = model(**data) obj.clean() obj.save() @@ -337,7 +338,7 @@ class Mock: return None def rir_status(self, data, reftag=None): - return "ok" + return "assigned" def rir_status_updated(self, data, reftag=None): return None diff --git a/peeringdb_server/serializers.py b/peeringdb_server/serializers.py index 477b8817..6eae75da 100644 --- a/peeringdb_server/serializers.py +++ b/peeringdb_server/serializers.py @@ -57,6 +57,7 @@ from peeringdb_server.inet import ( RdapLookup, get_prefix_protocol, rdap_pretty_error_message, + rir_status_is_ok, ) from peeringdb_server.models import ( QUEUE_ENABLED, @@ -2520,7 +2521,7 @@ class NetworkSerializer(ModelSerializer): required=False, allow_null=True, allow_blank=True, default="" ) - rir_status = serializers.CharField(default="", read_only=True) + rir_status = serializers.SerializerMethodField() rir_status_updated = RemoveMillisecondsDateTimeField(default=None, read_only=True) social_media = SocialMediaSerializer(required=False, many=True) @@ -2746,6 +2747,17 @@ class NetworkSerializer(ModelSerializer): def get_org(self, inst): return self.sub_serializer(OrganizationSerializer, inst.org) + def get_rir_status(self, inst): + """ + Normalized RIR status for network + """ + # backwards compatibility for rir status on the api + # `ok` if ok + # None if not ok + if rir_status_is_ok(inst.rir_status): + return "ok" + return None + def create(self, validated_data): request = self._context.get("request") request.user diff --git a/peeringdb_server/signals.py b/peeringdb_server/signals.py index 96948dd5..d67eefa7 100644 --- a/peeringdb_server/signals.py +++ b/peeringdb_server/signals.py @@ -708,7 +708,6 @@ def rir_status_initial(sender, instance=None, **kwargs): Anytime a network is saved: if an ASN is added, set rir_status="ok" and set `=created - if an ASN is deleted (manually), set rir_status="notok" and set rir_status_updated=updated if an ASN is re-added, set rir_status="ok" and set rir_status_updated=updated """ @@ -717,7 +716,7 @@ def rir_status_initial(sender, instance=None, **kwargs): created = not instance.id - # if an ASN is added, set rir_status="ok" and set rir_status_updated=created + # if an ASN is added, set rir_status=ok (reset) and set rir_status_updated=created if created: instance.rir_status = "pending" @@ -726,14 +725,8 @@ def rir_status_initial(sender, instance=None, **kwargs): else: old = Network.objects.get(id=instance.id) - # if an ASN is deleted (manually), set rir_status="notok" and set rir_status_updated=updated + # if an ASN is re-added, set rir_status=ok (reset) and set rir_status_updated=updated - if old.status == "ok" and instance.status == "deleted": - instance.rir_status = "" - instance.rir_status_updated = timezone.now() - - # if an ASN is re-added, set rir_status="ok" and set rir_status_updated=updated - - elif old.status == "deleted" and instance.status == "ok": + if old.status == "deleted" and instance.status == "ok": instance.rir_status = "pending" instance.rir_status_updated = timezone.now() diff --git a/peeringdb_server/templates/email/notify-pdb-admin-rir-status.txt b/peeringdb_server/templates/email/notify-pdb-admin-rir-status.txt index f6816532..ef4ff810 100644 --- a/peeringdb_server/templates/email/notify-pdb-admin-rir-status.txt +++ b/peeringdb_server/templates/email/notify-pdb-admin-rir-status.txt @@ -1,4 +1,8 @@ {% load i18n %} {% language 'en' %} - Network {{ net.name }} ({{ net.id }}) RIR Status updated to '{{ net.rir_status }}' at {{ net.rir_status_updated }} +For the following networks RIR assignment status changed from ok to not ok on {{ date }}: +They are flagged for automatic deletion in {{ days_until_deletion }} days unless their RIR status changes back to ok. + +{% for net in networks %}- AS{{ net.asn }} - {{ net.name }} - RIR Status: {{ net.rir_status }} +{% endfor %} {% endlanguage %} diff --git a/peeringdb_server/verified_update/const.py b/peeringdb_server/verified_update/const.py index e01c76c6..734958ad 100644 --- a/peeringdb_server/verified_update/const.py +++ b/peeringdb_server/verified_update/const.py @@ -4,6 +4,7 @@ SUPPORTED_FIELDS = { "route_server", "looking_glass", "info_type", + "info_types", "info_prefixes4", "info_prefixes6", "info_traffic", diff --git a/peeringdb_server/verified_update/views.py b/peeringdb_server/verified_update/views.py index e9f1f5e2..8238d1fc 100644 --- a/peeringdb_server/verified_update/views.py +++ b/peeringdb_server/verified_update/views.py @@ -108,6 +108,12 @@ def view_verified_update(request): invalid_permissions[obj] = obj._meta.verbose_name continue + # backwards compatibility for network info_type + if "info_type" in data: + info_type = data.pop("info_type") + if "info_types" not in data: + data["info_types"] = [info_type] + update_data = {} diff = {} update_data.update({"ref_tag": ref_tag, "obj_id": obj_id}) diff --git a/tests/test_generate_test_data.py b/tests/test_generate_test_data.py index 26a7e46f..cdc0ed95 100644 --- a/tests/test_generate_test_data.py +++ b/tests/test_generate_test_data.py @@ -10,4 +10,6 @@ class TestGenerateTestData(TestCase): for reftag, cls in list(REFTAG_MAP.items()): self.assertGreater(cls.objects.count(), 0) for instance in cls.objects.all(): + if hasattr(instance, "rir_status"): + print("RIR STATUS", instance.rir_status, type(instance.rir_status)) instance.full_clean() diff --git a/tests/test_rir_status.py b/tests/test_rir_status.py index c3936020..2b635a36 100644 --- a/tests/test_rir_status.py +++ b/tests/test_rir_status.py @@ -9,12 +9,6 @@ def test_network_auto_initial_rir_status(): """ Tests `Anytime` network update logic for RIR status handling laid out in https://github.com/peeringdb/peeringdb/issues/1280 - - Anytime a network is saved: - - if an ASN is added, set rir_status="ok" and set rir_status_updated=created - if an ASN is deleted (manually), set rir_status="notok" and set rir_status_updated=updated - if an ASN is re-added, set rir_status="ok" and set rir_status_updated=updated """ org = Organization.objects.create(name="Test org", status="ok") @@ -22,9 +16,10 @@ def test_network_auto_initial_rir_status(): assert net.rir_status == "pending" + net.rir_status = "missing" net.delete() - assert net.rir_status == "" + assert net.rir_status == "missing" net.status = "ok" net.save() diff --git a/tests/test_verified_update.py b/tests/test_verified_update.py index 49109231..d4e089bf 100644 --- a/tests/test_verified_update.py +++ b/tests/test_verified_update.py @@ -47,7 +47,11 @@ class VerifiedUpdateTestCase(TestCase): { "ref_tag": "net", "obj_id": self.net.id, - "data": {"info_prefixes4": 4, "info_prefixes6": 6}, + "data": { + "info_prefixes4": 4, + "info_prefixes6": 6, + "info_type": "Content", + }, }, { "ref_tag": "poc",