mirror/peeringdb-peeringdb
1
0
Fork 0
mirror of https://github.com/peeringdb/peeringdb.git synced 2024-05-11 05:55:09 +00:00
Code Releases Activity

Support 202311 fixes 3 (#1510)

Browse Source 
* 1280 fixes

* cleanup and fixes for rir_status update, also add --reset

* linting

* comment

* fix tests and some adjustments

* fix mock according to new rir_status values

* more rir_status update fixes and safety checks
fix tests
This commit is contained in:
Stefan Pratter
2024-01-22 21:20:13 +02:00
committed by GitHub
parent b879343da8
commit 87d5df3c22
13 changed files with 242 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
peeringdb_server/deskpro.py
View File

@@ -114,16 +114,35 @@ def ticket_queue_prefixauto_approve(user, ix, prefix):
)
def ticket_queue_rir_status_update(net):
def ticket_queue_rir_status_updates(networks: list, threshold: int, date: datetime):
"""
Queue deskro ticket creation for prefix automation action: create.
Queue a single deskpro ticket creation for multiple network RIR status
updates and raise an exception if the threshold is exceeded.
:param networks: List of network objects that have updated RIR status.
:param threshold: Threshold number for network count to raise exception.
:param date: Date of RIR status update.
"""
if not threshold:
threshold = 100
if len(networks) > threshold:
raise Exception(
f"RIR status update threshold of {threshold} exceeded. Manual review required."
)
ticket_body = loader.get_template("email/notify-pdb-admin-rir-status.txt").render(
{
"networks": networks,
"date": date,
"days_until_deletion": settings.KEEP_RIR_STATUS,
}
)
ticket_queue_email_only(
f"[RIR_STATUS] RIR status updated on Network '{net.name}' in '{net.rir_status_updated}'",
loader.get_template("email/notify-pdb-admin-rir-status.txt").render(
{"ix": net}
),
"[RIR_STATUS] RIR status degradation updates",
ticket_body,
None,
)

5
peeringdb_server/inet.py
View File

@@ -153,10 +153,11 @@ def rir_status_is_ok(rir_status: str) -> bool:
"""
return rir_status in [
# actual rir statuses returned via rdap rir assigment check
# that we consider 'ok'
"assigned",
"allocated",
# status peeringdb sets on creation of network, indicating rir status
# is pending
# peeringdb initial status (after creation or undeletion)
# should be treated as `ok`
"pending",
]

17
peeringdb_server/management/commands/pdb_api_test.py
View File

@@ -2121,7 +2121,7 @@ class TestJSON(unittest.TestCase):
"net",
data,
)
assert r_data.get("rir_status") == "pending"
assert r_data.get("rir_status") == "ok"
##########################################################################
@@ -2197,24 +2197,19 @@ class TestJSON(unittest.TestCase):
##########################################################################
def test_org_admin_002_PUT_net_rir_status(self):
def test_org_admin_002_GET_net_rir_status(self):
net = SHARED["net_rw_ok"]
now = timezone.now()
net.rir_status = "ok"
net.rir_status = "assigned"
net.rir_status_updated = now
net.save()
self.assert_update(
self.db_org_admin,
"net",
SHARED["net_rw_ok"].id,
{"name": self.make_name("TesT")},
)
api_data = self.db_org_admin.get("net", SHARED["net_rw_ok"].id)
net.refresh_from_db()
assert net.rir_status == "ok"
assert api_data[0]["rir_status"] == "ok"
assert net.rir_status == "assigned"
assert net.rir_status_updated == now
##########################################################################

185
peeringdb_server/management/commands/pdb_rir_status.py
View File

@@ -1,11 +1,11 @@
import reversion
from django.conf import settings as pdb_settings
from django.core.management.base import BaseCommand
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.utils import timezone
from rdap.assignment import RIRAssignmentLookup
from peeringdb_server.deskpro import ticket_queue_rir_status_update
from peeringdb_server.deskpro import ticket_queue_rir_status_updates
from peeringdb_server.inet import rir_status_is_ok
from peeringdb_server.models import Network
@@ -22,6 +22,21 @@ class Command(BaseCommand):
type=int,
help="Only check networks with a RIR status older than this age",
)
parser.add_argument(
"--reset", action="store_true", help="Reset all RIR status."
)
parser.add_argument(
"-o",
"--output",
help="Output file for --reset, will contain all networks with bad RIR status",
)
parser.add_argument(
"-M",
"--max-networks-from-good-to-bad",
type=int,
default=100,
help="Maximum amount of networks going from good to bad. If exceeded, script will exit with error and a human should look at. This is to help prevent mass flagging of networks because of bad RIR data. Default to 100.",
)
def log(self, msg):
if self.commit:
@@ -29,17 +44,77 @@ class Command(BaseCommand):
else:
self.stdout.write(f"[pretend] {msg}")
def reset(self):
"""
Reset RIR status for all networks, setting their
rir_status to the value read from the RIR allocation data.
This will also set the rir_status_updated field to now.
Running this essentially resets the rir status state, resetting
timelines for stale network deletion.
This will NOT send any deskpro notifications.
If the --output option is provided, all networks with a bad
RIR status will be written to the file.
"""
# reset all rir status
rir = RIRAssignmentLookup()
rir.load_data(
pdb_settings.RIR_ALLOCATION_DATA_PATH,
pdb_settings.RIR_ALLOCATION_DATA_CACHE_DAYS,
)
self.log("Resetting all RIR status")
qset = Network.objects.filter(status="ok")
now = timezone.now()
bad_networks = []
batch_save = []
for net in qset:
net.rir_status = rir.get_status(net.asn)
net.rir_status_updated = now
if not rir_status_is_ok(net.rir_status):
bad_networks.append(net)
batch_save.append(net)
self.log(f"Saving {len(batch_save)} networks")
if self.commit:
Network.objects.bulk_update(
batch_save, ["rir_status", "rir_status_updated"]
)
if self.output:
with open(self.output, "w") as f:
for net in bad_networks:
f.write(f"AS{net.asn} {net.rir_status}\n")
self.log(f"{len(bad_networks)} bad networks written to {self.output}")
@transaction.atomic()
@reversion.create_revision()
def handle(self, *args, **options):
# dont update network `updated` field on rir status changes
Network._meta.get_field("updated").auto_now = False
self.commit = options.get("commit")
self.asn = options.get("asn")
self.max_age = options.get("max_age")
self.limit = options.get("limit")
self.output = options.get("output")
self.max_networks_from_good_to_bad = options.get(
"max_networks_from_good_to_bad"
)
# dont update network `updated` field on rir status changes
Network._meta.get_field("updated").auto_now = False
reset = options.get("reset")
if reset:
self.reset()
return
now = timezone.now()
networks = None
@@ -70,22 +145,60 @@ class Command(BaseCommand):
reversion.set_comment("pdb_rir_status script")
batch_save = []
# tracks networks going from ok rir status to not ok
# rir status, networks in this list will be notified to the AC via
# deskpro API
networks_from_good_to_bad = []
for net in networks:
new_rir_status = rir.get_status(net.asn)
old_rir_status = net.rir_status
if rir_status_is_ok(old_rir_status) and old_rir_status != new_rir_status:
self.log(f"{net.name} ({net.asn}) RIR status: {new_rir_status}")
if self.commit:
net.rir_status = new_rir_status
net.rir_status_updated = now
net.save(update_fields=["rir_status", "rir_status_updated"])
elif not rir_status_is_ok(old_rir_status):
if rir_status_is_ok(new_rir_status):
if not new_rir_status:
# missing from rir data, we use None to indicate
# never checked, so we set this to missing to
# indicate that we have checked and it is missing
new_rir_status = "missing"
if rir_status_is_ok(old_rir_status):
# old status was ok (assigned) or never set
if not rir_status_is_ok(new_rir_status):
# new status is not ok (!assigned) or old status was never set
self.log(f"{net.name} ({net.asn}) RIR status: {new_rir_status}")
net.rir_status_updated = now
net.rir_status = new_rir_status
networks_from_good_to_bad.append(net)
if self.commit:
ticket_queue_rir_status_update(net)
else:
batch_save.append(net)
elif old_rir_status != new_rir_status:
# both old and new status are ok (assigned), but they are different
net.rir_status_updated = now
net.rir_status = new_rir_status
if self.commit:
batch_save.append(net)
elif not rir_status_is_ok(new_rir_status):
# new status is not ok
if not rir_status_is_ok(old_rir_status):
# old status was not ok (!assigned)
# check if we should delete the network, because
# it has been unassigned for too long
notok_since = now - net.rir_status_updated
if (
notok_since.total_seconds()
@@ -95,4 +208,44 @@ class Command(BaseCommand):
f"{net.name} ({net.asn}) has been RIR unassigned for too long, deleting"
)
if self.commit:
net.delete()
# call with force so delete isn't blocked by DOTF protection
net.delete(force=True)
else:
days = pdb_settings.KEEP_RIR_STATUS - notok_since.days
self.log(
f"Network still unassigned, {days} days left to deletion"
)
if networks_from_good_to_bad:
# if we have too many networks going from good to bad
# we exit with an error to prevent mass flagging of networks due to bad
# RIR data
num_networks_from_good_to_bad = len(networks_from_good_to_bad)
self.log(
f"Found {num_networks_from_good_to_bad} networks going from good to bad"
)
if num_networks_from_good_to_bad > self.max_networks_from_good_to_bad:
raise CommandError(
f"Too many networks going from good to bad ({num_networks_from_good_to_bad}), exiting to prevent mass flagging of networks. Please check manually. You can specify a threshold for this via the -M option."
)
# batch update
if self.commit:
if networks_from_good_to_bad:
# notify admin comittee on networks changed from ok RIR status to not ok RIR status
ticket_queue_rir_status_updates(
networks_from_good_to_bad,
self.max_networks_from_good_to_bad,
now,
)
self.log(f"Applying rir status updates for {len(batch_save)} networks")
Network.objects.bulk_update(
batch_save, ["rir_status", "rir_status_updated"]
)

3
peeringdb_server/mock.py
View File

@@ -150,6 +150,7 @@ class Mock:
# with the same name as the field name
else:
data[field.name] = getattr(self, field.name)(data, reftag=reftag)
obj = model(**data)
obj.clean()
obj.save()
@@ -337,7 +338,7 @@ class Mock:
return None
def rir_status(self, data, reftag=None):
return "ok"
return "assigned"
def rir_status_updated(self, data, reftag=None):
return None

14
peeringdb_server/serializers.py
View File

@@ -57,6 +57,7 @@ from peeringdb_server.inet import (
RdapLookup,
get_prefix_protocol,
rdap_pretty_error_message,
rir_status_is_ok,
)
from peeringdb_server.models import (
QUEUE_ENABLED,
@@ -2520,7 +2521,7 @@ class NetworkSerializer(ModelSerializer):
required=False, allow_null=True, allow_blank=True, default=""
)
rir_status = serializers.CharField(default="", read_only=True)
rir_status = serializers.SerializerMethodField()
rir_status_updated = RemoveMillisecondsDateTimeField(default=None, read_only=True)
social_media = SocialMediaSerializer(required=False, many=True)
@@ -2746,6 +2747,17 @@ class NetworkSerializer(ModelSerializer):
def get_org(self, inst):
return self.sub_serializer(OrganizationSerializer, inst.org)
def get_rir_status(self, inst):
"""
Normalized RIR status for network
"""
# backwards compatibility for rir status on the api
# `ok` if ok
# None if not ok
if rir_status_is_ok(inst.rir_status):
return "ok"
return None
def create(self, validated_data):
request = self._context.get("request")
request.user

13
peeringdb_server/signals.py
View File

@@ -708,7 +708,6 @@ def rir_status_initial(sender, instance=None, **kwargs):
Anytime a network is saved:
if an ASN is added, set rir_status="ok" and set `=created
if an ASN is deleted (manually), set rir_status="notok" and set rir_status_updated=updated
if an ASN is re-added, set rir_status="ok" and set rir_status_updated=updated
"""
@@ -717,7 +716,7 @@ def rir_status_initial(sender, instance=None, **kwargs):
created = not instance.id
# if an ASN is added, set rir_status="ok" and set rir_status_updated=created
# if an ASN is added, set rir_status=ok (reset) and set rir_status_updated=created
if created:
instance.rir_status = "pending"
@@ -726,14 +725,8 @@ def rir_status_initial(sender, instance=None, **kwargs):
else:
old = Network.objects.get(id=instance.id)
# if an ASN is deleted (manually), set rir_status="notok" and set rir_status_updated=updated
# if an ASN is re-added, set rir_status=ok (reset) and set rir_status_updated=updated
if old.status == "ok" and instance.status == "deleted":
instance.rir_status = ""
instance.rir_status_updated = timezone.now()
# if an ASN is re-added, set rir_status="ok" and set rir_status_updated=updated
elif old.status == "deleted" and instance.status == "ok":
if old.status == "deleted" and instance.status == "ok":
instance.rir_status = "pending"
instance.rir_status_updated = timezone.now()

6
peeringdb_server/templates/email/notify-pdb-admin-rir-status.txt
View File

@@ -1,4 +1,8 @@
{% load i18n %}
{% language 'en' %}
Network {{ net.name }} ({{ net.id }}) RIR Status updated to '{{ net.rir_status }}' at {{ net.rir_status_updated }}
For the following networks RIR assignment status changed from ok to not ok on {{ date }}:
They are flagged for automatic deletion in {{ days_until_deletion }} days unless their RIR status changes back to ok.
{% for net in networks %}- AS{{ net.asn }} - {{ net.name }} - RIR Status: {{ net.rir_status }}
{% endfor %}
{% endlanguage %}

1
peeringdb_server/verified_update/const.py
View File

@@ -4,6 +4,7 @@ SUPPORTED_FIELDS = {
"route_server",
"looking_glass",
"info_type",
"info_types",
"info_prefixes4",
"info_prefixes6",
"info_traffic",

6
peeringdb_server/verified_update/views.py
View File

@@ -108,6 +108,12 @@ def view_verified_update(request):
invalid_permissions[obj] = obj._meta.verbose_name
continue
# backwards compatibility for network info_type
if "info_type" in data:
info_type = data.pop("info_type")
if "info_types" not in data:
data["info_types"] = [info_type]
update_data = {}
diff = {}
update_data.update({"ref_tag": ref_tag, "obj_id": obj_id})

2
tests/test_generate_test_data.py
View File

@@ -10,4 +10,6 @@ class TestGenerateTestData(TestCase):
for reftag, cls in list(REFTAG_MAP.items()):
self.assertGreater(cls.objects.count(), 0)
for instance in cls.objects.all():
if hasattr(instance, "rir_status"):
print("RIR STATUS", instance.rir_status, type(instance.rir_status))
instance.full_clean()

9
tests/test_rir_status.py
View File

@@ -9,12 +9,6 @@ def test_network_auto_initial_rir_status():
"""
Tests `Anytime` network update logic for RIR status handling
laid out in https://github.com/peeringdb/peeringdb/issues/1280
Anytime a network is saved:
if an ASN is added, set rir_status="ok" and set rir_status_updated=created
if an ASN is deleted (manually), set rir_status="notok" and set rir_status_updated=updated
if an ASN is re-added, set rir_status="ok" and set rir_status_updated=updated
"""
org = Organization.objects.create(name="Test org", status="ok")
@@ -22,9 +16,10 @@ def test_network_auto_initial_rir_status():
assert net.rir_status == "pending"
net.rir_status = "missing"
net.delete()
assert net.rir_status == ""
assert net.rir_status == "missing"
net.status = "ok"
net.save()

6
tests/test_verified_update.py
View File

@@ -47,7 +47,11 @@ class VerifiedUpdateTestCase(TestCase):
{
"ref_tag": "net",
"obj_id": self.net.id,
"data": {"info_prefixes4": 4, "info_prefixes6": 6},
"data": {
"info_prefixes4": 4,
"info_prefixes6": 6,
"info_type": "Content",
},
},
{
"ref_tag": "poc",