diff --git a/mainsite/settings/__init__.py b/mainsite/settings/__init__.py index e512a69b..54ecd99a 100644 --- a/mainsite/settings/__init__.py +++ b/mainsite/settings/__init__.py @@ -547,6 +547,9 @@ set_option("IXF_PARSE_ERROR_NOTIFICATION_PERIOD", 36) # conflicts set_option("IXF_TICKET_ON_CONFLICT", True) +# send the ix-f importer generated tickets to deskpro +set_option("IXF_SEND_TICKETS", False) + # toggle the notification of exchanges via email # for ix-f importer conflicts set_option("IXF_NOTIFY_IX_ON_CONFLICT", False) @@ -555,6 +558,10 @@ set_option("IXF_NOTIFY_IX_ON_CONFLICT", False) # for ix-f importer conflicts set_option("IXF_NOTIFY_NET_ON_CONFLICT", False) +# number of days of a conflict being unresolved before +# deskpro ticket is created +set_option("IXF_IMPORTER_DAYS_UNTIL_TICKET", 6) + # when a user tries to delete a protected object, a deskpro # ticket is dispatched. This setting throttles repeat diff --git a/peeringdb_server/admin.py b/peeringdb_server/admin.py index 11b4d159..4cef9889 100644 --- a/peeringdb_server/admin.py +++ b/peeringdb_server/admin.py @@ -2,6 +2,7 @@ import datetime import time import json import ipaddress +import re from . import forms from operator import or_ @@ -15,6 +16,7 @@ from django.contrib.auth import forms from django.contrib.admin import helpers from django.contrib.admin.actions import delete_selected from django.contrib.admin.views.main import ChangeList +from django.db.utils import OperationalError from django.http import HttpResponseForbidden from django import forms as baseForms from django.utils import html @@ -24,6 +26,7 @@ from django.template import loader from django.template.response import TemplateResponse from django.db.models import Q from django.db.models.functions import Concat +from django.db.utils import OperationalError from django_namespace_perms.admin import ( UserPermissionInline, UserPermissionInlineAdd, @@ -66,6 +69,8 @@ from peeringdb_server.models import ( CommandLineTool, UTC, DeskProTicket, + IXFImportEmail, + EnvironmentSetting, ) from peeringdb_server.mail import mail_users_entity_merge from peeringdb_server.inet import RdapLookup, RdapException @@ -326,9 +331,7 @@ soft_delete.short_description = _("SOFT DELETE") class SanitizedAdmin: def get_readonly_fields(self, request, obj=None): - return ("version",) + tuple( - super().get_readonly_fields(request, obj=obj) - ) + return ("version",) + tuple(super().get_readonly_fields(request, obj=obj)) class SoftDeleteAdmin( @@ -367,9 +370,7 @@ class ModelAdminWithVQCtrl: defined """ - fieldsets = tuple( - super().get_fieldsets(request, obj=obj) - ) + fieldsets = tuple(super().get_fieldsets(request, obj=obj)) # on automatically defined fieldsets it will insert the controls # somewhere towards the bottom, we dont want that - so we look for it and @@ -703,9 +704,7 @@ class IXLanIXFMemberImportLogEntryInline(admin.TabularInline): elif rs == -1: text = _("HAS BEEN ROLLED BACK") color = "#d6f0f3" - return mark_safe( - f'
{text}
' - ) + return mark_safe(f'
{text}
') class IXLanIXFMemberImportLogAdmin(admin.ModelAdmin): @@ -1543,11 +1542,90 @@ class CommandLineToolAdmin(admin.ModelAdmin): ) +class IXFImportEmailAdmin(admin.ModelAdmin): + list_display = ("subject", "recipients", "created", "sent", "net", "ix") + readonly_fields = ( + "net", + "ix", + ) + search_fields = ("subject", "ix__name", "net__name") + change_list_template = "admin/change_list_with_regex_search.html" + + def get_search_results(self, request, queryset, search_term): + queryset, use_distinct = super().get_search_results( + request, queryset, search_term + ) + # Require ^ and $ for regex + if search_term.startswith("^") and search_term.endswith("$"): + # Convert search to raw string + try: + search_term = search_term.encode("unicode-escape").decode() + except AttributeError: + return queryset, use_distinct + + # Validate regex expression + try: + re.compile(search_term) + except re.error: + return queryset, use_distinct + + # Add (case insensitive) regex search results to standard search results + try: + queryset = self.model.objects.filter(subject__iregex=search_term) + except OperationalError: + return queryset, use_distinct + + return queryset, use_distinct + + class DeskProTicketAdmin(admin.ModelAdmin): - list_display = ("id", "subject", "user", "created", "published") + list_display = ( + "id", + "subject", + "user", + "created", + "published", + "deskpro_ref", + "deskpro_id", + ) readonly_fields = ("user",) + search_fields = ("subject",) + change_list_template = "admin/change_list_with_regex_search.html" + + def get_search_results(self, request, queryset, search_term): + queryset, use_distinct = super().get_search_results( + request, queryset, search_term + ) + + # Require ^ and $ for regex + if search_term.startswith("^") and search_term.endswith("$"): + # Convert search to raw string + try: + search_term = search_term.encode("unicode-escape").decode() + except AttributeError: + return queryset, use_distinct + + # Validate regex expression + try: + re.compile(search_term) + except re.error: + return queryset, use_distinct + + # Add (case insensitive) regex search results to standard search results + try: + queryset = self.model.objects.filter(subject__iregex=search_term) + except OperationalError: + return queryset, use_distinct + + return queryset, use_distinct + + def save_model(self, request, obj, form, change): + if not obj.id and not obj.user_id: + obj.user = request.user + return super().save_model(request, obj, form, change) +@reversion.create_revision() def apply_ixf_member_data(modeladmin, request, queryset): for ixf_member_data in queryset: try: @@ -1577,7 +1655,9 @@ class IXFMemberDataAdmin(admin.ModelAdmin): "updated", "fetched", "changes", - "error", + "actionable_error", + "reason", + "requirements", ) readonly_fields = ( "marked_for_removal", @@ -1589,9 +1669,14 @@ class IXFMemberDataAdmin(admin.ModelAdmin): "netixlan", "log", "error", + "actionable_error", "created", "updated", + "status", "remote_data", + "requirements", + "requirement_of", + "requirement_detail", ) search_fields = ("asn", "ixlan__id", "ixlan__ix__name", "ipaddr4", "ipaddr6") @@ -1614,6 +1699,10 @@ class IXFMemberDataAdmin(admin.ModelAdmin): "error", "log", "remote_data", + "requirement_of", + "requirement_detail", + "deskpro_id", + "deskpro_ref", ) actions = [apply_ixf_member_data] @@ -1624,9 +1713,42 @@ class IXFMemberDataAdmin(admin.ModelAdmin): "fk": ["ixlan",], } + def get_queryset(self, request): + qset = super().get_queryset(request) + + if request.resolver_match.kwargs.get("object_id"): + return qset + + return qset.filter(requirement_of__isnull=True) + + ids = [ + row.id + for row in qset.exclude(requirement_of__isnull=False) + if row.action != "noop" + ] + + return qset.filter(id__in=ids) + def ix(self, obj): return obj.ixlan.ix + def requirements(self, obj): + return len(obj.requirements) + + def requirement_detail(self, obj): + lines = [] + + for requirement in obj.requirements: + url = django.urls.reverse( + "admin:peeringdb_server_ixfmemberdata_change", args=(requirement.id,) + ) + lines.append(f'{requirement} {requirement.action}') + + if not lines: + return _("No requirements") + + return mark_safe("
".join(lines)) + def netixlan(self, obj): if not obj.netixlan.id: return "-" @@ -1651,6 +1773,7 @@ class IXFMemberDataAdmin(admin.ModelAdmin): def remote_data(self, obj): return obj.json + @reversion.create_revision() def response_change(self, request, obj): if "_save-and-apply" in request.POST: obj.save() @@ -1658,6 +1781,30 @@ class IXFMemberDataAdmin(admin.ModelAdmin): return super().response_change(request, obj) +class EnvironmentSettingForm(baseForms.ModelForm): + + value = baseForms.CharField(required=True, label=_("Value")) + + class Meta: + fields = ["setting", "value"] + + +class EnvironmentSettingAdmin(admin.ModelAdmin): + list_display = ["setting", "value", "created", "updated", "user"] + + fields = ["setting", "value"] + + readonly_fields = ["created", "updated"] + search_fields = ["setting"] + + form = EnvironmentSettingForm + + def save_model(self, request, obj, form, save): + obj.user = request.user + return obj.set_value(form.cleaned_data["value"]) + + +admin.site.register(EnvironmentSetting, EnvironmentSettingAdmin) admin.site.register(IXFMemberData, IXFMemberDataAdmin) admin.site.register(Facility, FacilityAdmin) admin.site.register(InternetExchange, InternetExchangeAdmin) @@ -1678,3 +1825,4 @@ admin.site.register(IXLanIXFMemberImportLog, IXLanIXFMemberImportLogAdmin) admin.site.register(CommandLineTool, CommandLineToolAdmin) admin.site.register(UserOrgAffiliationRequest, UserOrgAffiliationRequestAdmin) admin.site.register(DeskProTicket, DeskProTicketAdmin) +admin.site.register(IXFImportEmail, IXFImportEmailAdmin) diff --git a/peeringdb_server/admin_commandline_tools.py b/peeringdb_server/admin_commandline_tools.py index c2d31187..262b449f 100644 --- a/peeringdb_server/admin_commandline_tools.py +++ b/peeringdb_server/admin_commandline_tools.py @@ -6,6 +6,7 @@ from reversion.models import Version from dal import autocomplete from django import forms +from django.conf import settings from django.core.management import call_command from peeringdb_server.models import ( REFTAG_MAP, @@ -385,3 +386,61 @@ class ToolUndelete(CommandLineToolWrapper): ) if obj.status != "deleted": raise ValueError(f"{obj} is not currently marked as deleted") + + +@register_tool +class ToolIXFIXPMemberImport(CommandLineToolWrapper): + """ + Allows resets for various parts of the ix-f member data import protocol. + And import ix-f member data for a single Ixlan at a time. + """ + + tool = "pdb_ixf_ixp_member_import" + queue = 1 + + class Form(forms.Form): + ix = forms.ModelChoiceField( + queryset=InternetExchange.objects.all(), + widget=autocomplete.ModelSelect2(url="/autocomplete/ix/json"), + help_text=_( + "Select an Internet Exchange to perform an ix-f memberdata import" + ), + ) + + if settings.RELEASE_ENV != "prod": + + # reset toggles are not available on production + # environment + + reset = forms.BooleanField( + required=False, initial=False, help_text=_("Reset all") + ) + reset_hints = forms.BooleanField( + required=False, initial=False, help_text=_("Reset hints") + ) + reset_dismisses = forms.BooleanField( + required=False, initial=False, help_text=_("Reset dismisses") + ) + reset_email = forms.BooleanField( + required=False, initial=False, help_text=_("Reset email") + ) + reset_tickets = forms.BooleanField( + required=False, initial=False, help_text=_("Reset tickets") + ) + + @property + def description(self): + return "IX-F Member Import Tool" + + def set_arguments(self, form_data): + for key in [ + "reset", + "reset_hints", + "reset_dismisses", + "reset_email", + "reset_tickets", + ]: + self.kwargs[key] = form_data.get(key, False) + + if form_data.get("ix"): + self.kwargs["ixlan"] = [form_data.get("ix").id] diff --git a/peeringdb_server/api_cache.py b/peeringdb_server/api_cache.py index a993024d..98a7bc92 100644 --- a/peeringdb_server/api_cache.py +++ b/peeringdb_server/api_cache.py @@ -47,8 +47,7 @@ class APICacheLoader: if self.fields: self.fields = self.fields.split(",") self.path = os.path.join( - settings.API_CACHE_ROOT, - f"{viewset.model.handleref.tag}-{self.depth}.json", + settings.API_CACHE_ROOT, f"{viewset.model.handleref.tag}-{self.depth}.json", ) def qualifies(self): @@ -243,8 +242,8 @@ class APICacheLoader: ids = [r[proxy_id] for r in data] return { - r["id"]: r[target_id] - for r in model.objects.filter(id__in=ids).values("id", target_id) + r["id"]: r[target_id] + for r in model.objects.filter(id__in=ids).values("id", target_id) } # permissioning functions for each handlref type diff --git a/peeringdb_server/api_schema.py b/peeringdb_server/api_schema.py index e7d180ec..ca059071 100644 --- a/peeringdb_server/api_schema.py +++ b/peeringdb_server/api_schema.py @@ -127,9 +127,7 @@ class BaseSchema(AutoSchema): # check if we have an augmentation method set for the operation_type and object type # combination, if so run it - augment = getattr( - self, f"augment_{op_type}_{model.HandleRef.tag}", None - ) + augment = getattr(self, f"augment_{op_type}_{model.HandleRef.tag}", None) if augment: augment(serializer, model, op_dict) @@ -313,9 +311,7 @@ class BaseSchema(AutoSchema): choices = getattr(fld, "choices", None) if choices: description.append( - "{}".format( - ", ".join([f"`{_id}`" for _id, label in choices]) - ) + "{}".format(", ".join([f"`{_id}`" for _id, label in choices])) ) if supported_filters: diff --git a/peeringdb_server/deskpro.py b/peeringdb_server/deskpro.py index c0487580..debd7c12 100644 --- a/peeringdb_server/deskpro.py +++ b/peeringdb_server/deskpro.py @@ -2,6 +2,7 @@ DeskPro API Client """ +import uuid import re import requests import datetime @@ -18,9 +19,7 @@ def ticket_queue(subject, body, user): """ queue a deskpro ticket for creation """ ticket = DeskProTicket.objects.create( - subject=f"{settings.EMAIL_SUBJECT_PREFIX}{subject}", - body=body, - user=user, + subject=f"{settings.EMAIL_SUBJECT_PREFIX}{subject}", body=body, user=user, ) @@ -178,17 +177,22 @@ class APIClient: def create_ticket(self, ticket): person = self.require_person(ticket.user) - ticket_response = self.create( - "tickets", - { - "subject": ticket.subject, - "person": {"id": person["id"]}, - "status": "awaiting_agent", - }, - ) + + if not ticket.deskpro_id: + ticket_response = self.create( + "tickets", + { + "subject": ticket.subject, + "person": {"id": person["id"]}, + "status": "awaiting_agent", + }, + ) + + ticket.deskpro_ref = ticket_response["ref"] + ticket.deskpro_id = ticket_response["id"] self.create( - "tickets/{}/messages".format(ticket_response["id"]), + "tickets/{}/messages".format(ticket.deskpro_id), { "message": ticket.body.replace("\n", "
\n"), "person": person["id"], @@ -197,6 +201,33 @@ class APIClient: ) +class MockAPIClient(APIClient): + + """ + A mock api client for the deskpro API + + The IX-F importer uses this when + IXF_SEND_TICKETS=False + """ + + def __init__(self, *args, **kwargs): + self.ticket_count = 0 + + def get(self, endpoint, param): + + if endpoint == "people": + return {"id": 1} + + return {} + + def create(self, endpoint, param): + if endpoint == "tickets": + self.ticket_count += 1 + ref = "{}".format(uuid.uuid4()) + return {"ref": ref[:16], "id": self.ticket_count} + return {} + + def ticket_queue_deletion_prevented(user, instance): """ queue deskpro ticket to notify about the prevented diff --git a/peeringdb_server/import_views.py b/peeringdb_server/import_views.py index 4d9994de..83dbb8bf 100644 --- a/peeringdb_server/import_views.py +++ b/peeringdb_server/import_views.py @@ -160,10 +160,7 @@ def view_import_net_ixf_preview(request, net_id): total_log["data"].extend(log_data) total_log["errors"].extend( - [ - f"{ixlan.ix.name}({ixlan.id}): {err}" - for err in importer.log["errors"] - ] + [f"{ixlan.ix.name}({ixlan.id}): {err}" for err in importer.log["errors"]] ) return pretty_response(total_log) diff --git a/peeringdb_server/ixf.py b/peeringdb_server/ixf.py index e84f6319..7aae8b13 100644 --- a/peeringdb_server/ixf.py +++ b/peeringdb_server/ixf.py @@ -9,6 +9,7 @@ from django.db import transaction from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ from django.core.exceptions import ValidationError +from django.template import loader from django.conf import settings import reversion @@ -20,7 +21,14 @@ from peeringdb_server.models import ( Network, NetworkIXLan, IXFMemberData, + NetworkProtocolsDisabled, + User, + DeskProTicket, + EnvironmentSetting, + debug_mail, + IXFImportEmail, ) +import peeringdb_server.deskpro as deskpro REASON_ENTRY_GONE_FROM_REMOTE = _( "The entry for (asn and IPv4 and IPv6) does not exist " @@ -54,6 +62,63 @@ class Importer: "operational", ] + @property + def ticket_user(self): + """ + Returns the User instance for the user to use + to create DeskPRO tickets + """ + if not hasattr(self, "_ticket_user"): + self._ticket_user = User.objects.get(username="ixf_importer") + return self._ticket_user + + @property + def deskpro_client(self): + if not hasattr(self, "_deskpro_client"): + if settings.IXF_SEND_TICKETS: + cls = deskpro.APIClient + else: + cls = deskpro.MockAPIClient + + self._deskpro_client = cls(settings.DESKPRO_URL, settings.DESKPRO_KEY) + return self._deskpro_client + + @property + def tickets_enabled(self): + """ + Returns whether or not deskpr ticket creation for ix-f + conflicts are enabled or not + + This can be controlled by the IXF_TICKET_ON_CONFLICT + setting + """ + + return getattr(settings, "IXF_TICKET_ON_CONFLICT", True) + + @property + def notify_ix_enabled(self): + """ + Returns whether or not notifications to the exchange + are enabled. + + This can be controlled by the IXF_NOTIFY_IX_ON_CONFLICT + setting + """ + + return getattr(settings, "IXF_NOTIFY_IX_ON_CONFLICT", False) + + @property + def notify_net_enabled(self): + """ + Returns whether or not notifications to the network + are enabled. + + This can be controlled by the IXF_NOTIFY_NET_ON_CONFLICT + setting + """ + + return getattr(settings, "IXF_NOTIFY_NET_ON_CONFLICT", False) + def __init__(self): self.cache_only = False self.skip_import = False @@ -69,12 +134,14 @@ class Importer: "noop": [], } self.pending_save = [] + self.deletions = {} self.asns = [] self.ixlan = ixlan self.save = save self.asn = asn self.now = datetime.datetime.now(datetime.timezone.utc) self.invalid_ip_errors = [] + self.notifications = [] def fetch(self, url, timeout=5): """ @@ -223,7 +290,6 @@ class Importer: return False if self.skip_import: - self.cleanup_ixf_member_data() return True try: @@ -235,17 +301,22 @@ class Importer: self.log_error(f"Internal Error 'KeyError': {exc}", save=save) return False - # process any netixlans that need to be deleted - self.process_deletions() + with transaction.atomic(): + # process any netixlans that need to be deleted + self.process_deletions() - # process creation of new netixlans and updates - # of existing netixlans. This needs to happen - # after process_deletions in order to avoid potential - # ip conflicts - self.process_saves() + # process creation of new netixlans and updates + # of existing netixlans. This needs to happen + # after process_deletions in order to avoid potential + # ip conflicts + self.process_saves() self.cleanup_ixf_member_data() + # create tickets for unresolved proposals + + self.ticket_aged_proposals() + # archive the import so we can roll it back later if needed self.archive() @@ -322,49 +393,82 @@ class Importer: for netixlan in netixlan_qset: if netixlan.ixf_id not in self.ixf_ids: + ixf_member_data = IXFMemberData.instantiate( netixlan.asn, netixlan.ipaddr4, netixlan.ipaddr6, netixlan.ixlan, + speed=netixlan.speed, + operational=netixlan.operational, + is_rs_peer=netixlan.is_rs_peer, + delete=True, data={}, ) + self.deletions[ixf_member_data.ixf_id] = ixf_member_data if netixlan.network.allow_ixp_update: self.log_apply( ixf_member_data.apply(save=self.save), reason=REASON_ENTRY_GONE_FROM_REMOTE, ) else: - ixf_member_data.set_remove( + notify = ixf_member_data.set_remove( save=self.save, reason=REASON_ENTRY_GONE_FROM_REMOTE ) + if notify: + self.queue_notification(ixf_member_data, "remove") self.log_ixf_member_data(ixf_member_data) def cleanup_ixf_member_data(self): + if not self.save: + + """ + In some cases you dont want to run a cleanup process + For example when the importer runs in preview mode + triggered by a network admin + """ + + return + + qset = IXFMemberData.objects.filter(ixlan=self.ixlan) + + if self.asn: + + # if we are only processing for a specified asn + # we only clean up member data for that asn + + qset = qset.filter(asn=self.asn) + # clean up old ix-f memeber data objects - for ixf_member in IXFMemberData.objects.filter(ixlan=self.ixlan): + for ixf_member in qset: # proposed deletion got fulfilled if ixf_member.action == "delete": if ixf_member.netixlan.status == "deleted": - ixf_member.set_resolved() + if ixf_member.set_resolved(save=self.save): + self.queue_notification(ixf_member, "resolved") # noop means the ask has been fulfilled but the # ixf member data entry has not been set to resolved yet elif ixf_member.action == "noop": - ixf_member.set_resolved() + if ( + ixf_member.set_resolved(save=self.save) + and not ixf_member.requirement_of + ): + self.queue_notification(ixf_member, "resolved") # proposed change / addition is now gone from # ix-f data elif not self.skip_import and ixf_member.ixf_id not in self.ixf_ids: if ixf_member.action in ["add", "modify"]: - ixf_member.set_resolved() + if ixf_member.set_resolved(save=self.save): + self.queue_notification(ixf_member, "resolved") @transaction.atomic() def archive(self): @@ -468,7 +572,7 @@ class Importer: asn = member["asnum"] for connection in connection_list: - self.connection_errors = [] + self.connection_errors = {} state = connection.get("state", "active").lower() if state in self.allowed_states: @@ -515,6 +619,9 @@ class Importer: ipv4_addr = ipv4.get("address") ipv6_addr = ipv6.get("address") + ipv4_support = network.ipv4_support + ipv6_support = network.ipv6_support + # parse and validate the ipaddresses attached to the vlan # append a unqiue ixf identifier to self.ixf_ids # @@ -536,7 +643,6 @@ class Importer: ixf_id.append(None) ixf_id = tuple(ixf_id) - self.ixf_ids.append(ixf_id) except (ipaddress.AddressValueError, ValueError) as exc: self.invalid_ip_errors.append(f"{exc}") @@ -547,14 +653,81 @@ class Importer: ) continue - if not self.save and ( - not self.ixlan.test_ipv4_address(ipv4_addr) - and not self.ixlan.test_ipv6_address(ipv6_addr) + ipv4_valid_for_ixlan = self.ixlan.test_ipv4_address(ipv4_addr) + ipv6_valid_for_ixlan = self.ixlan.test_ipv6_address(ipv6_addr) + + if ( + ipv4_addr + and not ipv4_valid_for_ixlan + and ipv6_addr + and not ipv6_valid_for_ixlan ): - # for the preview we don't care at all about new ip addresses - # not at the ixlan if they dont match the prefix + # neither ipaddress falls into address space + # for this ixlan, ignore + continue + elif not ipv4_valid_for_ixlan and not ipv6_addr: + + # ipv4 address does not fall into address space + # and ipv6 is not provided, ignore + + continue + + elif not ipv6_valid_for_ixlan and not ipv4_addr: + + # ipv6 address does not fall into address space + # and ipv4 is not provided, ignore + + continue + + protocol_conflicts = [] + + # keep track of conflicts between ix/net in terms of ip + # protocols supported. + + if ipv4_addr and not ipv4_support: + protocol_conflicts.append(4) + + if ipv6_addr and not ipv6_support: + protocol_conflicts.append(6) + + if protocol_conflicts: + self.queue_notification( + IXFMemberData.instantiate( + asn, + ipv4_addr, + ipv6_addr, + ixlan=self.ixlan, + save=False, + validate_network_protocols=False, + ), + "protocol-conflict", + ac=False, + net=True, + ix=True, + ipaddr4=ipv4_addr, + ipaddr6=ipv6_addr, + ) + + self.ixf_ids.append(ixf_id) + + if not network.ipv6_support: + self.ixf_ids.append((asn, ixf_id[1], None)) + netixlan = NetworkIXLan.objects.filter( + status="ok", ipaddr4=ixf_id[1] + ).first() + if netixlan: + self.ixf_ids.append((asn, ixf_id[1], netixlan.ipaddr6)) + + if not network.ipv4_support: + self.ixf_ids.append((asn, None, ixf_id[2])) + netixlan = NetworkIXLan.objects.filter( + status="ok", ipaddr6=ixf_id[2] + ).first() + if netixlan: + self.ixf_ids.append((asn, netixlan.ipaddr4, ixf_id[2])) + if connection.get("state", "active") == "inactive": operational = False else: @@ -564,20 +737,24 @@ class Importer: "routeserver", False ) - ixf_member_data = IXFMemberData.instantiate( - asn, - ipv4_addr, - ipv6_addr, - speed=speed, - operational=operational, - is_rs_peer=is_rs_peer, - data=json.dumps(member), - ixlan=self.ixlan, - save=self.save, - ) + try: + ixf_member_data = IXFMemberData.instantiate( + asn, + ipv4_addr, + ipv6_addr, + speed=speed, + operational=operational, + is_rs_peer=is_rs_peer, + data=json.dumps(member), + ixlan=self.ixlan, + save=self.save, + ) + except NetworkProtocolsDisabled as exc: + self.log_error(f"{exc}") + continue if self.connection_errors: - ixf_member_data.error = "\n".join(self.connection_errors) + ixf_member_data.error = json.dumps(self.connection_errors) else: ixf_member_data.error = ixf_member_data.previous_error @@ -600,7 +777,9 @@ class Importer: except ValueError: log_msg = _("Invalid speed value: {}").format(iface.get("if_speed")) self.log_error(log_msg) - self.connection_errors.append(log_msg) + if "speed" not in self.connection_errors: + self.connection_errors["speed"] = [] + self.connection_errors["speed"].append(log_msg) return speed def apply_add_or_update(self, ixf_member_data): @@ -627,8 +806,24 @@ class Importer: self.apply_add(ixf_member_data) + def queue_notification( + self, ixf_member_data, typ, ac=True, ix=True, net=True, **context + ): + self.notifications.append( + { + "ixf_member_data": ixf_member_data, + "ac": ac, + "ix": ix, + "net": net, + "typ": typ, + "action": ixf_member_data.action, + "context": context, + } + ) + def resolve(self, ixf_member_data): - ixf_member_data.set_resolved(save=self.save) + if ixf_member_data.set_resolved(save=self.save): + self.queue_notification(ixf_member_data, "resolved") def apply_update(self, ixf_member_data): changed_fields = ", ".join(ixf_member_data.changes.keys()) @@ -638,11 +833,12 @@ class Importer: try: self.log_apply(ixf_member_data.apply(save=self.save), reason=reason) except ValidationError as exc: - ixf_member_data.set_conflict(error=exc, save=self.save) + if ixf_member_data.set_conflict(error=exc, save=self.save): + self.queue_notification(ixf_member_data, ixf_member_data.action) else: - ixf_member_data.set_update( - save=self.save, reason=reason, - ) + notify = ixf_member_data.set_update(save=self.save, reason=reason,) + if notify: + self.queue_notification(ixf_member_data, "modify") self.log_ixf_member_data(ixf_member_data) def apply_add(self, ixf_member_data): @@ -653,12 +849,91 @@ class Importer: self.log_apply( ixf_member_data.apply(save=self.save), reason=REASON_NEW_ENTRY ) + if not self.save: + self.consolidate_delete_add(ixf_member_data) except ValidationError as exc: - ixf_member_data.set_conflict(error=exc, save=self.save) + if ixf_member_data.set_conflict(error=exc, save=self.save): + self.queue_notification(ixf_member_data, ixf_member_data.action) else: - ixf_member_data.set_add(save=self.save, reason=REASON_NEW_ENTRY) + notify = ixf_member_data.set_add(save=self.save, reason=REASON_NEW_ENTRY) + self.log_ixf_member_data(ixf_member_data) + self.consolidate_delete_add(ixf_member_data) + + if notify and ixf_member_data.net_present_at_ix: + self.queue_notification(ixf_member_data, ixf_member_data.action) + elif notify: + self.queue_notification( + ixf_member_data, ixf_member_data.action, ix=False, ac=False + ) + + def consolidate_delete_add(self, ixf_member_data): + + ip4_deletion = None + ip6_deletion = None + + for ixf_id, deletion in self.deletions.items(): + if deletion.asn == ixf_member_data.asn: + if deletion.ipaddr4 == ixf_member_data.init_ipaddr4: + ip4_deletion = deletion + if deletion.ipaddr6 == ixf_member_data.init_ipaddr6: + ip6_deletion = deletion + + if ip4_deletion and ip6_deletion: + break + + if not ip4_deletion and not ip6_deletion: + return + + ip4_req = ixf_member_data.set_requirement(ip4_deletion, save=self.save) + ip6_req = ixf_member_data.set_requirement(ip6_deletion, save=self.save) + + if not ip4_req and not ip6_req: + return + + if not ixf_member_data.has_requirements: + return + + if ip4_deletion: + try: + self.log["data"].remove(ip4_deletion.ixf_log_entry) + except ValueError: + pass + if ip6_deletion: + try: + self.log["data"].remove(ip6_deletion.ixf_log_entry) + except ValueError: + pass + + log_entry = ixf_member_data.ixf_log_entry + + log_entry["action"] = log_entry["action"].replace("add", "modify") + changed_fields = ", ".join( + ixf_member_data._changes( + getattr(ip4_deletion, "netixlan", None) + or getattr(ip6_deletion, "netixlan", None) + ).keys() + ) + + ipaddr_info = "" + + if ip4_deletion and ip6_deletion: + ipaddr_info = _("IP addresses moved to same entry") + elif ip4_deletion: + ipaddr_info = _("IPv6 not set") + elif ip6_deletion: + ipaddr_info = _("IPv4 not set") + + log_entry["reason"] = f"{REASON_VALUES_CHANGED}: {changed_fields} {ipaddr_info}" + + ixf_member_data.reason = log_entry["reason"] + ixf_member_data.error = None + if self.save: + if ixf_member_data.updated: + ixf_member_data.save_without_update() + else: + ixf_member_data.save() def save_log(self): """ @@ -687,10 +962,14 @@ class Importer: } ) - return self.log_peer( + result = self.log_peer( netixlan.asn, apply_result["action"], reason, netixlan=netixlan ) + apply_result["ixf_member_data"].ixf_log_entry = netixlan.ixf_log_entry + + return result + def log_ixf_member_data(self, ixf_member_data): return self.log_peer( ixf_member_data.net.asn, @@ -736,12 +1015,452 @@ class Importer: "operational": netixlan.operational, } ) + entry = { + "peer": peer, + "action": action, + "reason": f"{reason}", + } + self.log["data"].append(entry) - self.log["data"].append( - {"peer": peer, "action": action, "reason": f"{reason}",} + if netixlan: + netixlan.ixf_log_entry = entry + + def _email(self, subject, message, recipients, net=None, ix=None): + """ + Send email + + Honors the MAIL_DEBUG setting + + Will create IXFImportEmail entry + """ + + if not recipients: + return + + email_log = None + + logged_subject = f"{settings.EMAIL_SUBJECT_PREFIX}[IX-F] {subject}" + + if net: + email_log = IXFImportEmail.objects.create( + subject=logged_subject, + message=message, + recipients=",".join(recipients), + net=net, + ) + + if not self.notify_net_enabled: + return + + if ix: + email_log = IXFImportEmail.objects.create( + subject=logged_subject, + message=message, + recipients=",".join(recipients), + ix=ix, + ) + + if not self.notify_ix_enabled: + return + + if not getattr(settings, "MAIL_DEBUG", False): + mail = EmailMultiAlternatives( + subject, strip_tags(message), settings.DEFAULT_FROM_EMAIL, recipients, + ) + mail.send(fail_silently=False) + else: + print("EMAIL", subject, recipients) + # debug_mail( + # subject, message, settings.DEFAULT_FROM_EMAIL, recipients, + # ) + + if email_log: + email_log.sent = datetime.datetime.now(datetime.timezone.utc) + email_log.save() + + def _ticket(self, ixf_member_data, subject, message): + + """ + Create and send a deskpro ticket + + Return the DeskPROTicket instance + + Argument(s): + + - ixf_member_data (`IXFMemberData`) + - subject (`str`) + - message (`str`) + + """ + + subject = f"{settings.EMAIL_SUBJECT_PREFIX}[IX-F] {subject}" + + client = self.deskpro_client + + if not ixf_member_data.deskpro_id: + old_ticket = DeskProTicket.objects.filter( + subject=subject, deskpro_id__isnull=False + ).first() + if old_ticket: + ixf_member_data.deskpro_id = old_ticket.deskpro_id + ixf_member_data.deskpro_ref = old_ticket.deskpro_ref + + ticket = DeskProTicket.objects.create( + subject=subject, + body=message, + user=self.ticket_user, + deskpro_id=ixf_member_data.deskpro_id, + deskpro_ref=ixf_member_data.deskpro_ref, ) + try: + client.create_ticket(ticket) + ticket.published = datetime.datetime.now(datetime.timezone.utc) + ticket.save() + except Exception as exc: + ticket.subject = f"[FAILED]{ticket.subject}" + ticket.body = f"{ticket.body}\n\n{exc.data}" + ticket.save() + return ticket + + def consolidate_proposals(self): + + """ + Renders and consolidates all proposals for each net and ix + (#772) + + Returns a dict + + { + "net": { + Network : { + "proposals": { + InternetExchange { + "add" : [, ...], + "modify" : [, ...], + "delete" : [, ...], + }, + }, + "count": + "entity": Network, + "contacts": [, ...] + } + }, + "ix": { + InternetExchange : { + "proposals": { + Network : { + "add" : [, ...], + "modify" : [, ...], + "delete" : [, ...], + }, + }, + "count": + "entity": InternetExchange, + "contacts": [, ...] + } + } + } + """ + + net_notifications = {} + ix_notifications = {} + + for notification in self.notifications: + + ixf_member_data = notification["ixf_member_data"] + action = notification["action"] + typ = notification["typ"] + notify_ix = notification["ix"] + notify_net = notification["net"] + context = notification["context"] + + # we don't care about resolved proposals + + if typ == "resolved": + if ixf_member_data.deskpro_id: + self.ticket_proposal(**notification) + continue + + if typ == "protocol-conflict": + action = "protocol_conflict" + + # in some edge cases (ip4 set on netixlan, network indicating + # only ipv6 support) we can get empty modify notifications + # that we need to throw out. (#771) + if typ == "modify": + if not ixf_member_data.actionable_changes: + continue + + # we don't care about proposals that are hidden + # requirements of other proposals + + if ixf_member_data.requirement_of: + continue + + asn = ixf_member_data.net + ix = ixf_member_data.ix + ix_contacts = ixf_member_data.ix_contacts + net_contacts = ixf_member_data.net_contacts + + # no suitable contact points found for + # one of the sides, immediately make a ticket + + if not ix_contacts or not net_contacts: + if typ != "protocol-conflict": + self.ticket_proposal(**notification) + + template_file = f"email/notify-ixf-{typ}-inline.txt" + + # prepare consolidation rocketship + + if asn not in net_notifications: + net_notifications[asn] = { + "proposals": {}, + "count": 0, + "entity": ixf_member_data.net, + "contacts": ixf_member_data.net_contacts, + } + + if ix not in net_notifications[asn]["proposals"]: + net_notifications[asn]["proposals"][ix] = { + "add": [], + "modify": [], + "delete": [], + "protocol_conflict": None, + } + + if ix not in ix_notifications: + ix_notifications[ix] = { + "proposals": {}, + "count": 0, + "entity": ixf_member_data.ix, + "contacts": ixf_member_data.ix_contacts, + } + + if asn not in ix_notifications[ix]["proposals"]: + ix_notifications[ix]["proposals"][asn] = { + "add": [], + "modify": [], + "delete": [], + "protocol_conflict": None, + } + + # render and push proposal text for network + + if notify_net and ixf_member_data.actionable_for_network: + proposals = net_notifications[asn]["proposals"][ix] + message = ixf_member_data.render_notification( + template_file, recipient="net", context=context, + ) + + if action == "protocol_conflict": + proposals[action] = message + else: + proposals[action].append(message) + net_notifications[asn]["count"] += 1 + + # render and push proposal text for exchange + + if notify_ix: + proposals = ix_notifications[ix]["proposals"][asn] + message = ixf_member_data.render_notification( + template_file, recipient="ix", context=context, + ) + + if action == "protocol_conflict": + proposals[action] = message + else: + proposals[action].append(message) + ix_notifications[ix]["count"] += 1 + + return { + "net": net_notifications, + "ix": ix_notifications, + } + + def notify_proposals(self): + + """ + Sends all collected notification proposals + """ + + if not self.save: + return + + # consolidate proposals into net,ix and ix,net + # groupings + + consolidated = self.consolidate_proposals() + + ticket_days = EnvironmentSetting.get_setting_value( + "IXF_IMPORTER_DAYS_UNTIL_TICKET" + ) + + template = loader.get_template("email/notify-ixf-consolidated.txt") + + for recipient in ["ix", "net"]: + for other_entity, data in consolidated[recipient].items(): + contacts = data["contacts"] + + # we did not find any suitable contact points + # skip + + if not contacts: + continue + + # no messages + + if not data["count"]: + continue + + # render the consolidated message + + message = template.render( + { + "recipient": recipient, + "entity": data["entity"], + "count": data["count"], + "ticket_days": ticket_days, + "proposals": data["proposals"], + } + ) + + if recipient == "net": + subject = _( + "PeeringDB: Action May Be Needed: IX-F Importer " + "data mismatch between AS{} and one or more IXPs" + ).format(data["entity"].asn) + self._email(subject, message, contacts, net=data["entity"]) + else: + subject = _( + "PeeringDB: Action May Be Needed: IX-F Importer " + "data mismatch between {} and one or more networks" + ).format(data["entity"].name) + self._email(subject, message, contacts, ix=data["entity"]) + + def ticket_aged_proposals(self): + + """ + Cycle through all IXFMemberData objects that + and create tickets for those that are older + than the period specified in IXF_IMPORTER_DAYS_UNTIL_TICKET + and that don't have any ticket associated with + them yet + """ + + if not self.save: + return + + qset = IXFMemberData.objects.filter( + deskpro_id__isnull=True, requirement_of__isnull=True + ) + + # get ticket days period + ticket_days = EnvironmentSetting.get_setting_value( + "IXF_IMPORTER_DAYS_UNTIL_TICKET" + ) + + if ticket_days > 0: + + # we adjust the query to only get proposals + # that are older than the specified period + + now = datetime.datetime.now(datetime.timezone.utc) + max_age = now - datetime.timedelta(days=ticket_days) + qset = qset.filter(created__lte=max_age) + + for ixf_member_data in qset: + + action = ixf_member_data.action + if action == "delete": + action = "remove" + typ = action + + # create the ticket + # and also notify the net and ix with + # a reference to the ticket in the subject + + self.ticket_proposal( + ixf_member_data, typ, True, True, True, {}, ixf_member_data.action + ) + + def ticket_proposal(self, ixf_member_data, typ, ac, ix, net, context, action): + + """ + Creates a deskpro ticket and contexts net and ix with + ticket reference in the subject + + Argument(s) + + - ixf_member_data (IXFMemberData) + - typ (str): proposal type 'add','delete','modify','resolve','conflict' + - ac (bool): If true DeskProTicket will be created + - ix (bool): If true email will be sent to ix + - net (bool): If true email will be sent to net + - context (dict): extra template context + """ + + if typ == "add" and ixf_member_data.requirements: + typ = ixf_member_data.action + subject = f"{ixf_member_data.primary_requirement}" + else: + subject = f"{ixf_member_data}" + + subject = f"{subject} IX-F Conflict Resolution" + + template_file = f"email/notify-ixf-{typ}.txt" + + # DeskPRO ticket + + if ac and self.tickets_enabled: + message = ixf_member_data.render_notification( + template_file, recipient="ac", context=context + ) + + ticket = self._ticket(ixf_member_data, subject, message) + ixf_member_data.deskpro_id = ticket.deskpro_id + ixf_member_data.deskpro_ref = ticket.deskpro_ref + if ixf_member_data.id: + ixf_member_data.save() + + # we have deskpro reference number, put it in the + # subject + + if ixf_member_data.deskpro_ref: + subject = f"{subject} [#{ixf_member_data.deskpro_ref}]" + + # Notify Exchange + + if ix: + message = ixf_member_data.render_notification( + template_file, recipient="ix", context=context + ) + self._email( + subject, message, ixf_member_data.ix_contacts, ix=ixf_member_data.ix + ) + + # Notify network + + if net and ixf_member_data.actionable_for_network: + message = ixf_member_data.render_notification( + template_file, recipient="net", context=context + ) + self._email( + subject, message, ixf_member_data.net_contacts, net=ixf_member_data.net + ) + def notify_error(self, error): + + """ + Notifies the exchange and AC of any errors that + were encountered when the IX-F data was + parsed + """ + + if not self.save: + return + now = datetime.datetime.now(datetime.timezone.utc) notified = self.ixlan.ixf_ixp_import_error_notified prev_error = self.ixlan.ixf_ixp_import_error @@ -756,14 +1475,18 @@ class Importer: self.ixlan.save() ixf_member_data = IXFMemberData(ixlan=self.ixlan, asn=0) - ixf_member_data._notify( - "email/notify-ixf-source-error.txt", - "Could not process IX-F Data", - context={"error": error, "dt": now}, - save=False, - ix=True, - ac=True, + + subject = "Could not process IX-F Data" + template = loader.get_template("email/notify-ixf-source-error.txt") + message = template.render( + {"error": error, "dt": now, "instance": ixf_member_data} ) + self._ticket(ixf_member_data, subject, message) + + if ixf_member_data.ix_contacts: + self._email( + subject, message, ixf_member_data.ix_contacts, ix=ixf_member_data.ix + ) def log_error(self, error, save=False): """ diff --git a/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py b/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py index 4b3fab4c..f508edc5 100644 --- a/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py +++ b/peeringdb_server/management/commands/pdb_ixf_ixp_member_import.py @@ -3,12 +3,14 @@ import json from django.core.management.base import BaseCommand, CommandError from django.db import transaction - +from django.conf import settings from peeringdb_server.models import ( IXLan, NetworkIXLan, Network, IXFMemberData, + DeskProTicket, + IXFImportEmail, ) from peeringdb_server import ixf @@ -38,10 +40,31 @@ class Command(BaseCommand): help="Just update IX-F cache, do NOT perform any import logic", ) parser.add_argument( - "--delete-all-ixfmemberdata", + "--reset-hints", action="store_true", help="This removes all IXFMemberData objects", ) + parser.add_argument( + "--reset-dismisses", + action="store_true", + help="This resets all dismissed IXFMemberData objects", + ) + parser.add_argument( + "--reset-tickets", + action="store_true", + help="This removes DeskProTicket objects where subject contains '[IX-F]'", + ) + parser.add_argument( + "--reset-email", + action="store_true", + help="This empties the IXFImportEmail table", + ) + parser.add_argument( + "--reset", + action="store_true", + help="This removes all IXFMemberData objects", + ) + def log(self, msg, debug=False): if self.preview: @@ -53,6 +76,67 @@ class Command(BaseCommand): else: self.stdout.write("[Pretend] {}".format(msg)) + def release_env_check(self, flag): + if settings.RELEASE_ENV != "prod": + return True + else: + raise PermissionError("Flag {} is not permitted to be used in production.") + + def initiate_reset_flags(self, **options): + flags = ["reset", "reset_hints", "reset_dismisses", "reset_tickets", "reset_email"] + self.active_flags = [] + for flag in flags: + setattr(self, flag, options.get(flag, False)) + if options.get(flag, False): + self.active_flags.append(flag) + + self.release_env_check(self.active_flags) + return self.active_flags + + def release_env_check(self, active_flags): + if settings.RELEASE_ENV == "prod": + if len(active_flags) == 1: + raise PermissionError( + "Cannot use flag '{}'' in production".format(active_flags[0])) + elif len(active_flags) >= 1: + raise PermissionError( + "Cannot use flags '{}' in production".format(", ".join(active_flags))) + return True + + def reset_all_hints(self): + self.log("Resetting hints: deleting IXFMemberData instances") + if self.commit: + IXFMemberData.objects.all().delete() + + def reset_all_dismisses(self): + self.log("Resetting dismisses: setting IXFMemberData.dismissed=False on all IXFMemberData instances") + if self.commit: + for ixfmemberdata in IXFMemberData.objects.all(): + ixfmemberdata.dismissed = False + ixfmemberdata.save() + + def reset_all_email(self): + self.log("Resetting email: emptying the IXFImportEmail table") + if self.commit: + IXFImportEmail.objects.all().delete() + + def reset_all_tickets(self): + self.log("Resetting tickets: removing DeskProTicket objects where subject contains '[IX-F]'") + if self.commit: + DeskProTicket.objects.filter(subject__contains="[IX-F]").delete() + + def create_reset_ticket(self): + self.log("Creating deskproticket for the following resets: {}".format( + ", ".join(self.active_flags) + )) + if self.commit: + DeskProTicket.objects.create( + user=ixf.Importer().ticket_user, + subject="[IX-F] command-line reset", + body="Applied the following resets to the IX-F data: {}".format( + ", ".join(self.active_flags)), + ) + def handle(self, *args, **options): self.commit = options.get("commit", False) self.debug = options.get("debug", False) @@ -60,9 +144,19 @@ class Command(BaseCommand): self.cache = options.get("cache", False) self.skip_import = options.get("skip_import", False) - if options.get("delete_all_ixfmemberdata"): - self.log("Deleting IXFMemberData Instances ...") - IXFMemberData.objects.all().delete() + self.active_reset_flags = self.initiate_reset_flags(**options) + + if self.reset or self.reset_hints: + self.reset_all_hints() + if self.reset or self.reset_dismisses: + self.reset_all_dismisses() + if self.reset or self.reset_email: + self.reset_all_email() + if self.reset or self.reset_tickets: + self.reset_all_tickets() + + if len(self.active_reset_flags) >= 1: + self.create_reset_ticket() if self.preview and self.commit: self.commit = False @@ -85,6 +179,7 @@ class Command(BaseCommand): qset = qset.filter(id__in=ixlan_ids) total_log = {"data": [], "errors": []} + total_notifications = [] for ixlan in qset: self.log( @@ -115,6 +210,7 @@ class Command(BaseCommand): for err in importer.log["errors"] ] ) + total_notifications += importer.notifications except Exception as inst: self.log("ERROR: {}".format(inst)) @@ -122,3 +218,11 @@ class Command(BaseCommand): if self.preview: self.stdout.write(json.dumps(total_log, indent=2)) + + # send cosolidated notifications to ix and net for + # new proposals (#771) + + importer = ixf.Importer() + importer.reset(save=self.commit) + importer.notifications = total_notifications + importer.notify_proposals() diff --git a/peeringdb_server/migrations/0046_ixfmemberdata_requirement_of.py b/peeringdb_server/migrations/0046_ixfmemberdata_requirement_of.py new file mode 100644 index 00000000..4b1c9c3b --- /dev/null +++ b/peeringdb_server/migrations/0046_ixfmemberdata_requirement_of.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.14 on 2020-07-22 09:07 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('peeringdb_server', '0045_fac_rencode_null_out'), + ] + + operations = [ + migrations.AddField( + model_name='ixfmemberdata', + name='requirement_of', + field=models.ForeignKey(blank=True, help_text='Requirement of another IXFMemberData entry and will be applied alongside it', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='requirement_set', to='peeringdb_server.IXFMemberData'), + ), + migrations.AlterField( + model_name='ixlan', + name='ixf_ixp_member_list_url', + field=models.URLField(blank=True, null=True, verbose_name='IX-F Member Export URL'), + ), + ] diff --git a/peeringdb_server/migrations/0047_ixf_import_email.py b/peeringdb_server/migrations/0047_ixf_import_email.py new file mode 100644 index 00000000..327c8068 --- /dev/null +++ b/peeringdb_server/migrations/0047_ixf_import_email.py @@ -0,0 +1,31 @@ +# Generated by Django 2.2.14 on 2020-07-23 17:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('peeringdb_server', '0046_ixfmemberdata_requirement_of'), + ] + + operations = [ + migrations.CreateModel( + name='IXFImportEmail', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('subject', models.CharField(max_length=255)), + ('message', models.TextField()), + ('recipients', models.CharField(max_length=255)), + ('created', models.DateTimeField(auto_now_add=True)), + ('sent', models.DateTimeField(blank=True, null=True)), + ('ix', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ix_email_set', to='peeringdb_server.InternetExchange', blank=True, null=True)), + ('net', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='network_email_set', to='peeringdb_server.Network', blank=True, null=True)), + ], + options={ + 'verbose_name': 'IXF Import Email', + 'verbose_name_plural': 'IXF Import Emails', + } + ), + ] diff --git a/peeringdb_server/migrations/0048_environmentsetting.py b/peeringdb_server/migrations/0048_environmentsetting.py new file mode 100644 index 00000000..a801e524 --- /dev/null +++ b/peeringdb_server/migrations/0048_environmentsetting.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.14 on 2020-07-24 12:13 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('peeringdb_server', '0047_ixf_import_email'), + ] + + operations = [ + migrations.CreateModel( + name='EnvironmentSetting', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('setting', models.CharField(choices=[('IXF_IMPORTER_DAYS_UNTIL_TICKET', 'IX-F Importer: Days until DeskPRO ticket is created')], max_length=255, unique=True)), + ('value_str', models.CharField(blank=True, max_length=255, null=True)), + ('value_int', models.IntegerField(blank=True, null=True)), + ('value_bool', models.BooleanField(blank=True, default=False)), + ('value_float', models.FloatField(blank=True, null=True)), + ('updated', models.DateTimeField(auto_now=True, null=True, verbose_name='Last Updated')), + ('created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Configured on')), + ('user', models.ForeignKey(help_text='Last updated by this user', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='admincom_setting_set', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Environment Setting', + 'verbose_name_plural': 'Environment Settings', + 'db_table': 'peeringdb_settings', + }, + ), + ] diff --git a/peeringdb_server/migrations/0049_deskrpo_ticket_additions.py b/peeringdb_server/migrations/0049_deskrpo_ticket_additions.py new file mode 100644 index 00000000..31f8ff4a --- /dev/null +++ b/peeringdb_server/migrations/0049_deskrpo_ticket_additions.py @@ -0,0 +1,38 @@ +# Generated by Django 2.2.14 on 2020-07-24 13:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('peeringdb_server', '0048_environmentsetting'), + ] + + operations = [ + migrations.AddField( + model_name='deskproticket', + name='deskpro_id', + field=models.IntegerField(blank=True, help_text='Ticket id on the DeskPRO side', null=True), + ), + migrations.AddField( + model_name='deskproticket', + name='deskpro_ref', + field=models.CharField(blank=True, help_text='Ticket reference on the DeskPRO side', max_length=32, null=True), + ), + migrations.AddField( + model_name='ixfmemberdata', + name='deskpro_id', + field=models.IntegerField(blank=True, help_text='Ticket id on the DeskPRO side', null=True), + ), + migrations.AddField( + model_name='ixfmemberdata', + name='deskpro_ref', + field=models.CharField(blank=True, help_text='Ticket reference on the DeskPRO side', max_length=32, null=True), + ), + migrations.AlterField( + model_name='deskproticket', + name='published', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/peeringdb_server/mock.py b/peeringdb_server/mock.py index 194de8e1..8eb3215a 100644 --- a/peeringdb_server/mock.py +++ b/peeringdb_server/mock.py @@ -48,14 +48,14 @@ class Mock: # Pool of IPv4 addresses (100 per prefix) self.ipaddr_pool_v4 = { - prefix: list(get_hosts(ipaddress.IPv4Network(prefix))) - for prefix in self.prefix_pool_v4 + prefix: list(get_hosts(ipaddress.IPv4Network(prefix))) + for prefix in self.prefix_pool_v4 } # Pool of IPv6 addresses (100 per prefix) self.ipaddr_pool_v6 = { - prefix: list(get_hosts(ipaddress.IPv6Network(prefix))) - for prefix in self.prefix_pool_v6 + prefix: list(get_hosts(ipaddress.IPv6Network(prefix))) + for prefix in self.prefix_pool_v6 } def create(self, reftag, **kwargs): diff --git a/peeringdb_server/models.py b/peeringdb_server/models.py index 5f25f1e5..b30e16cb 100644 --- a/peeringdb_server/models.py +++ b/peeringdb_server/models.py @@ -22,6 +22,7 @@ from django.utils.html import strip_tags from django.utils.http import urlquote from django.utils.translation import ugettext_lazy as _ from django.utils.translation import override +from django.utils.functional import Promise from django.conf import settings from django.template import loader from django_namespace_perms.util import autodiscover_namespaces, has_perms @@ -177,6 +178,17 @@ class URLField(pdb_models.URLField): pass +class ValidationErrorEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, ValidationError): + if hasattr(obj, "error_dict"): + return obj.error_dict + return obj.message + elif isinstance(obj, Promise): + return f"{obj}" + return super().default(obj) + + class ProtectedAction(ValueError): def __init__(self, obj): super().__init__(obj.not_deletable_reason) @@ -553,7 +565,18 @@ class DeskProTicket(models.Model): body = models.TextField() user = models.ForeignKey("peeringdb_server.User", on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) - published = models.DateTimeField(null=True) + published = models.DateTimeField(null=True, blank=True) + + deskpro_ref = models.CharField( + max_length=32, + null=True, + blank=True, + help_text=_("Ticket reference on the DeskPRO side"), + ) + + deskpro_id = models.IntegerField( + null=True, blank=True, help_text=_("Ticket id on the DeskPRO side") + ) class Meta: verbose_name = _("DeskPRO Ticket") @@ -1123,7 +1146,9 @@ class Facility(ProtectedMixin, pdb_models.FacilityBase, GeocodeBaseMixin): @classmethod def nsp_namespace_from_id(cls, org_id, fac_id): - return "{}.facility.{}".format(Organization.nsp_namespace_from_id(org_id), fac_id) + return "{}.facility.{}".format( + Organization.nsp_namespace_from_id(org_id), fac_id + ) @classmethod def related_to_net(cls, value=None, filt=None, field="network_id", qset=None): @@ -1505,8 +1530,7 @@ class InternetExchange(ProtectedMixin, pdb_models.InternetExchangeBase): Returns permissioning namespace for an exchange """ return "{}.internetexchange.{}".format( - Organization.nsp_namespace_from_id(org_id), - ix_id, + Organization.nsp_namespace_from_id(org_id), ix_id, ) @property @@ -1700,7 +1724,9 @@ class InternetExchangeFacility(pdb_models.InternetExchangeFacilityBase): """ Returns permissioning namespace for an ixfac """ - return "{}.fac.{}".format(InternetExchange.nsp_namespace_from_id(org_id, ix_id), id) + return "{}.fac.{}".format( + InternetExchange.nsp_namespace_from_id(org_id, ix_id), id + ) @property def nsp_namespace(self): @@ -1955,11 +1981,11 @@ class IXLan(pdb_models.IXLanBase): # and bail if ipv4 and not ipv4_valid: raise ValidationError( - f"IPv4 {ipv4} does not match any prefix " "on this ixlan" + {"ipaddr4": f"IPv4 {ipv4} does not match any prefix on this ixlan"} ) if ipv6 and not ipv6_valid: raise ValidationError( - f"IPv6 {ipv6} does not match any prefix " "on this ixlan" + {"ipaddr6": f"IPv6 {ipv6} does not match any prefix on this ixlan"} ) # Next we check if an active netixlan with the ipaddress exists in ANOTHER lan, and bail @@ -1971,7 +1997,9 @@ class IXLan(pdb_models.IXLanBase): .count() > 0 ): - raise ValidationError(f"Ip address {ipv4} already exists in another lan") + raise ValidationError( + {"ipaddr4": f"Ip address {ipv4} already exists in another lan"} + ) if ( ipv6 @@ -1980,7 +2008,9 @@ class IXLan(pdb_models.IXLanBase): .count() > 0 ): - raise ValidationError(f"Ip address {ipv6} already exists in another lan") + raise ValidationError( + {"ipaddr6": f"Ip address {ipv6} already exists in another lan"} + ) # now we need to figure out if the ipaddresses already exist in this ixlan, # we need to check ipv4 and ipv6 separately as they might exist on different @@ -2072,6 +2102,11 @@ class IXLan(pdb_models.IXLanBase): netixlan.is_rs_peer = netixlan_info.is_rs_peer changed.append("is_rs_peer") + # Is the netixlan operational? + if netixlan_info.operational != netixlan.operational: + netixlan.operational = netixlan_info.operational + changed.append("operational") + # Speed if netixlan_info.speed != netixlan.speed and ( netixlan_info.speed > 0 or netixlan.speed is None @@ -2089,11 +2124,9 @@ class IXLan(pdb_models.IXLanBase): netixlan.network = netixlan_info.network changed.append("network_id") - # Finally we attempt to validate the data and then save the netixlan instance - netixlan.full_clean() - - if save and changed: + if save and (changed or netixlan.status == "deleted"): netixlan.status = "ok" + netixlan.full_clean() netixlan.save() return result(netixlan) @@ -2231,6 +2264,13 @@ class IXLanIXFMemberImportLogEntry(models.Model): return 1 +class NetworkProtocolsDisabled(ValueError): + """ + raised when a network has both ipv6 and ipv4 support + disabled during ix-f import + """ + + class IXFMemberData(pdb_models.NetworkIXLanBase): """ @@ -2267,6 +2307,29 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): ixlan = models.ForeignKey(IXLan, related_name="ixf_set", on_delete=models.CASCADE) + requirement_of = models.ForeignKey( + "self", + on_delete=models.CASCADE, + related_name="requirement_set", + null=True, + blank=True, + help_text=_( + "Requirement of another IXFMemberData entry " + "and will be applied alongside it" + ), + ) + + deskpro_ref = models.CharField( + max_length=32, + null=True, + blank=True, + help_text=_("Ticket reference on the DeskPRO side"), + ) + + deskpro_id = models.IntegerField( + null=True, blank=True, help_text=_("Ticket id on the DeskPRO side") + ) + # field names of fields that can receive # modifications from ix-f @@ -2285,24 +2348,31 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): tag = "ixfmember" @classmethod - def id_filters(cls, asn, ipaddr4, ipaddr6): + def id_filters(cls, asn, ipaddr4, ipaddr6, check_protocols=True): """ returns a dict of filters to use with a IXFMemberData or NetworkIXLan query set to retrieve a unique entry """ + net = Network.objects.get(asn=asn) + + ipv4_support = net.ipv4_support or not check_protocols + ipv6_support = net.ipv6_support or not check_protocols + filters = {"asn": asn} - if ipaddr4 is None: - filters["ipaddr4__isnull"] = True - else: - filters["ipaddr4"] = ipaddr4 + if ipv4_support: + if ipaddr4: + filters["ipaddr4"] = ipaddr4 + else: + filters["ipaddr4__isnull"] = True - if ipaddr6 is None: - filters["ipaddr6__isnull"] = True - else: - filters["ipaddr6"] = ipaddr6 + if ipv6_support: + if ipaddr6: + filters["ipaddr6"] = ipaddr6 + else: + filters["ipaddr6__isnull"] = True return filters @@ -2325,9 +2395,32 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): """ fetched = datetime.datetime.now().replace(tzinfo=UTC()) + net = Network.objects.get(asn=asn) + validate_network_protocols = kwargs.get("validate_network_protocols", True) + for_deletion = kwargs.get("delete", False) try: - instance = cls.objects.get(**cls.id_filters(asn, ipaddr4, ipaddr6)) + id_filters = cls.id_filters(asn, ipaddr4, ipaddr6) + + instances = cls.objects.filter(**id_filters) + + if not instances.exists(): + raise cls.DoesNotExist() + + if instances.count() > 1: + + # this only happens when a network switches on/off + # ipv4/ipv6 protocol support inbetween importer + # runs. + + for instance in instances: + if ipaddr4 != instance.ipaddr4 or ipaddr6 != instance.ipaddr6: + instance.delete(hard=True) + + instance = cls.objects.get(**id_filters) + else: + instance = instances.first() + for field in cls.data_fields: setattr(instance, f"previous_{field}", getattr(instance, field)) @@ -2340,13 +2433,39 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): instance._meta.get_field("updated").auto_now = True except cls.DoesNotExist: - instance = cls(asn=asn, ipaddr4=ipaddr4, ipaddr6=ipaddr6, status="ok") + ip_args = {} + + if net.ipv4_support or not ipaddr4 or for_deletion: + ip_args.update(ipaddr4=ipaddr4) + + if net.ipv6_support or not ipaddr6 or for_deletion: + ip_args.update(ipaddr6=ipaddr6) + + if not ip_args and validate_network_protocols: + raise NetworkProtocolsDisabled( + _( + "No suitable ipaddresses when validating against the enabled network protocols" + ) + ) + + instance = cls(asn=asn, status="ok", **ip_args) instance.speed = kwargs.get("speed", 0) instance.operational = kwargs.get("operational", True) instance.is_rs_peer = kwargs.get("is_rs_peer", False) instance.ixlan = ixlan instance.fetched = fetched + instance.for_deletion = for_deletion + + if ipaddr4: + instance.init_ipaddr4 = ipaddress.ip_address(ipaddr4) + else: + instance.init_ipaddr4 = None + + if ipaddr6: + instance.init_ipaddr6 = ipaddress.ip_address(ipaddr6) + else: + instance.init_ipaddr6 = None if "data" in kwargs: instance.set_data(kwargs.get("data")) @@ -2419,12 +2538,18 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): action = ixf_member_data.action error = ixf_member_data.error + # not actionable for anyone + if action == "noop": continue + # not actionable for network + if not ixf_member_data.actionable_for_network: continue + # dismissed by network + if ixf_member_data.dismissed: continue @@ -2480,8 +2605,60 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): if error and "address outside of prefix" in error: return False + if error and "does not match any prefix" in error: + return False + + if error and "speed value" in error: + return False + return True + @property + def actionable_error(self): + """ + Returns whether or not the error is actionable + by exchange or network. + + If actionable will return self.error otherwise + will return None + """ + + if not self.error: + return None + + try: + error_data = json.loads(self.error) + except: + return None + + IPADDR_EXIST = "already exists" + DELETED_NETIXLAN_BAD_ASN = "This entity was created for the ASN" + + if IPADDR_EXIST in error_data.get("ipaddr4", [""])[0]: + for requirement in self.requirements: + if requirement.netixlan.ipaddr4 == self.ipaddr4: + return None + if NetworkIXLan.objects.filter( + ipaddr4=self.ipaddr4, status="deleted" + ).exists(): + return None + + if IPADDR_EXIST in error_data.get("ipaddr6", [""])[0]: + for requirement in self.requirements: + if requirement.netixlan.ipaddr6 == self.ipaddr6: + return None + + if NetworkIXLan.objects.filter( + ipaddr6=self.ipaddr6, status="deleted" + ).exists(): + return None + + if DELETED_NETIXLAN_BAD_ASN in error_data.get("__all__", [""])[0]: + if self.netixlan.status == "deleted": + return None + + return self.error + @property def net_contacts(self): """ @@ -2492,7 +2669,7 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): qset = self.net.poc_set_active.exclude(email="") qset = qset.exclude(email__isnull=True) - role_priority = ["Policy", "Technical", "NOC", "Maintenance"] + role_priority = ["Technical", "NOC", "Policy"] contacts = [] @@ -2541,6 +2718,23 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): return f"AS{self.asn} - {ipaddr4} - {ipaddr6}" + @property + def actionable_changes(self): + + requirements = self.requirements + + _changes = self.changes + + for requirement in self.requirements: + _changes.update(self._changes(requirement.netixlan)) + + if self.ipaddr4_on_requirement: + _changes.update(ipaddr4=self.ipaddr4_on_requirement) + if self.ipaddr6_on_requirement: + _changes.update(ipaddr6=self.ipaddr6_on_requirement) + + return _changes + @property def changes(self): @@ -2561,9 +2755,13 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): """ netixlan = self.netixlan + return self._changes(netixlan) + + def _changes(self, netixlan): + changes = {} - if self.marked_for_removal: + if self.marked_for_removal or not netixlan: return changes if netixlan.is_rs_peer != self.is_rs_peer: @@ -2676,20 +2874,111 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): has_data = self.remote_data_missing == False + action = "noop" + if has_data: if not self.netixlan.id: - return "add" + action = "add" - if self.status == "ok" and self.netixlan.status == "deleted": - return "add" + elif self.status == "ok" and self.netixlan.status == "deleted": + action = "add" - if self.changes: - return "modify" + elif self.changes: + action = "modify" else: if self.marked_for_removal: - return "delete" + action = "delete" - return "noop" + # the proposal is to add a netixlan, but we have + # the requirement of a deletion of another netixlan + # that has one of the ips set but not the other. + # + # action re-classified to modify (#770) + if action == "add" and self.has_requirements: + action = "modify" + + return action + + @property + def has_requirements(self): + """ + Return whether or not this IXFMemberData has + other IXFMemberData objects as requirements + """ + + return len(self.requirements) > 0 + + @property + def requirements(self): + """ + Returns list of all IXFMemberData objects + that are still active requirements for this + IXFMemberData object + """ + return [ + requirement + for requirement in self.requirement_set.all() + # if requirement.action != "noop" + ] + + @property + def primary_requirement(self): + """ + Return the initial requirement IXFMemberData + for this IXFMemberData instance, None if there + isn't any + """ + try: + return self.requirements[0] + except IndexError: + return None + + @property + def secondary_requirements(self): + """ + Return a list of secondary requirement IXFMemberData + objects for this IXFMemberData object. Currently this + only happens on add proposals that require two netixlans + to be deleted because both ipaddresses exist on separate + netixlans (#770) + """ + return self.requirements[1:] + + @property + def ipaddr4_on_requirement(self): + """ + Returns true if the ipv4 address claimed by this IXFMemberData + object exists on one of it's requirement IXFMemberData objects + """ + + ipaddr4 = self.ipaddr4 + if not ipaddr4 and hasattr(self, "init_ipaddr4"): + ipaddr4 = self.init_ipaddr4 + + if not ipaddr4: + return False + for requirement in self.requirements: + if requirement.ipaddr4 == ipaddr4: + return True + return False + + @property + def ipaddr6_on_requirement(self): + """ + Returns true if the ipv6 address claimed by this IXFMemberData + object exists on one of it's requirement IXFMemberData objects + """ + + ipaddr6 = self.ipaddr6 + if not ipaddr6 and hasattr(self, "init_ipaddr6"): + ipaddr6 = self.init_ipaddr6 + + if not ipaddr6: + return False + for requirement in self.requirements: + if requirement.ipaddr6 == ipaddr6: + return True + return False @property def netixlan(self): @@ -2706,19 +2995,20 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): """ if not hasattr(self, "_netixlan"): + + if not hasattr(self, "for_deletion"): + self.for_deletion = self.remote_data_missing + try: - - filters = {"asn": self.asn} - - if self.ipaddr4 is None: - filters["ipaddr4__isnull"] = True + if self.for_deletion: + filters = self.id_filters( + self.asn, self.ipaddr4, self.ipaddr6, check_protocols=False + ) else: - filters["ipaddr4"] = self.ipaddr4 + filters = self.id_filters(self.asn, self.ipaddr4, self.ipaddr6) - if self.ipaddr6 is None: - filters["ipaddr6__isnull"] = True - else: - filters["ipaddr6"] = self.ipaddr6 + if "ipaddr6" not in filters and "ipaddr4" not in filters: + raise NetworkIXLan.DoesNotExist() self._netixlan = NetworkIXLan.objects.get(**filters) except NetworkIXLan.DoesNotExist: @@ -2744,52 +3034,6 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): """ return self.netixlan.id and self.netixlan.status != "deleted" - @property - def ticket_user(self): - """ - Returns the User instance for the user to use - to create DeskPRO tickets - """ - if not hasattr(self, "_ticket_user"): - self._ticket_user = User.objects.get(username="ixf_importer") - return self._ticket_user - - @property - def tickets_enabled(self): - """ - Returns whether or not deskpr ticket creation for ix-f - conflicts are enabled or not - - This can be controlled by the IXF_TICKET_ON_CONFLICT - setting - """ - - return getattr(settings, "IXF_TICKET_ON_CONFLICT", True) - - @property - def notify_ix_enabled(self): - """ - Returns whether or not notifications to the exchange - are enabled. - - This can be controlled by the IXF_NOTIFY_IX_ON_CONFLICT - setting - """ - - return getattr(settings, "IXF_NOTIFY_IX_ON_CONFLICT", False) - - @property - def notify_net_enabled(self): - """ - Returns whether or not notifications to the network - are enabled. - - This can be controlled by the IXF_NOTIFY_NET_ON_CONFLICT - setting - """ - - return getattr(settings, "IXF_NOTIFY_NET_ON_CONFLICT", False) - def __str__(self): parts = [ self.ixlan.ix.name, @@ -2808,7 +3052,35 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): return " ".join(parts) - @reversion.create_revision() + def set_requirement(self, ixf_member_data, save=True): + """ + Sets another IXFMemberData object to be a requirement + of the resolution of this IXFMemberData object + """ + if not ixf_member_data: + return + + if ixf_member_data in self.requirements: + return + + if ixf_member_data.netixlan == self.netixlan: + return + + ixf_member_data.requirement_of = self + + if save: + ixf_member_data.save() + + return ixf_member_data + + def apply_requirements(self, save=True): + """ + Apply all requirements + """ + + for requirement in self.requirements: + requirement.apply(save=save) + def apply(self, user=None, comment=None, save=True): """ Applies the data. @@ -2842,6 +3114,8 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): if comment: reversion.set_comment(comment) + self.apply_requirements(save=save) + action = self.action netixlan = self.netixlan changes = self.changes @@ -2850,7 +3124,13 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): self.validate_speed() + if not self.net.ipv6_support: + netixlan.ipaddr6 = None + if not self.net.ipv4_support: + netixlan.ipaddr4 = None + result = self.ixlan.add_netixlan(netixlan, save=save, save_others=save) + self._netixlan = netixlan = result["netixlan"] elif action == "modify": @@ -2859,17 +3139,15 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): netixlan.speed = self.speed netixlan.is_rs_peer = self.is_rs_peer netixlan.operational = self.operational - netixlan.full_clean() if save: + netixlan.full_clean() netixlan.save() elif action == "delete": + if save: netixlan.delete() - if save: - self.set_resolved() - - return {"action": action, "netixlan": netixlan} + return {"action": action, "netixlan": netixlan, "ixf_member_data": self} def validate_speed(self): """ @@ -2883,8 +3161,9 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): TODO: find a better way to do this """ if self.speed == 0 and self.error: - if "Invalid speed value" in self.error: - raise ValidationError({"speed": self.error}) + error_data = json.loads(self.error) + if "speed" in self.error: + raise ValidationError(error_data) def save_without_update(self): self._meta.get_field("updated").auto_now = False @@ -2901,7 +3180,7 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): try: self.netixlan.full_clean() except ValidationError as exc: - self.error = f"{exc}" + self.error = json.dumps(exc, cls=ValidationErrorEncoder) def set_resolved(self, save=True): """ @@ -2911,9 +3190,9 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): this will delete the IXFMemberData instance """ - if self.id and save: - self.notify_resolve(ac=True, ix=True, net=True) + if self.id and save and not self.requirement_of: self.delete(hard=True) + return True def set_conflict(self, error=None, save=True): """ @@ -2922,11 +3201,32 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): to the corresponding netixlan to ac, ix and net as warranted as warranted """ + + if not self.id: + existing_conflict = IXFMemberData.objects.filter( + asn=self.asn, error__isnull=False + ) + + if self.ipaddr4 and self.ipaddr6: + existing_conflict = existing_conflict.filter( + models.Q(ipaddr4=self.ipaddr4) | models.Q(ipaddr6=self.ipaddr6) + ) + elif self.ipaddr4: + existing_conflict = existing_conflict.filter(ipaddr4=self.ipaddr4) + elif self.ipaddr6: + existing_conflict = existing_conflict.filter(ipaddr6=self.ipaddr6) + + if existing_conflict.exists(): + return None + if (self.remote_changes or (error and not self.previous_error)) and save: - self.error = error + if error: + self.error = json.dumps(error, cls=ValidationErrorEncoder) + else: + self.error = None self.dismissed = False self.save() - self.notify_update(ac=True, ix=True, net=True) + return True elif self.previous_data != self.data and save: # since remote_changes only tracks changes to the @@ -2947,7 +3247,7 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): self.grab_validation_errors() self.dismissed = False self.save() - self.notify_update(ac=True, ix=True, net=True) + return True elif self.previous_data != self.data and save: # since remote_changes only tracks changes to the @@ -2964,13 +3264,11 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): as warranted """ self.reason = reason + if not self.id and save: self.grab_validation_errors() self.save() - if self.net_present_at_ix: - self.notify_add(ac=True, ix=True, net=True) - else: - self.notify_add(net=True) + return True elif self.previous_data != self.data and save: @@ -3007,7 +3305,7 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): if (not_saved or gone) and save: self.set_data({}) self.save() - self.notify_remove(ac=True, ix=True, net=True) + return True def set_data(self, data): """ @@ -3015,23 +3313,15 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): """ self.data = json.dumps(data) - def notify(self, template_file, recipient, subject, context=None): + def render_notification(self, template_file, recipient, context=None): """ - Send notification - - Returns a dict containing information about the notification - - - contacts(list) - - subject(str) - - message(str) + Renders notification text for this ixfmemberdata + instance Argument(s): - template_file(str): email template file - recipient(str): ac, ix or net - - subject(str): subject text, this will only be used if - this IXFMemberData instance does not have a valid - asn, ip4, ip6 identifier - context(dict): if set will update the template context from this """ @@ -3044,110 +3334,7 @@ class IXFMemberData(pdb_models.NetworkIXLanBase): _context.update(context) template = loader.get_template(template_file) - message = template.render(_context) - - if self.asn and (self.ipaddr4 or self.ipaddr6): - subject = f"{settings.EMAIL_SUBJECT_PREFIX}[IX-F] {self}" - else: - subject = f"{settings.EMAIL_SUBJECT_PREFIX}[IX-F] {subject}" - - contacts = [] - - if recipient == "ac" and self.tickets_enabled: - contacts = [ - DeskProTicket.objects.create( - subject=subject, body=message, user=self.ticket_user, - ) - ] - elif recipient == "net" and self.actionable_for_network: - if self.notify_net_enabled: - contacts = self.net_contacts - self._email(subject, message, contacts) - elif recipient == "ix" and self.notify_ix_enabled: - contacts = self.ix_contacts - self._email(subject, message, contacts) - - return {"contacts": contacts, "subject": subject, "message": message} - - def _email(self, subject, message, recipients): - """ - Send email - - Honors the MAIL_DEBUG setting - - Called by self.notify depending on recipient - type - """ - - if not getattr(settings, "MAIL_DEBUG", False): - mail = EmailMultiAlternatives( - subject, strip_tags(message), settings.DEFAULT_FROM_EMAIL, recipients, - ) - mail.send(fail_silently=False) - else: - debug_mail( - subject, message, settings.DEFAULT_FROM_EMAIL, recipients, - ) - - def _notify(self, template_file, subject, context=None, save=True, **recipients): - - """ - Send notification to multiple recipient types - - Argument(s): - - - template_file(str): email template file - - subject(str): subject text, this will only be used if - this IXFMemberData instance does not have a valid - asn, ip4, ip6 identifier - - context(dict): if set will update the template context - from this - - save(bool=True): if True will save this IXFMemberData instance - (self.log will have been updated with notification lines) - - Keyword Argument(s): - - - ac(bool): if True send notification to admin committee - - net(bool): if True send notification to network - - ix(bool): if True send notification to exchange - """ - - log = [] - now = datetime.datetime.now().replace(tzinfo=UTC()) - - for recipient in ["ac", "ix", "net"]: - if not recipients.get(recipient): - continue - result = self.notify(template_file, recipient, subject, context,) - - if result["contacts"]: - log.append( - f"[{now}] notified {recipient} ({result['contacts']}) about {subject}" - ) - else: - log.append( - f"[{now}] could not notify {recipient} about {subject}: no suitable contacts found" - ) - - self.log = "\n".join(log) + "\n" + self.log - - if save: - self.save() - - def notify_resolve(self, **kwargs): - return self._notify("email/notify-ixf-resolved.txt", "Resolved", **kwargs) - - def notify_remote_changes(self, **kwargs): - return self._notify("email/notify-ixf-update.txt", _("IX-F changed"), **kwargs) - - def notify_update(self, **kwargs): - return self._notify("email/notify-ixf-update.txt", _("Modify"), **kwargs) - - def notify_add(self, **kwargs): - return self._notify("email/notify-ixf-add.txt", _("Add"), **kwargs) - - def notify_remove(self, **kwargs): - return self._notify("email/notify-ixf-remove.txt", _("Remove"), **kwargs) + return template.render(_context) @property def ac_netixlan_url(self): @@ -3195,8 +3382,7 @@ class IXLanPrefix(ProtectedMixin, pdb_models.IXLanPrefixBase): Returns permissioning namespace for an ixpfx """ return "{}.prefix.{}".format( - IXLan.nsp_namespace_from_id(org_id, ix_id, ixlan_id), - id, + IXLan.nsp_namespace_from_id(org_id, ix_id, ixlan_id), id, ) @classmethod @@ -3374,7 +3560,9 @@ class Network(pdb_models.NetworkBase): @classmethod def nsp_namespace_from_id(cls, org_id, net_id): - return "{}.network.{}".format(Organization.nsp_namespace_from_id(org_id), net_id) + return "{}.network.{}".format( + Organization.nsp_namespace_from_id(org_id), net_id + ) @classmethod def related_to_fac(cls, value=None, filt=None, field="facility_id", qset=None): @@ -3567,6 +3755,28 @@ class Network(pdb_models.NetworkBase): }, } + @property + def ipv4_support(self): + + # network has not indicated either ip4 or ip6 support + # so assume True (#771) + + if not self.info_unicast and not self.info_ipv6: + return True + + return self.info_unicast + + @property + def ipv6_support(self): + + # network has not indicated either ip4 or ip6 support + # so assume True (#771) + + if not self.info_unicast and not self.info_ipv6: + return True + + return self.info_ipv6 + @property def sponsorship(self): return self.org.sponsorship @@ -3580,6 +3790,15 @@ class Network(pdb_models.NetworkBase): settings.BASE_URL, django.urls.reverse("net-view", args=(self.id,)) ) + @property + def view_url_asn(self): + """ + Return the URL to this networks web view + """ + return "{}{}".format( + settings.BASE_URL, django.urls.reverse("net-view-asn", args=(self.asn,)) + ) + def nsp_has_perms_PUT(self, user, request): return validate_PUT_ownership(user, self, request.data, ["org"]) @@ -3640,7 +3859,9 @@ class NetworkContact(pdb_models.ContactBase): """ Returns permissioning namespace for a network contact """ - return "{}.poc_set.{}".format(Network.nsp_namespace_from_id(org_id, net_id), vis) + return "{}.poc_set.{}".format( + Network.nsp_namespace_from_id(org_id, net_id), vis + ) @property def nsp_namespace(self): @@ -3834,8 +4055,17 @@ class NetworkIXLan(pdb_models.NetworkIXLanBase): as a unqiue record by asn, ip4 and ip6 address """ + net = self.network return (self.asn, self.ipaddr4, self.ipaddr6) + @property + def ixf_id_pretty_str(self): + asn, ipaddr4, ipaddr6 = self.ixf_id + ipaddr4 = ipaddr4 or _("IPv4 not set") + ipaddr6 = ipaddr6 or _("IPv6 not set") + + return f"AS{asn} - {ipaddr4} - {ipaddr6}" + # FIXME # permission namespacing # right now it is assumed that the network owns the netixlan @@ -3846,7 +4076,9 @@ class NetworkIXLan(pdb_models.NetworkIXLanBase): """ Returns permissioning namespace for a netixlan """ - return "{}.ixlan.{}".format(Network.nsp_namespace_from_id(org_id, net_id), ixlan_id) + return "{}.ixlan.{}".format( + Network.nsp_namespace_from_id(org_id, net_id), ixlan_id + ) @property def nsp_namespace(self): @@ -4318,6 +4550,36 @@ def password_reset_token(): return token, hashed +class IXFImportEmail(models.Model): + """ + A copy of all emails sent by the IX-F importer. + """ + + subject = models.CharField(max_length=255, blank=False) + message = models.TextField(blank=False) + recipients = models.CharField(max_length=255, blank=False) + created = models.DateTimeField(auto_now_add=True) + sent = models.DateTimeField(blank=True, null=True) + net = models.ForeignKey( + Network, + on_delete=models.CASCADE, + related_name="network_email_set", + blank=True, + null=True, + ) + ix = models.ForeignKey( + InternetExchange, + on_delete=models.CASCADE, + related_name="ix_email_set", + blank=True, + null=True, + ) + + class Meta: + verbose_name = _("IXF Import Email") + verbose_name_plural = _("IXF Import Emails") + + class UserPasswordReset(models.Model): class Meta: db_table = "peeringdb_user_password_reset" @@ -4390,20 +4652,100 @@ class CommandLineTool(models.Model): self.status = "running" +class EnvironmentSetting(models.Model): + + """ + Environment settings overrides controlled through + django admin (/cp) + """ + + class Meta: + db_table = "peeringdb_settings" + verbose_name = _("Environment Setting") + verbose_name_plural = _("Environment Settings") + + setting = models.CharField( + max_length=255, + choices=( + ( + "IXF_IMPORTER_DAYS_UNTIL_TICKET", + _("IX-F Importer: Days until DeskPRO ticket is created"), + ), + ), + unique=True, + ) + + value_str = models.CharField(max_length=255, blank=True, null=True) + value_int = models.IntegerField(blank=True, null=True) + value_bool = models.BooleanField(blank=True, default=False) + value_float = models.FloatField(blank=True, null=True) + + updated = models.DateTimeField( + _("Last Updated"), auto_now=True, null=True, blank=True, + ) + + created = models.DateTimeField( + _("Configured on"), auto_now_add=True, blank=True, null=True, + ) + + user = models.ForeignKey( + User, + null=True, + on_delete=models.SET_NULL, + related_name="admincom_setting_set", + help_text=_("Last updated by this user"), + ) + + setting_to_field = { + "IXF_IMPORTER_DAYS_UNTIL_TICKET": "value_int", + } + + @classmethod + def get_setting_value(cls, setting): + + """ + Get the current value of the setting specified by + it's setting name + + If no instance has been saved for the specified setting + the default value will be returned + """ + try: + instance = cls.objects.get(setting=setting) + return instance.value + except cls.DoesNotExist: + return getattr(settings, setting) + + @property + def value(self): + """ + Get the value for this setting + """ + return getattr(self, self.setting_to_field[self.setting]) + + def set_value(self, value): + """ + Update the value for this setting + """ + setattr(self, self.setting_to_field[self.setting], value) + self.full_clean() + self.save() + + REFTAG_MAP = { - cls.handleref.tag: cls - for cls in [ - Organization, - Network, - Facility, - InternetExchange, - InternetExchangeFacility, - NetworkFacility, - NetworkIXLan, - NetworkContact, - IXLan, - IXLanPrefix, - ] + cls.handleref.tag: cls + for cls in [ + Organization, + Network, + Facility, + InternetExchange, + InternetExchangeFacility, + NetworkFacility, + NetworkIXLan, + NetworkContact, + IXLan, + IXLanPrefix, + ] } diff --git a/peeringdb_server/org_admin_views.py b/peeringdb_server/org_admin_views.py index 0e7896e9..84231afb 100644 --- a/peeringdb_server/org_admin_views.py +++ b/peeringdb_server/org_admin_views.py @@ -113,10 +113,8 @@ def load_user_permissions(org, user): # load all of the user's permissions related to this org uperms = { - p.namespace: p.permissions - for p in user.userpermission_set.filter( - namespace__startswith=org.nsp_namespace - ) + p.namespace: p.permissions + for p in user.userpermission_set.filter(namespace__startswith=org.nsp_namespace) } perms = {} @@ -158,24 +156,22 @@ def permission_ids(org): perms.update( { - "net.%d" % net.id: - _("Network - %(net_name)s") % {"net_name": net.name} - for net in org.net_set_active + "net.%d" % net.id: _("Network - %(net_name)s") % {"net_name": net.name} + for net in org.net_set_active } ) perms.update( { - "ix.%d" % ix.id: _("Exchange - %(ix_name)s") % {"ix_name": ix.name} - for ix in org.ix_set_active + "ix.%d" % ix.id: _("Exchange - %(ix_name)s") % {"ix_name": ix.name} + for ix in org.ix_set_active } ) perms.update( { - "fac.%d" % fac.id: - _("Facility - %(fac_name)s") % {"fac_name": fac.name} - for fac in org.fac_set_active + "fac.%d" % fac.id: _("Facility - %(fac_name)s") % {"fac_name": fac.name} + for fac in org.fac_set_active } ) diff --git a/peeringdb_server/rest.py b/peeringdb_server/rest.py index 144375ab..88078776 100644 --- a/peeringdb_server/rest.py +++ b/peeringdb_server/rest.py @@ -46,9 +46,7 @@ class DataMissingException(DataException): """ def __init__(self, method): - super().__init__( - f"No data was supplied with the {method} request" - ) + super().__init__(f"No data was supplied with the {method} request") class DataParseException(DataException): @@ -731,17 +729,17 @@ router.register("as_set", ASSetViewSet, basename="as_set") urls = router.urls REFTAG_MAP = { - cls.model.handleref.tag: cls - for cls in [ - OrganizationViewSet, - NetworkViewSet, - FacilityViewSet, - InternetExchangeViewSet, - InternetExchangeFacilityViewSet, - NetworkFacilityViewSet, - NetworkIXLanViewSet, - NetworkContactViewSet, - IXLanViewSet, - IXLanPrefixViewSet, - ] + cls.model.handleref.tag: cls + for cls in [ + OrganizationViewSet, + NetworkViewSet, + FacilityViewSet, + InternetExchangeViewSet, + InternetExchangeFacilityViewSet, + NetworkFacilityViewSet, + NetworkIXLanViewSet, + NetworkContactViewSet, + IXLanViewSet, + IXLanPrefixViewSet, + ] } diff --git a/peeringdb_server/serializers.py b/peeringdb_server/serializers.py index fc80a1fd..38b69bad 100644 --- a/peeringdb_server/serializers.py +++ b/peeringdb_server/serializers.py @@ -892,10 +892,18 @@ class ModelSerializer(PermissionedModelSerializer): # to identify a soft-deleted object that we want # to restore # - # At this point only `POST` (create) requests - # should ever attempt a restoration like this + # At this point `POST` (create) requests and + # `PUT` (update) requests are supported + # + # POST will undelete the blocking entity and re-claim it + # PUT will null the offending fields on the blocking entity - if filters and request and request.user and request.method == "POST": + if ( + filters + and request + and request.user + and request.method in ["POST", "PUT"] + ): if "fac_id" in filters: filters["facility_id"] = filters["fac_id"] @@ -906,14 +914,29 @@ class ModelSerializer(PermissionedModelSerializer): try: filters.update(status="deleted") - self.instance = self.Meta.model.objects.get(**filters) + instance = self.Meta.model.objects.get(**filters) except self.Meta.model.DoesNotExist: raise exc except FieldError as exc: raise exc + if request.method == "POST": + self.instance = instance + self._undelete = True + elif request.method == "PUT": + for field in filters.keys(): + if field == "status": + continue + setattr(instance, field, None) + + try: + # if field can't be nulled this will + # fail and raise the original error + instance.save() + except: + raise exc + rv = super().run_validation(data=data) - self._undelete = True return rv else: raise @@ -1191,9 +1214,7 @@ class InternetExchangeFacilitySerializer(ModelSerializer): raise ParentStatusException(data.get("ix"), self.Meta.model.handleref.tag) if data.get("fac") and data.get("fac").status != "ok": raise ParentStatusException(data.get("fac"), self.Meta.model.handleref.tag) - return super().has_create_perms( - user, data - ) + return super().has_create_perms(user, data) def nsp_namespace_create(self, data): return self.Meta.model.nsp_namespace_from_id( @@ -1736,7 +1757,7 @@ class NetworkSerializer(ModelSerializer): "fac_id", ], cls, - **kwargs + **kwargs, ) for field, e in list(filters.items()): @@ -2170,7 +2191,7 @@ class InternetExchangeSerializer(ModelSerializer): "net_count", ], cls, - **kwargs + **kwargs, ) for field, e in list(filters.items()): @@ -2355,17 +2376,17 @@ class OrganizationSerializer(ModelSerializer): REFTAG_MAP = { - cls.Meta.model.handleref.tag: cls - for cls in [ - OrganizationSerializer, - NetworkSerializer, - FacilitySerializer, - InternetExchangeSerializer, - InternetExchangeFacilitySerializer, - NetworkFacilitySerializer, - NetworkIXLanSerializer, - NetworkContactSerializer, - IXLanSerializer, - IXLanPrefixSerializer, - ] + cls.Meta.model.handleref.tag: cls + for cls in [ + OrganizationSerializer, + NetworkSerializer, + FacilitySerializer, + InternetExchangeSerializer, + InternetExchangeFacilitySerializer, + NetworkFacilitySerializer, + NetworkIXLanSerializer, + NetworkContactSerializer, + IXLanSerializer, + IXLanPrefixSerializer, + ] } diff --git a/peeringdb_server/static/peeringdb.js b/peeringdb_server/static/peeringdb.js index 999813c1..cfe5b291 100644 --- a/peeringdb_server/static/peeringdb.js +++ b/peeringdb_server/static/peeringdb.js @@ -444,14 +444,36 @@ PeeringDB.IXFProposals = twentyc.cls.define( modify : function(row) { var data=this.collect(row); var proposals = row.closest("[data-ixf-proposals-ix]") - var netixlan_row = this.netixlan_list.find('[data-edit-id="'+data.id+'"]') - netixlan_row.find('[data-edit-name="speed"] input').val(data.speed) - netixlan_row.find('[data-edit-name="is_rs_peer"] input').prop("checked", data.is_rs_peer) - netixlan_row.find('[data-edit-name="operational"] input').prop("checked", data.operational) - netixlan_row.addClass("newrow") - this.detach_row(row); - row.find('button').tooltip("hide") + var ixf_proposals = this; + var requirements = row.find("[data-ixf-require-delete]") + + var apply = () => { + var netixlan_row = this.netixlan_list.find('[data-edit-id="'+data.id+'"]') + netixlan_row.find('[data-edit-name="speed"] input').val(data.speed) + netixlan_row.find('[data-edit-name="ipaddr4"] input').val(data.ipaddr4) + netixlan_row.find('[data-edit-name="ipaddr6"] input').val(data.ipaddr6) + netixlan_row.find('[data-edit-name="is_rs_peer"] input').prop("checked", data.is_rs_peer) + netixlan_row.find('[data-edit-name="operational"] input').prop("checked", data.operational) + netixlan_row.addClass("newrow") + this.detach_row(row); + row.find('button').tooltip("hide") + } + + + if(!requirements.length) + return apply(); + + var promise = new Promise((resolve, reject) => { resolve(); }); + + requirements.each(function() { + var req_id = $(this).data("ixf-require-delete") + promise.then(ixf_proposals.delete( + proposals.find('.suggestions-delete [data-ixf-id="'+req_id+'"]') + )) + }); + + promise.then(apply) }, /** @@ -556,6 +578,7 @@ PeeringDB.IXFProposals = twentyc.cls.define( all_entries_for_action : function(proposals, action) { var rows = proposals.find('.suggestions-'+action+' .row.item') + rows = rows.not('.hidden') var ids = [] rows.each(function() { @@ -630,7 +653,7 @@ PeeringDB.IXFProposals = twentyc.cls.define( row.detach(); this.require_refresh = true; - if(!par.find('.row.item').length) { + if(!par.find('.row.item').not('.hidden').length) { par.prev(".header").detach() par.detach() this.sync_proposals_state(proposals); @@ -652,6 +675,9 @@ PeeringDB.IXFProposals = twentyc.cls.define( var button_add_all = proposals.find('button.add-all') var button_resolve_all = proposals.find('button.resolve-all') + if(!this.all_entries_for_action(proposals, 'delete').rows.length) { + proposals.find('.suggestions-delete').prev('.header').detach(); + } if(!this.all_entries_for_action(proposals, 'add').rows.length) { button_add_all.prop('disabled',true); diff --git a/peeringdb_server/templates/admin/change_list_with_regex_search.html b/peeringdb_server/templates/admin/change_list_with_regex_search.html new file mode 100644 index 00000000..d9a83dc9 --- /dev/null +++ b/peeringdb_server/templates/admin/change_list_with_regex_search.html @@ -0,0 +1,6 @@ +{% extends 'admin/change_list.html' %} +{% load i18n %} +{% block search %} +{{block.super}} +

{% trans "To use regex search, begin search term with ^ and end with $."%}

+{% endblock %} diff --git a/peeringdb_server/templates/email/notify-ixf-add-inline.txt b/peeringdb_server/templates/email/notify-ixf-add-inline.txt new file mode 100644 index 00000000..232d64f1 --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-add-inline.txt @@ -0,0 +1,15 @@ +{% spaceless %} +{% load util %} +CREATE {{ instance.ixf_id_pretty_str }} +- speed: {{ instance.speed|pretty_speed }} +- operational: {{ instance.operational }} +- is_rs_peer: {{ instance.is_rs_peer }} +- status: {{ instance.status }} +{% if instance.actionable_error %} +A validation error was raised when the IX-F importer attempted to process this change. + +``` +{{ instance.error|safe|striptags }} +``` +{% endif %} +{% endspaceless %} diff --git a/peeringdb_server/templates/email/notify-ixf-add.txt b/peeringdb_server/templates/email/notify-ixf-add.txt index dd32da59..b9c309f2 100644 --- a/peeringdb_server/templates/email/notify-ixf-add.txt +++ b/peeringdb_server/templates/email/notify-ixf-add.txt @@ -1,18 +1,8 @@ {% load util %} {% if recipient == "ix" %}Your{% else %}The{% endif %} IX-F data proposes the creation of peer {{ instance }} -- speed: {{ instance.speed|pretty_speed }} -- operational: {{ instance.operational }} -- is_rs_peer: {{ instance.is_rs_peer }} -- status: {{ instance.status }} +{% include "email/notify-ixf-add-inline.txt" with instance=instance %} -{% if instance.error %} -A validation error was raised when the IX-F importer attempted to process this change. - -``` -{{ instance.error|safe|striptags }} -``` -{% endif %} {% spaceless %} {% if recipient == "ac" %} You may review and manually accept the IX-F data at {{ instance.ac_url }} diff --git a/peeringdb_server/templates/email/notify-ixf-conflict-insert.txt b/peeringdb_server/templates/email/notify-ixf-conflict-insert.txt new file mode 100644 index 00000000..ebec12c6 --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-conflict-insert.txt @@ -0,0 +1,5 @@ +{% if recipient == "net" %} +Data supplied by your network {{ entity.name }} is in conflict with that provided by {{ other.name }}. Please work with {{ other.name }} to resolve this conflict or correct your data at {{ entity.view_url_asn }} as appropriate. Your network page will contain hints to assist with this. The Peering Admin Committee will assist with this conflict if it is not resolved in {{ ticket_days }} days. +{% else %} +Data supplied by your exchange {{ entity.name }} is in conflict with that provided by {{ other.name }}. Please work with {{ other.name }} to resolve this conflict or correct your data as appropriate. The Peering Admin Committee will assist with this conflict if it is not resolved in {{ ticket_days }} days. +{% endif %} diff --git a/peeringdb_server/templates/email/notify-ixf-consolidated.txt b/peeringdb_server/templates/email/notify-ixf-consolidated.txt new file mode 100644 index 00000000..7b16c3fe --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-consolidated.txt @@ -0,0 +1,26 @@ +{% spaceless %} +{% if recipient == "net" %} +This email details {{ count }} new data mismatch[es] discovered by the PeeringDB IX-F Importer. Please review and correct these mismatches at your PeeringDB network page ({{ entity.view_url_asn }}) or work with the indicated IXPs to correct the data. +{% elif recipient == "ix" %} +This email details {{ count }} new data mismatch[es] discovered by the PeeringDB IX-F Importer. Please review and correct these mismatches in your IX-F JSON export ({{ entity.ixlan.ixf_ixp_member_list_url }}) or work with the indicated networks to correct the data. +{% endif %} +----------------------------- +{% for other, messages in proposals.items %} +{% for message in messages.add %} +{% include "email/notify-ixf-conflict-insert.txt" with entity=entity other=other ticket_days=ticket_days %} +{{ message }} +-----------------------------{% endfor %} +{% for message in messages.modify %} +{% include "email/notify-ixf-conflict-insert.txt" with entity=entity other=other ticket_days=ticket_days %} +{{ message }} +-----------------------------{% endfor %} +{% for message in messages.delete %} +{% include "email/notify-ixf-conflict-insert.txt" with entity=entity other=other ticket_days=ticket_days %} +{{ message }} +-----------------------------{% endfor %} +{% if messages.protocol_conflict %} +{% include "email/notify-ixf-conflict-insert.txt" with entity=entity other=other ticket_days=ticket_days %} +{{ messages.protocol_conflict }} +-----------------------------{% endif %} +{% endfor %} +{% endspaceless %} diff --git a/peeringdb_server/templates/email/notify-ixf-modify-inline.txt b/peeringdb_server/templates/email/notify-ixf-modify-inline.txt new file mode 100644 index 00000000..a33f56ff --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-modify-inline.txt @@ -0,0 +1,25 @@ +MODIFY {% if instance.primary_requirement %}{{ instance.primary_requirement.netixlan.ixf_id_pretty_str }}{% else %}{{ instance.netixlan.ixf_id_pretty_str }}{% endif %} +{% spaceless %} +{% for name, value in instance.actionable_changes.items %} +{% if name != "ipaddr4" and name != "ipaddr6" %}- {{ name }}: {{ value.from }} to {{ value.to }}{% endif %} +{% endfor %} + +{% if instance.ipaddr4_on_requirement %} +- Set IPv6 address {{ instance.ipaddr6 }} +{% elif instance.ipaddr6_on_requirement %} +- Set IPv4 address {{ instance.ipaddr4 }} +{% endif %}{% if instance.remote_changes %} +IX-F data has changed since the last notification: +{% for name, value in instance.remote_changes.items %} +- {{ name }}: {{ value.from }} to {{ value.to }} +{% endfor %} +{% endif %} + +{% if instance.actionable_error %} +A validation error was raised when the IX-F importer attempted to process this change. + +``` +{{ instance.error|safe|striptags }} +``` +{% endif %} +{% endspaceless %} diff --git a/peeringdb_server/templates/email/notify-ixf-update.txt b/peeringdb_server/templates/email/notify-ixf-modify.txt similarity index 64% rename from peeringdb_server/templates/email/notify-ixf-update.txt rename to peeringdb_server/templates/email/notify-ixf-modify.txt index 275ec8b0..9c593d31 100644 --- a/peeringdb_server/templates/email/notify-ixf-update.txt +++ b/peeringdb_server/templates/email/notify-ixf-modify.txt @@ -1,25 +1,7 @@ -{% if recipient == "ix" %}Your{% else %}The{% endif %} IX-F data suggests the following changes to the entry {{ instance }} +{% if recipient == "ix" %}Your{% else %}The{% endif %} IX-F data suggests the following changes to the entry {{ instance.netixlan.ixf_id_pretty_str }} -{% spaceless %} -{% for name, value in instance.changes.items %} -- {{ name }}: {{ value.from }} to {{ value.to }} -{% endfor %} +{% include "email/notify-ixf-modify-inline.txt" with instance=instance %} -{% if instance.remote_changes %} -IX-F data has changed since the last notification: -{% for name, value in instance.remote_changes.items %} -- {{ name }}: {{ value.from }} to {{ value.to }} -{% endfor %} -{% endif %} -{% endspaceless %} - -{% if instance.error %} -A validation error was raised when the IX-F importer attempted to process this change. - -``` -{{ instance.error|safe|striptags }} -``` -{% endif %} {% spaceless %} {% if recipient == "ac" %} You may review and manually accept the IX-F data at {{ instance.ac_url }} diff --git a/peeringdb_server/templates/email/notify-ixf-protocol-conflict-inline.txt b/peeringdb_server/templates/email/notify-ixf-protocol-conflict-inline.txt new file mode 100644 index 00000000..1ac7748e --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-protocol-conflict-inline.txt @@ -0,0 +1,21 @@ +{% spaceless %} +{% if ipaddr4 and not instance.net.info_unicast %} +{% if recipient == "net" %} +{{ instance.ix.name }}'s IX-F data provides IPv4 addresses for some of your peer entries, but your network has IPv4 support disabled +{% elif recipient == "ix" %} +Your IX-F data provides IPv4 addresses for some of AS{{ instance.asn }}'s peer entries, but the network has IPv4 support disabled +{% elif recipient == "ac" %} +{{ instance.ix.name }}'s IX-F data provides IPv4 addresses for some of AS{{ instance.asn }}'s peer entries, but the network has IPv4 support disabled +{% endif %} +{% endif %} +{% if ipaddr6 and not instance.net.info_ipv6 %} +{% if recipient == "net" %} +{{ instance.ix.name }}'s IX-F data provides IPv6 addresses for some of your peer entries, but your network has IPv6 support disabled +{% elif recipient == "ix" %} +Your IX-F data provides IPv6 addresses for some of AS{{ instance.asn }}'s peer entries, but the network has IPv6 support disabled +{% elif recipient == "ac" %} +{{ instance.ix.name }}'s IX-F data provides IPv6 addresses for some of AS{{ instance.asn }}'s peer entries, but the network has IPv6 support disabled +{% endif %} +{% endif %} +{% endspaceless %} +{% if recipient == "ac" %}{% include "email/ixf-contact-points.txt" with instance=instance %}{% endif %} diff --git a/peeringdb_server/templates/email/notify-ixf-protocol-conflict.txt b/peeringdb_server/templates/email/notify-ixf-protocol-conflict.txt new file mode 100644 index 00000000..00d08c3e --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-protocol-conflict.txt @@ -0,0 +1,7 @@ +{% include "email/notify-ixf-protocol-conflict-inline.txt" with instance=instance recipient=recipient ipaddr4=ipaddr4 ipaddr6=ipaddr6 %} + +- Exchange: {{ instance.ix.view_url }} +- Network: {{ instance.net.view_url}} +{% if recipient == "ac" %} +- IX-F Data: {{ ixf_url }} +{% endif %} diff --git a/peeringdb_server/templates/email/notify-ixf-remove-inline.txt b/peeringdb_server/templates/email/notify-ixf-remove-inline.txt new file mode 100644 index 00000000..b34e8e55 --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-remove-inline.txt @@ -0,0 +1,11 @@ +{% spaceless %} +REMOVE {{ instance.ixf_id_pretty_str }} +The combination of ASN, IPv4 and IPv6 do not exist in the IX-F data under one member connection. +{% if instance.actionable_error %} +A validation error was raised when the IX-F importer attempted to process this change. + +``` +{{ instance.error|safe|striptags }} +``` +{% endif %} +{% endspaceless %} diff --git a/peeringdb_server/templates/email/notify-ixf-remove.txt b/peeringdb_server/templates/email/notify-ixf-remove.txt index 32cd8bcd..476b8404 100644 --- a/peeringdb_server/templates/email/notify-ixf-remove.txt +++ b/peeringdb_server/templates/email/notify-ixf-remove.txt @@ -1,14 +1,5 @@ -{% if recipient == "ix" %}Your{% else %}The{% endif %} IX-F data proposes the removal of peer {{ instance }} +{% include "email/notify-ixf-remove-inline.txt" with instance=instance recipient=recipient %} -This is because the combination of ASN, IPv4 and IPv6 no longer exists in the IX-F data under one member connection. - -{% if instance.error %} -A validation error was raised when the IX-F importer attempted to process this change. - -``` -{{ instance.error|safe|striptags }} -``` -{% endif %} {% spaceless %} {% if recipient == "ac" %} You may review and manually accept the IX-F data at {{ instance.ac_url }} diff --git a/peeringdb_server/templates/email/notify-ixf-resolved-inline.txt b/peeringdb_server/templates/email/notify-ixf-resolved-inline.txt new file mode 100644 index 00000000..e0704dff --- /dev/null +++ b/peeringdb_server/templates/email/notify-ixf-resolved-inline.txt @@ -0,0 +1,2 @@ +{{ instance }} +The data difference between IX-F and PeeringDB for this entry has been resolved. diff --git a/peeringdb_server/templates/email/notify-ixf-resolved.txt b/peeringdb_server/templates/email/notify-ixf-resolved.txt index 2b31917d..b98f284a 100644 --- a/peeringdb_server/templates/email/notify-ixf-resolved.txt +++ b/peeringdb_server/templates/email/notify-ixf-resolved.txt @@ -1,4 +1,4 @@ -The data difference between IX-F and PeeringDB for this entry has been resolved. +{% include "email/notify-ixf-resolved-inline.txt" with instance=instance %} - Peer: {{ instance }} - Exchange: {{ instance.ix.view_url }} diff --git a/peeringdb_server/templates/email/notify-ixf-source-error.txt b/peeringdb_server/templates/email/notify-ixf-source-error.txt index 88635762..d64263bf 100644 --- a/peeringdb_server/templates/email/notify-ixf-source-error.txt +++ b/peeringdb_server/templates/email/notify-ixf-source-error.txt @@ -4,6 +4,6 @@ There was an issue when we attempted to parse the IX-F data for {{ instance.ix.n {{ error|safe|striptags }} ``` -- IX-F data url: {{ ixf_url }} +- IX-F data url: {{ instance.ix.ixlan.ixf_ixp_member_list_url }} - Exchange: {{ instance.ix.view_url }} - Timestamp: {{ dt|date:"c" }} diff --git a/peeringdb_server/templates/site/view_network_ixf_suggestions.html b/peeringdb_server/templates/site/view_network_ixf_suggestions.html index 27cbe868..47d06b7e 100644 --- a/peeringdb_server/templates/site/view_network_ixf_suggestions.html +++ b/peeringdb_server/templates/site/view_network_ixf_suggestions.html @@ -105,10 +105,18 @@
+ {% if x.net.info_unicast %} + {% else %} + + {% endif %}
+ {% if x.net.info_ipv6 %} + {% else %} + + {% endif %}
@@ -184,7 +192,7 @@ {% if not x.dismissed %} {% with ixf_id=x.ixf_id_pretty_str ix_name=x.ix.name %} -
+
@@ -192,10 +200,10 @@
- {{ x.ipaddr4|none_blank }} + {% if x.ipaddr4 %}{{ x.ipaddr4 }}{% else %}{% trans "IPv4 not set" %}{% endif %}
- {{ x.ipaddr6|none_blank }} + {% if x.ipaddr6 %}{{ x.ipaddr6 }}{% else %}{% trans "IPv6 not set" %}{% endif %}
@@ -254,24 +262,56 @@
+ {% if x.primary_requirement %} +
+ {% else %}
+ {% endif %} + + {% for requirement in x.secondary_requirements %} +
+ {% endfor %} +
+ {% if x.net.info_unicast %}
{{ x.ipaddr4|none_blank }}
+ {% else %} +
+ {{ x.netixlan.ipaddr4|none_blank }} +
+ {% endif %} + + {% if x.net.info_ip6 %}
{{ x.ipaddr6|none_blank }}
+ {% else %} +
+ {{ x.netixlan.ipaddr6|none_blank }} +
+ {% endif %}
- + + {% if x.ipaddr4_on_requirement %} +
+ {% trans "Set IPv6 address" %} +
+ {% elif x.ipaddr6_on_requirement %} +
+ {% trans "Set IPv4 address" %} +
+ {% endif %} + {% if x.changes.speed %}
{% trans "Speed" %}: @@ -279,7 +319,6 @@ {{ x.changes.speed.to|pretty_speed }}
- {% else %} {% endif %} {% if x.changes.is_rs_peer %} diff --git a/peeringdb_server/templates/site/view_network_side.html b/peeringdb_server/templates/site/view_network_side.html index fb169f85..bcb613e4 100644 --- a/peeringdb_server/templates/site/view_network_side.html +++ b/peeringdb_server/templates/site/view_network_side.html @@ -16,7 +16,7 @@
{% trans "Public Peering Exchange Points" %}
- +
@@ -231,7 +231,7 @@
{% trans "Private Peering Facilities" %}
- +
diff --git a/peeringdb_server/urls.py b/peeringdb_server/urls.py index 6f92015e..dd369cea 100644 --- a/peeringdb_server/urls.py +++ b/peeringdb_server/urls.py @@ -130,7 +130,7 @@ urlpatterns = [ name="org-view", ), url(r"^%s$" % Network.handleref.tag, view_network_by_query), - url(r"^asn/(?P\d+)/?$", view_network_by_asn), + url(r"^asn/(?P\d+)/?$", view_network_by_asn, name="net-view-asn"), url(r"^org_admin/users$", peeringdb_server.org_admin_views.users), url( r"^org_admin/user_permissions$", diff --git a/peeringdb_server/validators.py b/peeringdb_server/validators.py index 2558f65c..538d7e79 100644 --- a/peeringdb_server/validators.py +++ b/peeringdb_server/validators.py @@ -79,12 +79,8 @@ def validate_address_space(prefix): if not network_is_pdb_valid(prefix): raise ValidationError(_("Address space invalid: {}").format(prefix)) - prefixlen_min = getattr( - settings, f"DATA_QUALITY_MIN_PREFIXLEN_V{prefix.version}" - ) - prefixlen_max = getattr( - settings, f"DATA_QUALITY_MAX_PREFIXLEN_V{prefix.version}" - ) + prefixlen_min = getattr(settings, f"DATA_QUALITY_MIN_PREFIXLEN_V{prefix.version}") + prefixlen_max = getattr(settings, f"DATA_QUALITY_MAX_PREFIXLEN_V{prefix.version}") if prefix.prefixlen < prefixlen_min: raise ValidationError( diff --git a/peeringdb_server/views.py b/peeringdb_server/views.py index dde6fcbc..62eb6a84 100644 --- a/peeringdb_server/views.py +++ b/peeringdb_server/views.py @@ -1007,9 +1007,7 @@ def view_organization(request, id): users = {} if perms.get("can_manage"): - users.update( - {user.id: user for user in org.admin_usergroup.user_set.all()} - ) + users.update({user.id: user for user in org.admin_usergroup.user_set.all()}) users.update({user.id: user for user in org.usergroup.user_set.all()}) users = sorted(list(users.values()), key=lambda x: x.full_name) @@ -1846,8 +1844,8 @@ def request_search(request): result = search(q) sponsors = { - org.id: sponsorship.label.lower() - for org, sponsorship in Sponsorship.active_by_org() + org.id: sponsorship.label.lower() + for org, sponsorship in Sponsorship.active_by_org() } for tag, rows in list(result.items()): diff --git a/tests/data/cmd/ixf/dismissals/test0.json b/tests/data/cmd/ixf/dismissals/test0.json new file mode 100644 index 00000000..670fefaa --- /dev/null +++ b/tests/data/cmd/ixf/dismissals/test0.json @@ -0,0 +1,130 @@ +{ + "timestamp": "2020-07-13T09:23:47Z", + "version": "1.0", + "ixp_list": [ + { + "shortname": "Test Exchange", + "ixp_id": 1, + "ixf_id": 1 + } + ], + "member_list": [ + { + "asnum": 1001, + "member_type": "peering", + "name": "Netflix", + "url": "http://netflix.com/", + "contact_email": [ + "peering@netflix.com", + "mrpeering@netflix.com" + ], + "contact_phone": [ + "+1 1234 5678" + ], + "contact_hours": "8/5", + "peering_policy": "open", + "peering_policy_url": "https://www.netflix.com/openconnect/", + "member_since": "2009-02-04T00:00:00Z", + "connection_list": [ + { + "ixp_id": 42, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 10000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2906:1", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 1, + "ipv4": { + "address": "195.69.148.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2907:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 2, + "ipv4": { + "address": "195.69.146.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2908:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 3, + "ipv4": { + "address": "195.69.149.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2909:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/data/cmd/ixf/email/test0.json b/tests/data/cmd/ixf/email/test0.json new file mode 100644 index 00000000..670fefaa --- /dev/null +++ b/tests/data/cmd/ixf/email/test0.json @@ -0,0 +1,130 @@ +{ + "timestamp": "2020-07-13T09:23:47Z", + "version": "1.0", + "ixp_list": [ + { + "shortname": "Test Exchange", + "ixp_id": 1, + "ixf_id": 1 + } + ], + "member_list": [ + { + "asnum": 1001, + "member_type": "peering", + "name": "Netflix", + "url": "http://netflix.com/", + "contact_email": [ + "peering@netflix.com", + "mrpeering@netflix.com" + ], + "contact_phone": [ + "+1 1234 5678" + ], + "contact_hours": "8/5", + "peering_policy": "open", + "peering_policy_url": "https://www.netflix.com/openconnect/", + "member_since": "2009-02-04T00:00:00Z", + "connection_list": [ + { + "ixp_id": 42, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 10000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2906:1", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 1, + "ipv4": { + "address": "195.69.148.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2907:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 2, + "ipv4": { + "address": "195.69.146.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2908:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 3, + "ipv4": { + "address": "195.69.149.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2909:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/data/cmd/ixf/hints/test0.json b/tests/data/cmd/ixf/hints/test0.json new file mode 100644 index 00000000..670fefaa --- /dev/null +++ b/tests/data/cmd/ixf/hints/test0.json @@ -0,0 +1,130 @@ +{ + "timestamp": "2020-07-13T09:23:47Z", + "version": "1.0", + "ixp_list": [ + { + "shortname": "Test Exchange", + "ixp_id": 1, + "ixf_id": 1 + } + ], + "member_list": [ + { + "asnum": 1001, + "member_type": "peering", + "name": "Netflix", + "url": "http://netflix.com/", + "contact_email": [ + "peering@netflix.com", + "mrpeering@netflix.com" + ], + "contact_phone": [ + "+1 1234 5678" + ], + "contact_hours": "8/5", + "peering_policy": "open", + "peering_policy_url": "https://www.netflix.com/openconnect/", + "member_since": "2009-02-04T00:00:00Z", + "connection_list": [ + { + "ixp_id": 42, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 10000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2906:1", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 1, + "ipv4": { + "address": "195.69.148.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2907:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 2, + "ipv4": { + "address": "195.69.146.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2908:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 3, + "ipv4": { + "address": "195.69.149.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2909:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/data/cmd/ixf/reset/test0.json b/tests/data/cmd/ixf/reset/test0.json new file mode 100644 index 00000000..670fefaa --- /dev/null +++ b/tests/data/cmd/ixf/reset/test0.json @@ -0,0 +1,130 @@ +{ + "timestamp": "2020-07-13T09:23:47Z", + "version": "1.0", + "ixp_list": [ + { + "shortname": "Test Exchange", + "ixp_id": 1, + "ixf_id": 1 + } + ], + "member_list": [ + { + "asnum": 1001, + "member_type": "peering", + "name": "Netflix", + "url": "http://netflix.com/", + "contact_email": [ + "peering@netflix.com", + "mrpeering@netflix.com" + ], + "contact_phone": [ + "+1 1234 5678" + ], + "contact_hours": "8/5", + "peering_policy": "open", + "peering_policy_url": "https://www.netflix.com/openconnect/", + "member_since": "2009-02-04T00:00:00Z", + "connection_list": [ + { + "ixp_id": 42, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 10000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2906:1", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 1, + "ipv4": { + "address": "195.69.148.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2907:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 2, + "ipv4": { + "address": "195.69.146.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2908:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 3, + "ipv4": { + "address": "195.69.149.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2909:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/data/cmd/ixf/tickets/test0.json b/tests/data/cmd/ixf/tickets/test0.json new file mode 100644 index 00000000..670fefaa --- /dev/null +++ b/tests/data/cmd/ixf/tickets/test0.json @@ -0,0 +1,130 @@ +{ + "timestamp": "2020-07-13T09:23:47Z", + "version": "1.0", + "ixp_list": [ + { + "shortname": "Test Exchange", + "ixp_id": 1, + "ixf_id": 1 + } + ], + "member_list": [ + { + "asnum": 1001, + "member_type": "peering", + "name": "Netflix", + "url": "http://netflix.com/", + "contact_email": [ + "peering@netflix.com", + "mrpeering@netflix.com" + ], + "contact_phone": [ + "+1 1234 5678" + ], + "contact_hours": "8/5", + "peering_policy": "open", + "peering_policy_url": "https://www.netflix.com/openconnect/", + "member_since": "2009-02-04T00:00:00Z", + "connection_list": [ + { + "ixp_id": 42, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 10000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2906:1", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 1, + "ipv4": { + "address": "195.69.148.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2907:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 2, + "ipv4": { + "address": "195.69.146.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2908:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + }, + { + "vlan_id": 3, + "ipv4": { + "address": "195.69.149.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2909:2", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/data/json_members_list/ixf.member.2.json b/tests/data/json_members_list/ixf.member.2.json index 4b175df8..6ab2f460 100644 --- a/tests/data/json_members_list/ixf.member.2.json +++ b/tests/data/json_members_list/ixf.member.2.json @@ -33,7 +33,7 @@ "if_list": [ { "switch_id": 1, - "if_speed": 10000, + "if_speed": 20000, "if_type": "LR4" } ], @@ -62,7 +62,7 @@ ] }, { - "ixp_id": 42, + "ixp_id": 43, "connected_since": "2009-02-04T00:00:00Z", "state": "connected", "if_list": [ @@ -76,7 +76,7 @@ { "vlan_id": 0, "ipv4": { - "address": "195.69.150.250", + "address": "195.69.147.251", "routeserver": true, "max_prefix": 42, "as_macro": "AS-NFLX-V4", @@ -85,7 +85,42 @@ ] }, "ipv6": { - "address": "2001:7f8:1::a500:2906:3", + "address": "2001:7f8:1::a500:2906:4", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + }, + { + "ixp_id": 44, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 15000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.249", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2907:1", "routeserver": true, "max_prefix": 42, "as_macro": "AS-NFLX-V6", diff --git a/tests/data/json_members_list/ixf.member.3.json b/tests/data/json_members_list/ixf.member.3.json new file mode 100644 index 00000000..b60e18f7 --- /dev/null +++ b/tests/data/json_members_list/ixf.member.3.json @@ -0,0 +1,102 @@ +{ + "timestamp": "2020-07-13T09:23:47Z", + "version": "1.0", + "ixp_list": [ + { + "shortname": "Test Exchange", + "ixp_id": 1, + "ixf_id": 1 + } + ], + "member_list": [ + { + "asnum": 1001, + "member_type": "peering", + "name": "Netflix", + "url": "http://netflix.com/", + "contact_email": [ + "peering@netflix.com", + "mrpeering@netflix.com" + ], + "contact_phone": [ + "+1 1234 5678" + ], + "contact_hours": "8/5", + "peering_policy": "open", + "peering_policy_url": "https://www.netflix.com/openconnect/", + "member_since": "2009-02-04T00:00:00Z", + "connection_list": [ + { + "ixp_id": 42, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 10000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.250", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2906:1", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + }, + { + "ixp_id": 42, + "connected_since": "2009-02-04T00:00:00Z", + "state": "connected", + "if_list": [ + { + "switch_id": 1, + "if_speed": 10000, + "if_type": "LR4" + } + ], + "vlan_list": [ + { + "vlan_id": 0, + "ipv4": { + "address": "195.69.147.251", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V4", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + }, + "ipv6": { + "address": "2001:7f8:1::a500:2906:3", + "routeserver": true, + "max_prefix": 42, + "as_macro": "AS-NFLX-V6", + "mac_address" : [ + "00:0a:95:9d:68:16" + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/django_init.py b/tests/django_init.py index 88d7e0ba..f3eb6edd 100644 --- a/tests/django_init.py +++ b/tests/django_init.py @@ -177,6 +177,7 @@ settings.configure( IXF_NOTIFY_IX_ON_CONFLICT=True, IXF_NOTIFY_NET_ON_CONFLICT=True, IXF_TICKET_ON_CONFLICT=True, + IXF_SEND_TICKETS=False, ABSTRACT_ONLY=True, GOOGLE_GEOLOC_API_KEY="AIzatest", RATELIMITS={ @@ -194,4 +195,7 @@ settings.configure( MAX_USER_AFFILIATION_REQUESTS=10, MAIL_DEBUG=True, IXF_PARSE_ERROR_NOTIFICATION_PERIOD=36, + IXF_IMPORTER_DAYS_UNTIL_TICKET=6, + DESKPRO_URL="test", + DESKPRO_KEY="test", ) diff --git a/tests/test_admin.py b/tests/test_admin.py index 8de2f298..9987b05d 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -1,10 +1,11 @@ import os import json import pytest +import urllib from django.test import Client, TestCase, RequestFactory from django.contrib.auth.models import Group -from django.urls import reverse +from django.urls import reverse, resolve from django.core.management import call_command import django_namespace_perms as nsp @@ -249,10 +250,7 @@ class AdminTests(TestCase): response = c.get(url, follow=True) self.assertEqual(response.status_code, 200) for i, n in models.COMMANDLINE_TOOLS: - assert ( - f'' - in response.content.decode() - ) + assert f'' in response.content.decode() def test_commandline_tool_renumber_lans(self): # test the form that runs the renumer ip space tool @@ -388,6 +386,54 @@ class AdminTests(TestCase): assert netixlan.ipaddr4 is None assert "Ip address already exists elsewhere" in response.content.decode("utf-8") + def _run_regex_search(self, model, search_term): + c = Client() + c.login(username="admin", password="admin") + url = reverse("admin:peeringdb_server_{}_changelist".format(model)) + search = url + "?q=" + urllib.parse.quote_plus(search_term) + response = c.get(search) + content = response.content.decode("utf-8") + return content + + def test_search_deskprotickets(self): + # Set up data + ixf_importer, _ = models.User.objects.get_or_create(username="ixf_importer") + for i in range(10): + models.DeskProTicket.objects.create( + subject="test number {}".format(i), body="test", user=ixf_importer + ) + + search_term = "^.*[0-5]$" + content = self._run_regex_search("deskproticket", search_term) + print(content) + expected = ["test number {}".format(i) for i in range(5)] + expected_not = ["test number {}".format(i) for i in range(6, 10)] + + for e in expected: + assert e in content + + for e in expected_not: + assert e not in content + + def test_search_ixfimportemails(self): + for i in range(10): + models.IXFImportEmail.objects.create( + subject="test number {}".format(i), message="test", recipients="test" + ) + search_term = "^.*[2-4]$" + content = self._run_regex_search("ixfimportemail", search_term) + print(content) + expected = ["test number {}".format(i) for i in range(2, 5)] + expected_not = ["test number 1"] + [ + "test number {}".format(i) for i in range(6, 10) + ] + + for e in expected: + assert e in content + + for e in expected_not: + assert e not in content + def test_all_views_readonly(self): self._test_all_views( self.readonly_admin, diff --git a/tests/test_cmd_ixf_import_resets.py b/tests/test_cmd_ixf_import_resets.py new file mode 100644 index 00000000..bf95d59e --- /dev/null +++ b/tests/test_cmd_ixf_import_resets.py @@ -0,0 +1,186 @@ +import json +import os +from pprint import pprint +import reversion +import requests +import jsonschema +import time +import io +import datetime + +from django.core.management import call_command + +from peeringdb_server.models import ( + Organization, + Network, + NetworkIXLan, + NetworkContact, + IXLan, + IXLanPrefix, + InternetExchange, + IXFMemberData, + IXLanIXFMemberImportLog, + User, + DeskProTicket, + IXFImportEmail, +) +from peeringdb_server import ixf +import pytest + + +@pytest.mark.django_db +def test_reset_hints(entities, data_cmd_ixf_hints): + ixf_import_data = json.loads(data_cmd_ixf_hints.json) + importer = ixf.Importer() + ixlan = entities["ixlan"] + # Create IXFMemberData + importer.update(ixlan, data=ixf_import_data) + + call_command("pdb_ixf_ixp_member_import", reset_hints=True, commit=True) + + assert IXFMemberData.objects.count() == 0 + assert DeskProTicket.objects.filter(body__contains="reset_hints").count() == 1 + + +@pytest.mark.django_db +def test_reset_dismissals(entities, data_cmd_ixf_dismissals): + ixf_import_data = json.loads(data_cmd_ixf_dismissals.json) + importer = ixf.Importer() + ixlan = entities["ixlan"] + # Create IXFMemberData + importer.update(ixlan, data=ixf_import_data) + + # Dismiss all IXFMemberData + for ixfm in IXFMemberData.objects.all(): + ixfm.dismissed = True + ixfm.save() + + call_command("pdb_ixf_ixp_member_import", reset_dismisses=True, commit=True) + + assert IXFMemberData.objects.filter(dismissed=False).count() == 4 + assert DeskProTicket.objects.filter(body__contains="reset_dismisses").count() == 1 + + +@pytest.mark.django_db +def test_reset_email(entities, data_cmd_ixf_email): + ixf_import_data = json.loads(data_cmd_ixf_email.json) + importer = ixf.Importer() + ixlan = entities["ixlan"] + # Create IXFMemberData + importer.update(ixlan, data=ixf_import_data) + importer.notify_proposals() + assert IXFImportEmail.objects.count() == 1 + + call_command("pdb_ixf_ixp_member_import", reset_email=True, commit=True) + + assert IXFImportEmail.objects.count() == 0 + assert DeskProTicket.objects.filter(body__contains="reset_email").count() == 1 + + +@pytest.mark.django_db +def test_reset_tickets(deskprotickets): + assert DeskProTicket.objects.count() == 5 + + call_command("pdb_ixf_ixp_member_import", reset_tickets=True, commit=True) + + assert DeskProTicket.objects.count() == 2 + assert DeskProTicket.objects.filter(body__contains="reset_tickets").count() == 1 + + +@pytest.mark.django_db +def test_reset_all(entities, deskprotickets, data_cmd_ixf_reset): + ixf_import_data = json.loads(data_cmd_ixf_reset.json) + importer = ixf.Importer() + ixlan = entities["ixlan"] + # Create IXFMemberData + importer.update(ixlan, data=ixf_import_data) + importer.notify_proposals() + + assert DeskProTicket.objects.count() == 5 + assert IXFMemberData.objects.count() == 4 + assert IXFImportEmail.objects.count() == 1 + + call_command("pdb_ixf_ixp_member_import", reset=True, commit=True) + + assert DeskProTicket.objects.count() == 2 + assert DeskProTicket.objects.filter(body__contains="reset").count() == 1 + assert IXFMemberData.objects.count() == 0 + assert IXFImportEmail.objects.count() == 0 + + +@pytest.fixture +def entities(): + entities = {} + with reversion.create_revision(): + entities["org"] = Organization.objects.create(name="Netflix", status="ok") + entities["ix"] = InternetExchange.objects.create( + name="Test Exchange One", + org=entities["org"], + status="ok", + tech_email="ix1@localhost", + ) + entities["ixlan"] = entities["ix"].ixlan + + # create ixlan prefix(s) + entities["ixpfx"] = [ + IXLanPrefix.objects.create( + ixlan=entities["ixlan"], + status="ok", + prefix="195.69.144.0/22", + protocol="IPv4", + ), + IXLanPrefix.objects.create( + ixlan=entities["ixlan"], + status="ok", + prefix="2001:7f8:1::/64", + protocol="IPv6", + ), + ] + entities["net"] = Network.objects.create( + name="Network w allow ixp update disabled", + org=entities["org"], + asn=1001, + allow_ixp_update=False, + status="ok", + info_prefixes4=42, + info_prefixes6=42, + website="http://netflix.com/", + policy_general="Open", + policy_url="https://www.netflix.com/openconnect/", + info_unicast=True, + info_ipv6=True, + ) + + entities["netcontact"] = NetworkContact.objects.create( + email="network1@localhost", + network=entities["net"], + status="ok", + role="Policy", + ) + + admin_user = User.objects.create_user("admin", "admin@localhost", "admin") + ixf_importer_user = User.objects.create_user( + "ixf_importer", "ixf_importer@localhost", "ixf_importer" + ) + entities["org"].admin_usergroup.user_set.add(admin_user) + return entities + + +@pytest.fixture +def deskprotickets(): + """ + Creates several deskprotickets. 4 begin with [IX-F], 1 doesn't. + """ + user, _ = User.objects.get_or_create(username="ixf_importer") + message = "test" + + subjects = [ + "[IX-F] Suggested:Add Test Exchange One AS1001 195.69.147.250 2001:7f8:1::a500:2906:1", + "[IX-F] Suggested:Add Test Exchange One AS1001 195.69.148.250 2001:7f8:1::a500:2907:2", + "[IX-F] Suggested:Add Test Exchange One AS1001 195.69.146.250 2001:7f8:1::a500:2908:2", + "[IX-F] Suggested:ADD Test Exchange One AS1001 195.69.149.250 2001:7f8:1::a500:2909:2", + "Unrelated Issue: Urgent!!!", + ] + for subject in subjects: + DeskProTicket.objects.create(subject=subject, body=message, user=user) + return DeskProTicket.objects.all() diff --git a/tests/test_geocode.py b/tests/test_geocode.py index 49d0f2d2..4a73e118 100644 --- a/tests/test_geocode.py +++ b/tests/test_geocode.py @@ -20,30 +20,28 @@ class ViewTestCase(TestCase): # create organizations cls.organizations = { - k: - models.Organization.objects.create( - name="Geocode Org %s" % k, status="ok" - ) + k: models.Organization.objects.create( + name="Geocode Org %s" % k, status="ok" + ) for k in ["a", "b", "c", "d"] } # create facilities cls.facilities = { - k: - models.Facility.objects.create( - name=f"Geocode Fac {k}", - status="ok", - org=cls.organizations[k], - address1="Some street", - address2=k, - city="Chicago", - country="US", - state="IL", - zipcode="1234", - latitude=1.23, - longitude=-1.23, - geocode_status=True, - ) + k: models.Facility.objects.create( + name=f"Geocode Fac {k}", + status="ok", + org=cls.organizations[k], + address1="Some street", + address2=k, + city="Chicago", + country="US", + state="IL", + zipcode="1234", + latitude=1.23, + longitude=-1.23, + geocode_status=True, + ) for k in ["a", "b", "c", "d"] } diff --git a/tests/test_inet_parse.py b/tests/test_inet_parse.py index 7b7489c7..83c5231f 100644 --- a/tests/test_inet_parse.py +++ b/tests/test_inet_parse.py @@ -6,9 +6,9 @@ import pytest_filedata def assert_parsed(data, parsed): # dump in json format for easily adding expected print( - "echo \\\n'{}'\\\n > {}/{}.expected".format( - data.dumps(parsed), data.path, data.name - ) + "echo \\\n'{}'\\\n > {}/{}.expected".format( + data.dumps(parsed), data.path, data.name + ) ) assert data.expected == parsed diff --git a/tests/test_ixf_member_import_protocol.py b/tests/test_ixf_member_import_protocol.py index 8b56aebb..ea997a85 100644 --- a/tests/test_ixf_member_import_protocol.py +++ b/tests/test_ixf_member_import_protocol.py @@ -7,6 +7,7 @@ import jsonschema import time import io import datetime +import ipaddress from peeringdb_server.models import ( Organization, @@ -20,13 +21,14 @@ from peeringdb_server.models import ( IXLanIXFMemberImportLog, User, DeskProTicket, + IXFImportEmail, ) from peeringdb_server import ixf import pytest @pytest.mark.django_db -def test_resolve_local_ixf(entities): +def test_resolve_local_ixf(entities, use_ip, save): """ Netixlan exists, remote data matches the netixlan, and there is a local-ixf entry that also matches all the data. @@ -41,8 +43,8 @@ def test_resolve_local_ixf(entities): ixlan=ixlan, asn=network.asn, speed=10000, - ipaddr4="195.69.147.250", - ipaddr6="2001:7f8:1::a500:2906:1", + ipaddr4=use_ip(4, "195.69.147.250"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), status="ok", is_rs_peer=True, operational=True, @@ -52,8 +54,8 @@ def test_resolve_local_ixf(entities): # Create a local IXF that matches remote details IXFMemberData.objects.create( asn=network.asn, - ipaddr4="195.69.147.250", - ipaddr6="2001:7f8:1::a500:2906:1", + ipaddr4=use_ip(4, "195.69.147.250"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), @@ -63,17 +65,24 @@ def test_resolve_local_ixf(entities): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() + assert IXFMemberData.objects.count() == 0 - assert_ticket_exists([(network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) + + # We do not email upon resolve + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 0 + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_update_data_attributes(entities): +def test_update_data_attributes(entities, use_ip, save): """ The NetIXLan differs from the remote data, but allow_ixp_update is enabled so we update automatically. @@ -89,8 +98,8 @@ def test_update_data_attributes(entities): ixlan=ixlan, asn=network.asn, speed=20000, - ipaddr4="195.69.147.250", - ipaddr6="2001:7f8:1::a500:2906:1", + ipaddr4=use_ip(4, "195.69.147.250"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), status="ok", is_rs_peer=False, operational=False, @@ -98,43 +107,76 @@ def test_update_data_attributes(entities): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() - assert len(importer.log["data"]) == 1 assert IXFMemberData.objects.count() == 0 - log = importer.log["data"][0] - assert log["action"] == "modify" - assert "is_rs_peer" in log["reason"] - assert "operational" in log["reason"] - assert "speed" in log["reason"] + print(importer.log) + print(use_ip(4), use_ip(6)) - netixlan = NetworkIXLan.objects.first() + if (network.ipv4_support and not use_ip(4)) or ( + network.ipv6_support and not use_ip(6) + ): + + # test (delete+add) consolidation (#770) + + assert len(importer.log["data"]) == 2 + log_delete = importer.log["data"][0] + log_add = importer.log["data"][1] + assert log_delete["action"] == "delete" + assert log_add["action"] == "add" + + else: + + # test modify + assert len(importer.log["data"]) == 1 + log = importer.log["data"][0] + + assert log["action"] == "modify" + assert "is_rs_peer" in log["reason"] + assert "operational" in log["reason"] + assert "speed" in log["reason"] + + netixlan = NetworkIXLan.objects.filter(status="ok").first() assert netixlan.operational == True assert netixlan.is_rs_peer == True assert netixlan.speed == 10000 # Assert idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 0 - assert NetworkIXLan.objects.count() == 1 + assert_idempotent(importer, ixlan, data) + + # Assert no emails + assert_no_emails() # test rollback import_log = IXLanIXFMemberImportLog.objects.first() import_log.rollback() - netixlan.refresh_from_db() + netixlan = NetworkIXLan.objects.filter(status="ok").first() assert netixlan.operational == False assert netixlan.is_rs_peer == False assert netixlan.speed == 20000 + assert netixlan.ipaddr4 == use_ip(4, "195.69.147.250") + assert netixlan.ipaddr6 == use_ip(6, "2001:7f8:1::a500:2906:1") @pytest.mark.django_db -def test_suggest_modify_local_ixf(entities): +def test_suggest_modify_local_ixf(entities, use_ip, save): """ - Netixlan is different from remote in terms of speed, operational, and is_rs_peer + a) Netixlan is different from remote in terms of speed, operational, and is_rs_peer BUT there is already a local-ixf for the update. + Automatic updates are disabled (so netixlan will not change). We do nothing and confirm the local-ixf stays the same. + + b) Netixlan is different from remote in terms of speed, operational, and is_rs_peer + BUT there is already a local-ixf for the update, however IX-F data suggest + to add one of the missing ip addresses (eg. signature changed) + """ data = setup_test_data("ixf.member.1") network = entities["net"]["UPDATE_DISABLED"] @@ -146,8 +188,8 @@ def test_suggest_modify_local_ixf(entities): ixlan=ixlan, asn=network.asn, speed=20000, - ipaddr4="195.69.147.250", - ipaddr6="2001:7f8:1::a500:2906:1", + ipaddr4=use_ip(4, "195.69.147.250"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), status="ok", is_rs_peer=False, operational=False, @@ -157,48 +199,118 @@ def test_suggest_modify_local_ixf(entities): # Matches the json data, doesn't match the existing netixlan. preexisting_ixfmember_data = IXFMemberData.objects.create( asn=network.asn, - ipaddr4="195.69.147.250", - ipaddr6="2001:7f8:1::a500:2906:1", + ipaddr4=use_ip(4, "195.69.147.250"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", + data={"foo": "bar"}, ) importer = ixf.Importer() - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert preexisting_ixfmember_data == IXFMemberData.objects.first() + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + + importer.update(ixlan, data=data) + importer.notify_proposals() + + for email in IXFImportEmail.objects.all(): + print(email.message) + + if (network.ipv4_support and not network.ipv6_support and not use_ip(4)) or ( + network.ipv6_support and not network.ipv4_support and not use_ip(6) + ): + + # edge case where network has the one ip set that + # its not supporting and the other ip nto set at all + # (see #771 and #770) and the existing suggestion was for + # a different combination protocols supported and signature + # + # this will generate a new proposal notification for the entry + # and there is nothing we can do about it at this point + # + # should only happen very rarely + + email_info = [ + ( + "MODIFY", + network.asn, + use_ip(4, "195.69.147.250"), + use_ip(6, "2001:7f8:1::a500:2906:1"), + ) + ] + + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) + + assert IXFMemberData.objects.count() == 2 + ixf_member_data_delete = IXFMemberData.objects.all()[0] + ixf_member_data_modify = IXFMemberData.objects.all()[1] + assert ixf_member_data_modify.action == "modify" + assert ixf_member_data_delete.action == "delete" + + assert ixf_member_data_delete.requirement_of == ixf_member_data_modify + + elif (network.ipv4_support and network.ipv6_support and not use_ip(4)) or ( + network.ipv6_support and network.ipv4_support and not use_ip(6) + ): + + # network supports both protocols, old ix-f data only has one + # of the ips set, suggest adding the other + # #770 #771 + + email_info = [ + ( + "MODIFY", + network.asn, + use_ip(4, "195.69.147.250"), + use_ip(6, "2001:7f8:1::a500:2906:1"), + ) + ] + + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) + + assert IXFMemberData.objects.count() == 2 + ixf_member_data_delete = IXFMemberData.objects.all()[0] + ixf_member_data_modify = IXFMemberData.objects.all()[1] + assert ixf_member_data_modify.action == "modify" + assert ixf_member_data_delete.action == "delete" + assert ixf_member_data_delete.requirement_of == ixf_member_data_modify + + else: + + assert_no_emails() + assert IXFMemberData.objects.count() == 1 + assert preexisting_ixfmember_data == IXFMemberData.objects.first() # Assert idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert preexisting_ixfmember_data == IXFMemberData.objects.first() + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_suggest_modify(entities, capsys): +def test_suggest_modify(entities, use_ip, save): """ Netixlan is different from remote in terms of speed, operational, and is_rs_peer. There is no local-ixf existing. - We need to create a local ixf, create a ticket for admin com, - email the network and email the ix. + + We need to send out notifications to net and ix """ data = setup_test_data("ixf.member.1") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] - entities["netixlan"].append( NetworkIXLan.objects.create( network=network, ixlan=ixlan, asn=network.asn, speed=20000, - ipaddr4="195.69.147.250", - ipaddr6="2001:7f8:1::a500:2906:1", + ipaddr4=use_ip(4, "195.69.147.250"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), status="ok", is_rs_peer=False, operational=False, @@ -206,36 +318,63 @@ def test_suggest_modify(entities, capsys): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() + + print(importer.log) # Create local ixf - assert IXFMemberData.objects.count() == 1 + if (not network.ipv4_support and not use_ip(6)) or ( + not network.ipv6_support and not use_ip(4) + ): + + # data changes and signature (ip) change with + # partial ip protocol support + # #770 and #771 + + assert IXFMemberData.objects.count() == 2 + elif (network.ipv4_support and network.ipv6_support) and ( + not use_ip(6) or not use_ip(4) + ): + + # data changes and signature (ip) change with + # full ip protocol support + # #770 and #771 + + assert IXFMemberData.objects.count() == 2 + else: + assert IXFMemberData.objects.count() == 1 assert len(importer.log["data"]) == 1 log = importer.log["data"][0] assert log["action"] == "suggest-modify" - # Create a ticket for Admin Com - assert_ticket_exists([(network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) - # NetIXLAN is unchanged assert NetworkIXLan.objects.first().speed == 20000 assert NetworkIXLan.objects.first().is_rs_peer == False assert NetworkIXLan.objects.first().operational == False - # Email is sent to the Network and the IX - stdout = capsys.readouterr().out - assert_email_sent( - stdout, (network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1") - ) + # Consolidated email is sent to the Network and the IX + email_info = [ + ( + "MODIFY", + network.asn, + use_ip(4, "195.69.147.250"), + use_ip(6, "2001:7f8:1::a500:2906:1"), + ) + ] + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert_ticket_exists([(network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) + assert_idempotent(importer, ixlan, data, save=save) @pytest.mark.django_db -def test_add_netixlan(entities): +def test_add_netixlan(entities, use_ip, save): """ No NetIXLan exists but remote IXF data has information to create one (without conflicts). Updates are enabled @@ -246,7 +385,12 @@ def test_add_netixlan(entities): ixlan = entities["ixlan"][0] importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() log = importer.log["data"][0] assert log["action"] == "add" @@ -254,9 +398,13 @@ def test_add_netixlan(entities): # Test idempotent importer.update(ixlan, data=data) + importer.notify_proposals() + assert IXFMemberData.objects.count() == 0 assert NetworkIXLan.objects.count() == 1 + assert_no_emails() + # test rollback import_log = IXLanIXFMemberImportLog.objects.first() import_log.rollback() @@ -266,7 +414,7 @@ def test_add_netixlan(entities): @pytest.mark.django_db -def test_add_netixlan_conflict_local_ixf(entities, capsys): +def test_add_netixlan_conflict_local_ixf(entities, use_ip, save): """ No NetIXLan exists. Network allows auto updates. While remote IXF data has information to create a new NetIXLan, there are conflicts with the ipaddresses that @@ -276,94 +424,178 @@ def test_add_netixlan_conflict_local_ixf(entities, capsys): data = setup_test_data("ixf.member.0") network = entities["net"]["UPDATE_ENABLED"] + ixlan = entities["ixlan"][1] # So we have conflicts with IPAddresses + # invalid prefix space error will only be raised if + # the other ipaddress (v6 in this case) matches + # so we move that prefix over + + if use_ip(4): + ixpfx = entities["ixlan"][0].ixpfx_set.filter(protocol="IPv6").first() + invalid_ip = 4 + else: + ixpfx = entities["ixlan"][0].ixpfx_set.filter(protocol="IPv4").first() + invalid_ip = 6 + ixpfx.ixlan = entities["ixlan"][1] + ixpfx.save() + preexisting_ixfmember_data = IXFMemberData.objects.create( asn=network.asn, - ipaddr4="195.69.147.250", # Matches remote-ixf, but conflicts with IXLan - ipaddr6="2001:7f8:1::a500:2906:1", # Matches remote-ixf, but conflicts with IXLan + # Matches remote-ixf, but conflicts with IXLan + ipaddr4=use_ip(4, "195.69.147.250"), + # Matches remote-ixf, but conflicts with IXLan + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, status="ok", - error=["IPv4 195.69.147.250 does not match any prefix on this ixlan"], + data=json.dumps({"foo": "bar"}), + error=json.dumps( + {"ipaddr4": ["IPv4 195.69.147.250 does not match any prefix on this ixlan"]} + ), ) importer = ixf.Importer() - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 0 + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + + importer.update(ixlan, data=data) + importer.notify_proposals() ixfmemberdata = IXFMemberData.objects.first() - assert ( - "IPv4 195.69.147.250 does not match any prefix on this ixlan" - in ixfmemberdata.error - ) - assert_no_ticket_exists() + for email in IXFImportEmail.objects.all(): + print(email.message) - stdout = capsys.readouterr().out - assert stdout == "" + if (not network.ipv4_support and invalid_ip == 4) or ( + not network.ipv6_support and invalid_ip == 6 + ): - updated_timestamp = ixfmemberdata.updated - importer.update(ixlan, data=data) - assert updated_timestamp == IXFMemberData.objects.first().updated + # edge case, signature changed, and invalid ip + # is on unsupported protocol, making the proposal + # irrelevant - # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 0 - assert_no_ticket_exists() + assert IXFMemberData.objects.count() == 0 + assert_no_emails() + assert_idempotent(importer, ixlan, data, save=save) + + elif (network.ipv4_support and not use_ip(4)) or ( + network.ipv6_support and not use_ip(6) + ): + + # edge case, signature changed, and invalid and + # conflicting ip changed causing a drop of the original + # erorring proposal, and a creation of a new one + # on the next one + + email_info = [ + ("CREATE", network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1",) + ] + assert_no_emails() + + assert IXFMemberData.objects.count() == 0 + assert NetworkIXLan.objects.count() == 0 + + importer.update(ixlan, data=data) + importer.notify_proposals() + + assert IXFMemberData.objects.count() == 1 + assert NetworkIXLan.objects.count() == 0 + + assert_ix_email(ixlan.ix, email_info) + assert_no_network_email(network) + + assert_idempotent(importer, ixlan, data, save=save) + + else: + assert IXFMemberData.objects.count() == 1 + assert NetworkIXLan.objects.count() == 0 + assert_no_emails() + + # assert ( + # "IPv4 195.69.147.250 does not match any prefix on this ixlan" + # in ixfmemberdata.error + # ) + + assert_idempotent(importer, ixlan, data, save=save) @pytest.mark.django_db -def test_add_netixlan_conflict(entities, capsys): +def test_add_netixlan_conflict(entities, save): """ No NetIXLan exists. Network allows auto updates. While remote IXF data has information to create a new NetIXLan, there are conflicts with the ipaddresses that prevent it from being created. There is no local-ixf so we create one. """ + data = setup_test_data("ixf.member.0") network = entities["net"]["UPDATE_ENABLED"] ixlan = entities["ixlan"][1] # So we have conflicts with IPAddresses + # invalid prefix space error will only be raised if + # the other ipaddress (v6 in this case) matches + # so we move that prefix over + + if network.ipv6_support: + ixpfx = entities["ixlan"][0].ixpfx_set.filter(protocol="IPv6").first() + else: + ixpfx = entities["ixlan"][0].ixpfx_set.filter(protocol="IPv4").first() + + ixpfx.ixlan = entities["ixlan"][1] + ixpfx.save() + importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() ixfmemberdata = IXFMemberData.objects.first() - assert IXFMemberData.objects.count() == 1 - assert ( - "IPv4 195.69.147.250 does not match any prefix on this ixlan" - in ixfmemberdata.error - ) - assert_ticket_exists([(network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) + if network.ipv4_support and network.ipv6_support: + assert IXFMemberData.objects.count() == 1 + email_info = [ + ("CREATE", network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1") + ] + assert_ix_email(ixlan.ix, email_info) + assert_no_network_email(network) - stdout = capsys.readouterr().out - assert_email_sent( - stdout, (network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1") - ) + assert "does not match any prefix on this ixlan" in ixfmemberdata.error + + # Assert that message to IX also includes the error + assert ( + "A validation error was raised when the IX-F importer attempted to process this change." + in IXFImportEmail.objects.filter(ix=ixlan.ix.id).first().message + ) + + else: + + # invalid ip is on unsupported protocol, so it was ignored + # #771 + + assert IXFMemberData.objects.count() == 0 + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 0 - assert_ticket_exists([(network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) + assert_idempotent(importer, ixlan, data, save=save) @pytest.mark.django_db -def test_suggest_add_local_ixf(entities, capsys): +def test_suggest_add_local_ixf(entities, use_ip, save): """ The netixlan described in the remote-ixf doesn't exist, but there is a relationship btw the network and ix (ie a different netixlan). The network does not have automatic updates. There's a local-ixf that matches the remote-ixf so we do nothing. """ - data = setup_test_data("ixf.member.2") + data = setup_test_data("ixf.member.3") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] @@ -376,8 +608,8 @@ def test_suggest_add_local_ixf(entities, capsys): ixlan=ixlan, asn=network.asn, speed=10000, - ipaddr4="195.69.150.250", - ipaddr6="2001:7f8:1::a500:2906:3", + ipaddr4=use_ip(4, "195.69.147.251"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:3"), status="ok", is_rs_peer=True, operational=True, @@ -385,36 +617,89 @@ def test_suggest_add_local_ixf(entities, capsys): ) preexisting_ixfmember_data = IXFMemberData.objects.create( - asn=1001, # Matches remote-ixf data - ipaddr4="195.69.147.250", # Matches remote-ixf data - ipaddr6="2001:7f8:1::a500:2906:1", # Matches remote-ixf data + # Matches remote-ixf data + asn=1001, + # Matches remote-ixf data + ipaddr4=use_ip(4, "195.69.147.250"), + # Matches remote-ixf data + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), operational=True, is_rs_peer=True, + data=json.dumps({"foo": "bar"}), status="ok", ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 1 + if (not network.ipv4_support and use_ip(4) and not use_ip(6)) or ( + not network.ipv6_support and use_ip(6) and not use_ip(4) + ): + # edge case, supported protocols changed + # one of the ips on an unsupported protocol + # effectively changing the signature, send + # out create notifications for both - stdout = capsys.readouterr().out - assert stdout == "" - assert_no_ticket_exists() + assert IXFMemberData.objects.count() == 3 + assert NetworkIXLan.objects.count() == 1 + + email_info = [ + ( + "CREATE", + network.asn, + use_ip(6, "195.69.147.251"), + use_ip(4, "2001:7f8:1::a500:2906:3"), + ) + ] + + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) + + elif (network.ipv4_support and network.ipv6_support and not use_ip(4)) or ( + network.ipv4_support and network.ipv6_support and not use_ip(6) + ): + + # edge case, supported protocols changed + # effectively changing the signature, send + # out modify to the existing netixlan and re-create + # for the existing ixfmemberdata + + assert IXFMemberData.objects.count() == 3 + assert NetworkIXLan.objects.count() == 1 + + email_info = [ + ("CREATE", network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1"), + ( + "MODIFY", + network.asn, + use_ip(4, "195.69.147.251"), + use_ip(6, "2001:7f8:1::a500:2906:3"), + ), + ] + + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) + + else: + assert IXFMemberData.objects.count() == 1 + assert NetworkIXLan.objects.count() == 1 + + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 1 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data, save=save) @pytest.mark.django_db -def test_suggest_add(entities, capsys): +def test_suggest_add(entities, use_ip, save): """ The netixlan described in the remote-ixf doesn't exist, but there is a relationship btw the network and ix (ie a different netixlan). @@ -424,7 +709,7 @@ def test_suggest_add(entities, capsys): network and IX. """ - data = setup_test_data("ixf.member.2") # asn1001 + data = setup_test_data("ixf.member.3") # asn1001 network = entities["net"]["UPDATE_DISABLED"] # asn1001 ixlan = entities["ixlan"][0] @@ -436,8 +721,8 @@ def test_suggest_add(entities, capsys): ixlan=ixlan, asn=network.asn, speed=10000, - ipaddr4="195.69.150.250", - ipaddr6="2001:7f8:1::a500:2906:3", + ipaddr4=use_ip(4, "195.69.147.251"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:3"), status="ok", is_rs_peer=True, operational=True, @@ -445,30 +730,105 @@ def test_suggest_add(entities, capsys): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 1 + print(importer.log) - log = importer.log["data"][0] - assert log["action"] == "suggest-add" + if (not network.ipv4_support and use_ip(4) and not use_ip(6)) or ( + not network.ipv6_support and use_ip(6) and not use_ip(4) + ): + # edge case, supported protocols changed + # one of the ips on an unsupported protocol + # effectively changing the signature, send + # out create with the apprp - stdout = capsys.readouterr().out - assert_email_sent( - stdout, (network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1") - ) + assert IXFMemberData.objects.count() == 3 + assert NetworkIXLan.objects.count() == 1 - assert_ticket_exists([(1001, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) + email_info = [ + ( + "CREATE", + network.asn, + use_ip(6, "195.69.147.250"), + use_ip(4, "2001:7f8:1::a500:2906:1"), + ) + ] + + log_250 = importer.log["data"][0] + log_251 = importer.log["data"][1] + + assert log_250["action"] == "suggest-add" + assert log_251["action"] == "suggest-modify" + + if use_ip(4): + assert log_250["peer"]["ipaddr4"] == "" + assert log_251["peer"]["ipaddr4"] == "" + assert log_250["peer"]["ipaddr6"] == "2001:7f8:1::a500:2906:1" + assert log_251["peer"]["ipaddr6"] == "2001:7f8:1::a500:2906:3" + elif use_ip(6): + assert log_250["peer"]["ipaddr4"] == "195.69.147.250" + assert log_251["peer"]["ipaddr4"] == "195.69.147.251" + assert log_250["peer"]["ipaddr6"] == "" + assert log_251["peer"]["ipaddr6"] == "" + + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) + + elif (network.ipv4_support and network.ipv6_support and not use_ip(4)) or ( + network.ipv4_support and network.ipv6_support and not use_ip(6) + ): + + # edge case, supported protocols changed + # effectively changing the signature, send + # out modify to the existing netixlan and re-create + # for the existing ixfmemberdata + + assert IXFMemberData.objects.count() == 3 + assert NetworkIXLan.objects.count() == 1 + + email_info = [ + ("CREATE", network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1"), + ( + "MODIFY", + network.asn, + use_ip(4, "195.69.147.251"), + use_ip(6, "2001:7f8:1::a500:2906:3"), + ), + ] + + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) + + else: + assert IXFMemberData.objects.count() == 1 + assert NetworkIXLan.objects.count() == 1 + + log = importer.log["data"][0] + assert log["action"] == "suggest-add" + + if network.ipv4_support and network.ipv6_support: + email_info = [ + ("CREATE", network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1") + ] + elif network.ipv4_support: + email_info = [("CREATE", network.asn, "195.69.147.250", None)] + elif network.ipv6_support: + email_info = [("CREATE", network.asn, None, "2001:7f8:1::a500:2906:1")] + + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 1 - assert_ticket_exists([(1001, "195.69.147.250", "2001:7f8:1::a500:2906:1")]) + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_suggest_add_no_netixlan_local_ixf(entities, capsys): +def test_suggest_add_no_netixlan_local_ixf(entities, use_ip, save): """ There isn't any netixlan between ix and network. Network does not have automatic updates. @@ -479,9 +839,12 @@ def test_suggest_add_no_netixlan_local_ixf(entities, capsys): ixlan = entities["ixlan"][0] preexisting_ixfmember_data = IXFMemberData.objects.create( - asn=1001, # Matches remote-ixf data - ipaddr4="195.69.147.250", # Matches remote-ixf data - ipaddr6="2001:7f8:1::a500:2906:1", # Matches remote-ixf data + # Matches remote-ixf data + asn=1001, + # Matches remote-ixf data + ipaddr4=use_ip(4, "195.69.147.250"), + # Matches remote-ixf data + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), ixlan=ixlan, speed=10000, fetched=datetime.datetime.now(datetime.timezone.utc), @@ -491,24 +854,60 @@ def test_suggest_add_no_netixlan_local_ixf(entities, capsys): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 0 - stdout = capsys.readouterr().out - assert stdout == "" - assert_no_ticket_exists() + if (not network.ipv4_support and use_ip(4) and not use_ip(6)) or ( + not network.ipv6_support and use_ip(6) and not use_ip(4) + ): + + # edge case where the network has only one ip + # set and its on an unsupported protocol + # we re-create the ixfmemberdata and re notify the + # network + + email_info = [ + ( + "CREATE", + network.asn, + use_ip(6, "195.69.147.250"), + use_ip(4, "2001:7f8:1::a500:2906:1"), + ) + ] + + assert_network_email(network, email_info) + + elif (network.ipv4_support and network.ipv6_support and not use_ip(4)) or ( + network.ipv4_support and network.ipv6_support and not use_ip(6) + ): + + # edge case, supported protocols changed + # effectively changing the signature, send + # we re-create the ixfmemberdata and re notify the + # network + + email_info = [ + ("CREATE", network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1"), + ] + + assert_network_email(network, email_info) + + else: + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 0 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_suggest_add_no_netixlan(entities, capsys): +def test_suggest_add_no_netixlan(entities, use_ip, save): """ There isn't any netixlan between ix and network. Network does not have automatic updates. @@ -521,7 +920,12 @@ def test_suggest_add_no_netixlan(entities, capsys): ixlan = entities["ixlan"][0] importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert IXFMemberData.objects.count() == 1 assert NetworkIXLan.objects.count() == 0 @@ -529,25 +933,30 @@ def test_suggest_add_no_netixlan(entities, capsys): log = importer.log["data"][0] assert log["action"] == "suggest-add" - stdout = capsys.readouterr().out - assert_email_sent( - stdout, (network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1") - ) - assert_no_ticket_exists() + if network.ipv4_support and network.ipv6_support: + email_info = [ + ("CREATE", network.asn, "195.69.147.250", "2001:7f8:1::a500:2906:1") + ] + elif network.ipv4_support: + email_info = [("CREATE", network.asn, "195.69.147.250", None)] + elif network.ipv6_support: + email_info = [("CREATE", network.asn, None, "2001:7f8:1::a500:2906:1")] + + assert_network_email(network, email_info) + assert_no_ix_email(ixlan.ix) # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 1 - assert NetworkIXLan.objects.count() == 0 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_single_ipaddr_matches(entities, capsys): +def test_single_ipaddr_matches(entities, save): """ - If only one ipaddr matches, it's the same as not matching at all. + If only one ipaddr matches, that's still a conflict. Here we expect to delete the two netixlans and create a new one from the remote-ixf. + + There are no notifications since updates are enabled. """ data = setup_test_data("ixf.member.0") @@ -583,24 +992,233 @@ def test_single_ipaddr_matches(entities, capsys): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() - assert len(importer.log["data"]) == 3 - assert NetworkIXLan.objects.filter(status="ok").count() == 1 + if not network.ipv4_support or not network.ipv6_support: - assert importer.log["data"][0]["action"] == "delete" - assert importer.log["data"][1]["action"] == "delete" - assert importer.log["data"][2]["action"] == "add" + # edge case + # + # one protocol is turned off, in this case we actually + # delete the netixlan on the unsupported protocol and keep + # and keep the one on the supported protocol (since we + # cant update it with the unsupported ip either) + + assert len(importer.log["data"]) == 1 + assert importer.log["data"][0]["action"] == "delete" + else: + assert len(importer.log["data"]) == 3 + + assert NetworkIXLan.objects.filter(status="ok").count() == 1 + + assert importer.log["data"][0]["action"] == "delete" + assert importer.log["data"][1]["action"] == "delete" + assert importer.log["data"][2]["action"] == "add" + + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 0 - assert NetworkIXLan.objects.filter(status="ok").count() == 1 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_delete(entities): +def test_single_ipaddr_matches_no_auto_update(entities, use_ip, save): + """ + For the Netixlan, Ipaddr4 matches the remote date but Ipaddr6 is Null. + In terms of IXFMemberData, we suggest delete the two Netixlans and create a new one. + In terms of notifications, we consolidate that deletions + addition into a single + MODIFY proposal. + This tests the changes in issue #770. + """ + + data = setup_test_data("ixf.member.1") + network = entities["net"]["UPDATE_DISABLED"] + ixlan = entities["ixlan"][0] + + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=10000, + ipaddr4=use_ip(4, "195.69.147.250"), + ipaddr6=use_ip(6, "2001:7f8:1::a500:2906:1"), + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + + importer.update(ixlan, data=data) + importer.notify_proposals() + + if use_ip(4) and use_ip(6): + assert_no_emails() + assert IXFMemberData.objects.count() == 0 + assert NetworkIXLan.objects.count() == 1 + + elif ( + (not network.ipv6_support or not network.ipv4_support) + and not (network.ipv4_support and use_ip(6) and not use_ip(4)) + and not (network.ipv6_support and use_ip(4) and not use_ip(6)) + ): + + assert len(importer.log["data"]) == 0 + assert_no_emails() + + else: + + # Assert NetworkIXLan is unchanged + assert NetworkIXLan.objects.filter(status="ok").count() == 1 + + # We consolidate notifications into a single MODIFY + assert len(importer.log["data"]) == 1 + assert importer.log["data"][0]["action"] == "suggest-modify" + + ixf_member_del = IXFMemberData.objects.get(id=1) + ixf_member_add = IXFMemberData.objects.get(id=2) + + assert ixf_member_del.requirement_of == ixf_member_add + assert ixf_member_add.action == "modify" + + netixlan = NetworkIXLan.objects.filter(status="ok").first() + + email_info = [("MODIFY", network.asn, netixlan.ipaddr4, netixlan.ipaddr6)] + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) + + # Test idempotent + assert_idempotent(importer, ixlan, data) + + +@pytest.mark.django_db +def test_two_missing_ipaddrs_no_auto_update(entities, save): + """ + Now we have two Netixlans, each missing 1 ipaddr. The remote has data for a single netixlan with + both ip addressses. + + In terms of IXFMemberData, we suggest delete the two Netixlans and create a new one. + In terms of notifications, we consolidate that deletions + addition into a single + MODIFY proposal. + + This tests the changes in issue #770. + """ + + data = setup_test_data("ixf.member.1") + network = entities["net"]["UPDATE_DISABLED"] + ixlan = entities["ixlan"][0] + + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=10000, + ipaddr4="195.69.147.250", + ipaddr6=None, + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=10000, + ipaddr4=None, + ipaddr6="2001:7f8:1::a500:2906:1", + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + + importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + + importer.update(ixlan, data=data) + importer.notify_proposals() + # Assert NetworkIXLans are unchanged + assert NetworkIXLan.objects.filter(status="ok").count() == 2 + + if not network.ipv4_support or not network.ipv6_support: + + # only one of the protocols is supported by the network + # suggest deletion of the other ip address + + assert IXFMemberData.objects.count() == 1 + + assert importer.log["data"][0]["action"] == "suggest-delete" + + if not network.ipv4_support: + ipaddr4 = "195.69.147.250" + else: + ipaddr4 = None + + if not network.ipv6_support: + ipaddr6 = "2001:7f8:1::a500:2906:1" + else: + ipaddr6 = None + + email_info = [("REMOVE", network.asn, ipaddr4, ipaddr6)] + + assert_network_email(network, email_info) + assert_ix_email(ixlan.ix, email_info) + + else: + # On the IXFMemberData side, we create instances + # for two deletions and one addition. + # The deletions will be the requirement of the addition. + + assert IXFMemberData.objects.count() == 3 + ixfmdata_d4 = IXFMemberData.objects.filter( + ipaddr4="195.69.147.250", ipaddr6=None + ).first() + ixfmdata_d6 = IXFMemberData.objects.filter( + ipaddr4=None, ipaddr6="2001:7f8:1::a500:2906:1" + ).first() + ixfmdata_m = IXFMemberData.objects.filter( + ipaddr4="195.69.147.250", ipaddr6="2001:7f8:1::a500:2906:1" + ).first() + + assert ixfmdata_d4.action == "delete" + assert ixfmdata_d6.action == "delete" + assert ixfmdata_m.action == "modify" + assert ixfmdata_d4.requirement_of == ixfmdata_m + assert ixfmdata_d6.requirement_of == ixfmdata_m + + assert ixfmdata_m.primary_requirement == ixfmdata_d4 + assert ixfmdata_m.secondary_requirements == [ixfmdata_d6] + + # We consolidate notifications into a single MODIFY + assert len(importer.log["data"]) == 1 + assert importer.log["data"][0]["action"] == "suggest-modify" + + # We only create an email for the primary requirement + email_info_4 = [("MODIFY", network.asn, "195.69.147.250", "IPv6 not set")] + assert IXFImportEmail.objects.count() == 2 + assert_ix_email(ixlan.ix, email_info_4) + assert_network_email(network, email_info_4) + + # Test idempotent + assert_idempotent(importer, ixlan, data) + + +@pytest.mark.django_db +def test_delete(entities, save): """ The ixf-remote doesn't contain an existing NetIXlan. Automatic updates are enabled so we delete it. @@ -639,19 +1257,22 @@ def test_delete(entities): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert len(importer.log["data"]) == 1 log = importer.log["data"][0] assert log["action"] == "delete" assert NetworkIXLan.objects.filter(status="ok").count() == 1 + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 0 - assert NetworkIXLan.objects.filter(status="ok").count() == 1 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data) # test rollback import_log = IXLanIXFMemberImportLog.objects.first() @@ -660,7 +1281,7 @@ def test_delete(entities): @pytest.mark.django_db -def test_suggest_delete_local_ixf_has_flag(entities, capsys): +def test_suggest_delete_local_ixf_has_flag(entities, save): """ Automatic updates for network are disabled. There is no remote-ixf corresponding to an existing netixlan. @@ -714,28 +1335,30 @@ def test_suggest_delete_local_ixf_has_flag(entities, capsys): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert NetworkIXLan.objects.count() == 2 assert IXFMemberData.objects.count() == 1 - assert_no_ticket_exists() + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert NetworkIXLan.objects.count() == 2 - assert IXFMemberData.objects.count() == 1 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_suggest_delete_local_ixf_no_flag(entities, capsys): +def test_suggest_delete_local_ixf_no_flag(entities, save): """ Automatic updates for network are disabled. There is no remote-ixf corresponding to an existing netixlan. There is a local-ixf corresponding to that netixlan but it does not flag it for deletion. - We flag the local-ixf for deletion, make a ticket, and email the ix and network. + We flag the local-ixf for deletion, and email the ix and network. """ data = setup_test_data("ixf.member.1") network = entities["net"]["UPDATE_DISABLED"] @@ -802,32 +1425,32 @@ def test_suggest_delete_local_ixf_no_flag(entities, capsys): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert importer.log["data"][0]["action"] == "suggest-delete" assert NetworkIXLan.objects.count() == 2 - # Test failing, IXFMember is getting resolved - # instead of being flagged for deletion. assert IXFMemberData.objects.count() == 1 - assert_ticket_exists([("AS1001", "195.69.147.251", "No IPv6")]) - stdout = capsys.readouterr().out - assert_email_sent(stdout, (1001, "195.69.147.251", "No IPv6")) + email_info = [("REMOVE", 1001, "195.69.147.251", "IPv6 not set")] + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) # Test idempotent - importer.update(ixlan, data=data) - assert NetworkIXLan.objects.count() == 2 - assert IXFMemberData.objects.count() == 1 - assert_ticket_exists([("AS1001", "195.69.147.251", "No IPv6")]) + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_suggest_delete_no_local_ixf(entities, capsys): +def test_suggest_delete_no_local_ixf(entities, save): """ Automatic updates for network are disabled. There is no remote-ixf corresponding to an existing netixlan. - We flag the local-ixf for deletion, make a ticket, and email the ix and network. + We flag the local-ixf for deletion, and email the ix and network. """ data = setup_test_data("ixf.member.1") @@ -863,25 +1486,27 @@ def test_suggest_delete_no_local_ixf(entities, capsys): ) importer = ixf.Importer() + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert importer.log["data"][0]["action"] == "suggest-delete" assert NetworkIXLan.objects.count() == 2 assert IXFMemberData.objects.count() == 1 - assert_ticket_exists([("AS1001", "195.69.147.251", "No IPv6")]) - stdout = capsys.readouterr().out - assert_email_sent(stdout, ("AS1001", "195.69.147.251", "No IPv6")) + email_info = [("REMOVE", 1001, "195.69.147.251", "IPv6 not set")] + assert_ix_email(ixlan.ix, email_info) + assert_network_email(network, email_info) # Test idempotent - importer.update(ixlan, data=data) - assert NetworkIXLan.objects.count() == 2 - assert IXFMemberData.objects.count() == 1 - assert_ticket_exists([("AS1001", "195.69.147.251", "No IPv6")]) + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_mark_invalid_remote_w_local_ixf_auto_update(entities, capsys): +def test_mark_invalid_remote_w_local_ixf_auto_update(entities, save): """ Our network allows automatic updates. Remote-ixf[as,ip4,ip6] contains invalid data **but** it can be parsed. @@ -917,7 +1542,7 @@ def test_mark_invalid_remote_w_local_ixf_auto_update(entities, capsys): operational=True, is_rs_peer=True, status="ok", - error="Invalid speed value: this is not valid", + error=json.dumps({"speed": "Invalid speed value: this is not valid"}), ) preexisting_ixfmember_data = IXFMemberData.objects.create( @@ -930,34 +1555,35 @@ def test_mark_invalid_remote_w_local_ixf_auto_update(entities, capsys): operational=True, is_rs_peer=True, status="ok", - error="Invalid speed value: this is not valid", + error=json.dumps({"speed": "Invalid speed value: this is not valid"}), ) importer = ixf.Importer() data = importer.sanitize(data) + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert IXFMemberData.objects.count() == 2 - stdout = capsys.readouterr().out - assert stdout == "" - assert_no_ticket_exists() + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 2 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_mark_invalid_remote_auto_update(entities, capsys): +def test_mark_invalid_remote_auto_update(entities, save): """ The network does enable automatic updates. Remote-ixf[as,ip4,ip6] contains invalid data **but** it can be parsed. There is not a local-ixf flagging that invalid data. We create a local-ixf[as,ip4,ip6] and flag as invalid Email the ix - Create/Update a ticket for admin com """ + data = setup_test_data("ixf.member.invalid.0") network = entities["net"]["UPDATE_ENABLED"] ixlan = entities["ixlan"][0] @@ -979,41 +1605,52 @@ def test_mark_invalid_remote_auto_update(entities, capsys): importer = ixf.Importer() data = importer.sanitize(data) - importer.update(ixlan, data=data) - print([(n.speed, n.ipaddr4, n.ipaddr6, n.asn) for n in NetworkIXLan.objects.all()]) - print(importer.log) + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + + importer.update(ixlan, data=data) + importer.notify_proposals() + assert NetworkIXLan.objects.count() == 1 assert IXFMemberData.objects.count() == 2 - ERROR_MESSAGE = "Invalid speed value" - stdout = capsys.readouterr().out - assert ERROR_MESSAGE in stdout - assert_ticket_exists( - [ - ("AS2906", "195.69.147.100", "2001:7f8:1::a500:2906:4"), - ("AS2906", "195.69.147.200", "2001:7f8:1::a500:2906:2"), + + # We email to say there is invalid data + if network.ipv4_support and network.ipv6_support: + email_info = [ + ("CREATE", network.asn, "195.69.147.100", "2001:7f8:1::a500:2906:4"), + ("MODIFY", network.asn, "195.69.147.200", "2001:7f8:1::a500:2906:2"), ] + elif network.ipv4_support: + email_info = [ + ("CREATE", network.asn, "195.69.147.100", None), + ("MODIFY", network.asn, "195.69.147.200", "2001:7f8:1::a500:2906:2"), + ] + elif network.ipv6_support: + email_info = [ + ("CREATE", network.asn, None, "2001:7f8:1::a500:2906:4"), + ("MODIFY", network.asn, "195.69.147.200", "2001:7f8:1::a500:2906:2"), + ] + + assert_ix_email(ixlan.ix, email_info) + assert ( + "Invalid speed value: This is invalid" + in IXFImportEmail.objects.filter(ix=ixlan.ix.id).first().message ) # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 2 - assert_ticket_exists( - [ - ("AS2906", "195.69.147.100", "2001:7f8:1::a500:2906:4"), - ("AS2906", "195.69.147.200", "2001:7f8:1::a500:2906:2"), - ] - ) + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_mark_invalid_remote_w_local_ixf_no_auto_update(entities, capsys): +def test_mark_invalid_remote_w_local_ixf_no_auto_update(entities, save): """ Our network does not allow automatic updates. Remote-ixf[as,ip4,ip6] contains invalid data **but** it can be parsed. There is already a local-ixf flagging that invalid data. Do nothing. """ + data = setup_test_data("ixf.member.invalid.1") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] @@ -1043,7 +1680,7 @@ def test_mark_invalid_remote_w_local_ixf_no_auto_update(entities, capsys): operational=True, is_rs_peer=True, status="ok", - error="Invalid speed value: this is not valid", + error=json.dumps({"speed": "Invalid speed value: this is not valid"}), ) preexisting_ixfmember_data = IXFMemberData.objects.create( @@ -1056,34 +1693,35 @@ def test_mark_invalid_remote_w_local_ixf_no_auto_update(entities, capsys): operational=True, is_rs_peer=True, status="ok", - error="Invalid speed value: this is not valid", + error=json.dumps({"speed": "Invalid speed value: this is not valid"}), ) importer = ixf.Importer() data = importer.sanitize(data) + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert IXFMemberData.objects.count() == 2 - stdout = capsys.readouterr().out - assert stdout == "" - assert_no_ticket_exists() + assert_no_emails() # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 2 - assert_no_ticket_exists() + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_mark_invalid_remote_no_auto_update(entities, capsys): +def test_mark_invalid_remote_no_auto_update(entities, save): """ Our network does not allow automatic updates. Remote-ixf[as,ip4,ip6] contains invalid data **but** it can be parsed. There is not a local-ixf flagging that invalid data. We create a local-ixf[as,ip4,ip6] and flag as invalid Email the ix - Create/Update a ticket for admin com """ + data = setup_test_data("ixf.member.invalid.1") network = entities["net"]["UPDATE_DISABLED"] ixlan = entities["ixlan"][0] @@ -1105,32 +1743,45 @@ def test_mark_invalid_remote_no_auto_update(entities, capsys): importer = ixf.Importer() data = importer.sanitize(data) + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() assert IXFMemberData.objects.count() == 2 - ERROR_MESSAGE = "Invalid speed value" - stdout = capsys.readouterr().out - assert ERROR_MESSAGE in stdout - assert_ticket_exists( - [ - ("AS1001", "195.69.147.100", "2001:7f8:1::a500:2906:4"), - ("AS1001", "195.69.147.200", "2001:7f8:1::a500:2906:2"), + + # We send an email about the updates + # But it also contains information about the invalid speed + if network.ipv4_support and network.ipv6_support: + email_info = [ + ("CREATE", network.asn, "195.69.147.100", "2001:7f8:1::a500:2906:4"), + ("MODIFY", network.asn, "195.69.147.200", "2001:7f8:1::a500:2906:2"), ] + elif network.ipv4_support: + email_info = [ + ("CREATE", network.asn, "195.69.147.100", None), + ("MODIFY", network.asn, "195.69.147.200", "2001:7f8:1::a500:2906:2"), + ] + elif network.ipv6_support: + email_info = [ + ("CREATE", network.asn, None, "2001:7f8:1::a500:2906:4"), + ("MODIFY", network.asn, "195.69.147.200", "2001:7f8:1::a500:2906:2"), + ] + + assert_ix_email(ixlan.ix, email_info) + assert ( + "Invalid speed value: This is invalid" in IXFImportEmail.objects.first().message ) + assert_no_network_email(network) # Test idempotent - importer.update(ixlan, data=data) - assert IXFMemberData.objects.count() == 2 - assert_ticket_exists( - [ - ("AS1001", "195.69.147.100", "2001:7f8:1::a500:2906:4"), - ("AS1001", "195.69.147.200", "2001:7f8:1::a500:2906:2"), - ] - ) + assert_idempotent(importer, ixlan, data) @pytest.mark.django_db -def test_remote_cannot_be_parsed(entities, capsys): +def test_remote_cannot_be_parsed(entities, save): """ Remote cannot be parsed. We create a ticket, email the IX, and create a lock. """ @@ -1139,27 +1790,32 @@ def test_remote_cannot_be_parsed(entities, capsys): start = datetime.datetime.now(datetime.timezone.utc) importer = ixf.Importer() importer.sanitize(data) + + if not save: + return assert_idempotent(importer, ixlan, data, save=False) + importer.update(ixlan, data=data) + importer.notify_proposals() ERROR_MESSAGE = "No entries in any of the vlan_list lists, aborting" assert importer.ixlan.ixf_ixp_import_error_notified > start # This sets the lock assert ERROR_MESSAGE in importer.ixlan.ixf_ixp_import_error - stdout = capsys.readouterr().out - assert ERROR_MESSAGE in stdout - assert DeskProTicket.objects.count() == 1 + assert ( + ERROR_MESSAGE in IXFImportEmail.objects.filter(ix=ixlan.ix.id).first().message + ) # Assert idempotent / lock importer.sanitize(data) importer.update(ixlan, data=data) - stdout = capsys.readouterr().out - assert stdout == "" - assert DeskProTicket.objects.count() == 1 + + assert ERROR_MESSAGE in importer.ixlan.ixf_ixp_import_error + assert IXFImportEmail.objects.filter(ix=ixlan.ix.id).count() == 1 def test_validate_json_schema(): schema_url_base = "https://raw.githubusercontent.com/euro-ix/json-schemas/master/versions/ixp-member-list-{}.schema.json" - for v in ["0.4", "0.5", "0.6", "0.7"]: + for v in ["0.4", "0.5", "0.6", "0.7", "1.0"]: schema = requests.get(schema_url_base.format(v)).json() for fn in [ @@ -1177,9 +1833,270 @@ def test_validate_json_schema(): jsonschema.validate(data, schema) +@pytest.mark.django_db +def test_create_deskpro_tickets_after_x_days(entities): + data = setup_test_data("ixf.member.2") + network = entities["net"]["UPDATE_DISABLED"] + ixlan = entities["ixlan"][0] + + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=10000, + ipaddr4="195.69.147.250", + ipaddr6="2001:7f8:1::a500:2906:1", + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=10000, + ipaddr4="195.69.147.240", + ipaddr6="2001:7f8:1::a500:2905:1", + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + importer = ixf.Importer() + importer.update(ixlan, data=data) + importer.notify_proposals() + + for ixfmd in IXFMemberData.objects.all(): + # Edit so that they've been created two weeks ago + ixfmd.created = datetime.datetime.now( + datetime.timezone.utc + ) - datetime.timedelta(days=14) + ixfmd.save() + + importer.update(ixlan, data=data) + + # Assert IXFMemberData still the same + assert IXFMemberData.objects.count() == 4 + + # Assert DeskProTickets are created + assert DeskProTicket.objects.count() == 4 + + # Assert emails go to IX and Network for each Ticket + deskpro_refs = [dpt.deskpro_ref for dpt in DeskProTicket.objects.all()] + for dpt in deskpro_refs: + assert IXFImportEmail.objects.filter( + subject__contains=dpt, ix=ixlan.ix.id + ).exists() + assert IXFImportEmail.objects.filter( + subject__contains=dpt, net=network.id + ).exists() + + +@pytest.mark.django_db +def test_create_deskpro_tickets_no_contacts(entities): + data = setup_test_data("ixf.member.2") + network = entities["net"]["UPDATE_DISABLED"] + ixlan = entities["ixlan"][0] + ix = ixlan.ix + + # Delete contacts + for netcontact in entities["netcontact"]: + netcontact.delete() + + ix.tech_email = "" + ix.save() + + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=10000, + ipaddr4="195.69.147.250", + ipaddr6="2001:7f8:1::a500:2906:1", + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=10000, + ipaddr4="195.69.147.240", + ipaddr6="2001:7f8:1::a500:2905:1", + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + importer = ixf.Importer() + importer.update(ixlan, data=data) + importer.notify_proposals() + + # Assert Tickets are created immediately + assert DeskProTicket.objects.count() == 4 + + +@pytest.mark.django_db +def test_resolve_deskpro_ticket(entities): + data = setup_test_data("ixf.member.1") + network = entities["net"]["UPDATE_DISABLED"] + ixlan = entities["ixlan"][0] + + entities["netixlan"].append( + NetworkIXLan.objects.create( + network=network, + ixlan=ixlan, + asn=network.asn, + speed=20000, + ipaddr4="195.69.147.250", + ipaddr6="2001:7f8:1::a500:2906:1", + status="ok", + is_rs_peer=True, + operational=True, + ) + ) + importer = ixf.Importer() + importer.update(ixlan, data=data) + importer.notify_proposals() + + assert IXFMemberData.objects.count() == 1 + ixf_member_data = IXFMemberData.objects.first() + + assert not ixf_member_data.deskpro_id + assert not ixf_member_data.deskpro_ref + + # Edit so that they've been created two weeks ago + ixf_member_data.created = datetime.datetime.now( + datetime.timezone.utc + ) - datetime.timedelta(days=14) + ixf_member_data.save_without_update() + + # re run importer to create tickets + importer.notifications = [] + importer.update(ixlan, data=data) + importer.notify_proposals() + + assert DeskProTicket.objects.count() == 1 + + ticket = DeskProTicket.objects.first() + assert ticket.deskpro_id + assert ticket.deskpro_ref + + # 1 member data instance + assert IXFMemberData.objects.count() == 1 + ixf_member_data = IXFMemberData.objects.first() + assert ixf_member_data.deskpro_id == ticket.deskpro_id + assert ixf_member_data.deskpro_ref == ticket.deskpro_ref + + # 4 emails total + # 2 emails for initial consolidated notification + # 2 emails for ticket + assert IXFImportEmail.objects.count() == 4 + conflict_emails = IXFImportEmail.objects.filter(subject__icontains="conflict") + consolid_emails = IXFImportEmail.objects.exclude(subject__icontains="conflict") + assert conflict_emails.count() == 2 + + for email in consolid_emails: + + # if network is only supporting one ip protocol + # since the ix is sending both it should be mentioned + if not network.ipv4_support: + assert "IX-F data provides IPv4 addresses" in email.message + if not network.ipv6_support: + assert "IX-F data provides IPv6 addresses" in email.message + + + for email in conflict_emails: + assert ticket.deskpro_ref in email.subject + + # Resolve issue + netixlan = entities["netixlan"][0] + netixlan.speed = 10000 + netixlan.save() + + # Re run import to notify resolution + importer.notifications = [] + importer.update(ixlan, data=data) + importer.notify_proposals() + + # resolved + assert IXFMemberData.objects.count() == 0 + + # resolution tickets created + assert DeskProTicket.objects.count() == 2 + + ticket_r = DeskProTicket.objects.last() + assert ticket_r.deskpro_id == ticket.deskpro_id + assert ticket_r.deskpro_ref == ticket.deskpro_ref + assert "resolved" in ticket_r.body + + conflict_emails = IXFImportEmail.objects.filter(subject__icontains="conflict") + assert conflict_emails.count() == 4 + + for email in conflict_emails.order_by("-id")[:2]: + assert "resolved" in email.message + assert ticket.deskpro_ref in email.subject + + # FIXTURES -@pytest.fixture -def entities(): +@pytest.fixture(params=[True, False]) +def save(request): + return request.param + + +def entities_ipv4_only(_entities): + """ + Same as entities, but network gets configured + to only support IPv4 + """ + for net in _entities["net"].values(): + net.info_ipv6 = False + net.info_unicast = True + net.save() + return _entities + + +def entities_ipv6_only(_entities): + """ + Same as entities, but network gets configured + to only support IPv6 + """ + for net in _entities["net"].values(): + net.info_unicast = False + net.info_ipv6 = True + net.save() + return _entities + + +def entities_ipv4_ipv6_implied(_entities): + """ + Same as entities, but network gets configured + to imply support for both protocols by having + neither set. + """ + for net in _entities["net"].values(): + net.info_unicast = False + net.info_ipv6 = False + net.save() + return _entities + + +def entities_ipv4_ipv6(_entities): + for net in _entities["net"].values(): + net.info_unicast = True + net.info_ipv6 = True + net.save() + return _entities + + +def entities_base(): entities = {} with reversion.create_revision(): entities["org"] = [Organization.objects.create(name="Netflix", status="ok")] @@ -1187,10 +2104,16 @@ def entities(): # create exchange(s) entities["ix"] = [ InternetExchange.objects.create( - name="Test Exchange One", org=entities["org"][0], status="ok" + name="Test Exchange One", + org=entities["org"][0], + status="ok", + tech_email="ix1@localhost", ), InternetExchange.objects.create( - name="Test Exchange Two", org=entities["org"][0], status="ok" + name="Test Exchange Two", + org=entities["org"][0], + status="ok", + tech_email="ix2@localhost", ), ] @@ -1239,6 +2162,8 @@ def entities(): allow_ixp_update=True, status="ok", irr_as_set="AS-NFLX", + info_unicast=True, + info_ipv6=True, ), "UPDATE_DISABLED": Network.objects.create( name="Network w allow ixp update disabled", @@ -1251,15 +2176,23 @@ def entities(): website="http://netflix.com/", policy_general="Open", policy_url="https://www.netflix.com/openconnect/", + info_unicast=True, + info_ipv6=True, ), } entities["netcontact"] = [ NetworkContact.objects.create( - email="network1@localhost", network=entities["net"]["UPDATE_ENABLED"] + email="network1@localhost", + network=entities["net"]["UPDATE_ENABLED"], + status="ok", + role="Policy", ), NetworkContact.objects.create( - email="network2@localhost", network=entities["net"]["UPDATE_DISABLED"] + email="network2@localhost", + network=entities["net"]["UPDATE_DISABLED"], + status="ok", + role="Policy", ), ] entities["netixlan"] = [] @@ -1271,6 +2204,59 @@ def entities(): return entities +@pytest.fixture( + params=[ + entities_ipv4_ipv6, + entities_ipv4_ipv6_implied, + entities_ipv4_only, + entities_ipv6_only, + ] +) +def entities(request): + _entities = entities_base() + _func = request.param + return _func(_entities) + + +class UseIPAddrWrapper: + + """ + To help test what happens when a network only + sets either ip4 or ip6 address on their netixlan + as well as both + """ + + def __init__(self, use_ipv4, use_ipv6): + self.use_ipv4 = use_ipv4 + self.use_ipv6 = use_ipv6 + + def __call__(self, ipv, value=True): + if ipv == 4: + if self.use_ipv4: + return ipaddress.ip_address(value) + return None + elif ipv == 6: + if self.use_ipv6: + return ipaddress.ip_address(value) + return None + raise ValueError(ipv) + + +@pytest.fixture(params=[(True, True), (True, False), (False, True)]) +def use_ip(request): + """ + Fixture that gives back 3 instances of UseIpAddrWrapper + + 1) use ip4, use ip6 + 2) use ip4, dont use ip6 + 3) dont use ip4, use ip6 + """ + + use_ipv4, use_ipv6 = request.param + + return UseIPAddrWrapper(use_ipv4, use_ipv6) + + # TEST FUNCTIONS def setup_test_data(filename): json_data = {} @@ -1278,10 +2264,7 @@ def setup_test_data(filename): with open( os.path.join( - os.path.dirname(__file__), - "data", - "json_members_list", - f"{filename}.json", + os.path.dirname(__file__), "data", "json_members_list", f"{filename}.json", ), ) as fh: json_data = json.load(fh) @@ -1297,17 +2280,117 @@ def assert_ticket_exists(ticket_info): """ assert DeskProTicket.objects.count() == len(ticket_info) - for i, dpt in enumerate(DeskProTicket.objects.all()): - assert all([str(s) in dpt.subject for s in ticket_info[i]]) + for ticket in DeskProTicket.objects.all(): + print(ticket.subject) + print("-" * 80) + + for asn, ip4, ip6 in ticket_info: + assert DeskProTicket.objects.filter( + subject__endswith=f"AS{asn} {ip4} {ip6}" + ).exists() + + +def assert_network_email(network, email_info): + network_email = IXFImportEmail.objects.filter(net=network.id).first() + print("Network email") + print("Body:") + print(network_email.message) + + for email_i in email_info: + email_str = create_email_str(email_i) + assert email_str in network_email.message + + +def assert_ix_email(ix, email_info): + ix_email = IXFImportEmail.objects.filter(ix=ix.id).first() + print("IX email") + print("Body:") + print(ix_email.message) + for email_i in email_info: + email_str = create_email_str(email_i) + assert email_str in ix_email.message + + +def create_email_str(email): + + email = list(email) + + if not email[2]: + email[2] = "IPv4 not set" + if not email[3]: + email[3] = "IPv6 not set" + + return "{} AS{} - {} - {}".format(*email) def assert_no_ticket_exists(): - """ - Input is a list of tuples containing (asn, ipaddr4, ipaddr6) that should appear - in deskpro tickets - """ assert DeskProTicket.objects.count() == 0 -def assert_email_sent(email_text, email_info): - assert all([str(s) in email_text for s in email_info]) +def assert_no_emails(): + assert IXFImportEmail.objects.count() == 0 + + +def assert_no_ix_email(ix): + assert IXFImportEmail.objects.filter(ix=ix.id).count() == 0 + + +def assert_no_network_email(network): + assert IXFImportEmail.objects.filter(net=network.id).count() == 0 + + +def ticket_list(): + return [(t.id, t.subject) for t in DeskProTicket.objects.all().order_by("id")] + + +def email_list(): + return [(t.id, t.subject) for t in IXFImportEmail.objects.all().order_by("id")] + + +def ixf_member_data_list(): + return [ + (m.id, m.ipaddr4, m.ipaddr6, m.updated) + for m in IXFMemberData.objects.all().order_by("id") + ] + + +def netixlan_list(): + return [ + (n.id, n.status, n.ipaddr4, n.ipaddr6, n.updated) + for n in NetworkIXLan.objects.all().order_by("id") + ] + + +def assert_idempotent(importer, ixlan, data, save=True): + + """ + run the importer for ixlan against data and + assert that there are + + - no changes made to netixlan + - no changes made to deskpro ticket + - no changes made to ixf member data + """ + + ixf_members = ixf_member_data_list() + tickets = ticket_list() + netixlans = netixlan_list() + emails = email_list() + + def assert_no_changes(): + assert ixf_members == ixf_member_data_list() + assert tickets == ticket_list() + assert netixlans == netixlan_list() + assert emails == email_list() + + # Test idempotent + importer.notifications = [] + importer.update(ixlan, data=data, save=save) + importer.notify_proposals() + assert_no_changes() + + # Test idempotent when running against single + # non-existing asn + importer.update(ixlan, data=data, asn=12345, save=save) + importer.notify_proposals() + assert_no_changes() diff --git a/tests/test_partners.py b/tests/test_partners.py index 2108319d..ac5239b4 100644 --- a/tests/test_partners.py +++ b/tests/test_partners.py @@ -31,10 +31,9 @@ class ViewTestCase(TestCase): # create organizations cls.organizations = { - k: - models.Organization.objects.create( - name="Partner Org %s" % k, status="ok" - ) + k: models.Organization.objects.create( + name="Partner Org %s" % k, status="ok" + ) for k in ["a", "b", "c", "d"] } diff --git a/tests/test_sponsors.py b/tests/test_sponsors.py index 11fd374b..f9f336ad 100644 --- a/tests/test_sponsors.py +++ b/tests/test_sponsors.py @@ -32,10 +32,9 @@ class ViewTestCase(TestCase): # create organizations cls.organizations = { - f"{k}": - models.Organization.objects.create( - name="Sponsor Org %s" % k, status="ok" - ) + f"{k}": models.Organization.objects.create( + name="Sponsor Org %s" % k, status="ok" + ) for k in range(1, 7) } diff --git a/tests/test_stats.py b/tests/test_stats.py index 4fd8281e..867042ad 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -33,9 +33,7 @@ def setup_data(): User = get_user_model() for i in range(1, 7): - User.objects.create_user( - f"user_{i}", f"user_{i}@localhost", "secret" - ) + User.objects.create_user(f"user_{i}", f"user_{i}@localhost", "secret") # move users 4, 5, 6 to the past diff --git a/tests/test_veriqueue.py b/tests/test_veriqueue.py index 26175c8f..234ac4fe 100644 --- a/tests/test_veriqueue.py +++ b/tests/test_veriqueue.py @@ -69,10 +69,7 @@ class VeriQueueTests(TestCase): vqi.user = user vqi.save() self.assertEqual( - qs.filter( - subject=f"[test]{vqi.content_type} - {inst}" - ).exists(), - True, + qs.filter(subject=f"[test]{vqi.content_type} - {inst}").exists(), True, ) def test_approve(self):