1
0
mirror of https://github.com/peeringdb/peeringdb.git synced 2024-05-11 05:55:09 +00:00
Files
peeringdb-peeringdb/peeringdb_server/admin_commandline_tools.py

537 lines
16 KiB
Python
Raw Normal View History

"""
Defines CLI wrappers for django commands that should
be executable through the django-admin interface.
Extend the CommandLineToolWrapper class and call the
register_tool decorator to add support for a new django
command to exposed in this manner.
"""
import io
2018-11-08 19:45:21 +00:00
import json
import traceback
2019-02-15 17:46:20 +00:00
2018-11-08 19:45:21 +00:00
import reversion
from dal import autocomplete
from django import forms
Dotf fixes (#781) * fix issue where ix-f import would raise suggestions ipaddresses not that ixlan (#764) * IX-F Suggestions: Leaving the editor and returning to it via back button issues (#765) * IX-F importer: Clicking "Preview" (IXP Update Tools) on /net/ page resulted in 170 ticket resolutions (#769) More robust testing * black formatting (was lost after pyupgrade) * Add regex searching to deskpro ticket subjects * Change operational error * IX-F suggestions: consolidate delete+add (#770) * Add reset functions to commandline tool * Fix commandline tool bugs * Fix reset commandline tool bugs * add commandline tool * Ixlan needs to be set for import commandline tool * Add email model * Add admin view to emails * Allow network and ix to be null * save emails as part of ixf import * Add email model * Add email delete * add iregex search and better comments * fix ixlan selection for import * redefine migration dependencies for this branch * only enable search w start and end char * Add caption to regex search * Remove delete all ixfmemberdata option * [beta] IX-F importer: don't bother about missing IPv{4,6} address when network is not doing IPv{4,6} (#771) * Add cmdline tests * Resolve email conflicts * Add cmd tool reset tests * add autocomplete to commandline tool * Fix email bugs * Fix email migrations * Fix typos * [beta] IX-F importer: prevent Admin Committee overload by initially limiting importer to IXes enabled by AC (#772) * Finalize regex search for emails and deskprotickets * Fix keyword bug * fix typo * protocol-conflict will now be handled in the notification consolidation 771 changes where if the network indicates neither ipv4 nor ipv6 support, it is handled as supporting both (eg the network didnt configure these at all) realised that the importer command re instantiates the `Importer` class for each ixlan it processes, so moved the sending of consolidated notifications (#772) out of the `update` function and into the command itself after its done processing all the ixlans. This means for tests you will need to call `importer.notify_proposals` after `importer.update` to test the consolidated notifications. fixed several MultipleObjectsReturned errors when network switch protocol support in between imports * should be checking for "ix" in the form data (#773) * Fix cmd ixf tests * fix issue in log_peer * Add commit check for reset tool * fix importer bugs * remove dupe IXFImportEmail definition * ixfimportemail support ix__name and net__name searching * ticket resolution responses * Add commit to command ixf import changes * fix modify entry header * remove whitespace in notification about remote data changes * Begin updating tests * ixf-import command line tool to queue * refactor conflict inserts * Update import protocol tests, including tests for 770 * More test edits * Change cmd tests * better ixfmemberdata error handling and fix some test data * dont reset the same ixfmemberdata requirement * fix many bugs add many tests * remove debug message * fix bug during import when consolidating delete+add * fix perfomance issue in IXFMemberData listing * dont show reset flags on prod env * Add regex search tests * Add 772 tests * remove debug output * fix `test_resolve_deskpro_ticket` test * black formatting * remove dupe import * fix issue with unique constraint error handling * add test for ixp / network ip protocol notification * add missing test data Co-authored-by: Stefan Pratter <stefan@20c.com> Co-authored-by: Elliot Frank <elliot@20c.com>
2020-07-26 23:36:27 -05:00
from django.conf import settings
2018-11-08 19:45:21 +00:00
from django.core.management import call_command
from django.db import transaction
from reversion.models import Version
from peeringdb_server import maintenance
2019-12-05 16:57:52 +00:00
from peeringdb_server.models import (
COMMANDLINE_TOOLS,
REFTAG_MAP,
2019-12-05 16:57:52 +00:00
CommandLineTool,
Facility,
InternetExchange,
2019-12-05 16:57:52 +00:00
)
2018-11-08 19:45:21 +00:00
2019-12-05 16:57:52 +00:00
2018-11-08 19:45:21 +00:00
def _(m):
return m
TOOL_MAP = {}
def register_tool(cls):
TOOL_MAP[cls.tool] = cls
return cls
2018-11-08 19:45:21 +00:00
def get_tool(tool_id, form):
"""
Arguments:
tool_id (str): tool_id as it exists in COMMANDLINE_TOOLS
form (django.forms.Form): form instance
Returns:
CommandLineToolWrapper instance
"""
t = TOOL_MAP.get(tool_id)
t = t(form)
return t
def get_tool_from_data(data):
"""
Arguments:
data (dict): dict containing form data, at the very least
needs to have a "tool" key containing the tool_id
Returns:
CommandLineToolWrapper instance
"""
tool_id = data.get("tool")
t = TOOL_MAP.get(tool_id)
form = t.Form(data)
form.is_valid()
t = t(form)
return t
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
class EmptyId:
2018-11-08 19:45:21 +00:00
id = 0
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
class CommandLineToolWrapper:
2018-11-08 19:45:21 +00:00
tool = None
2019-01-04 10:02:28 +00:00
queue = 0
maintenance = False
2018-11-08 19:45:21 +00:00
class Form(forms.Form):
pass
def __init__(self, form):
self.status = 0
self.result = None
self.args = []
self.kwargs = {}
self.form_instance = form
self.set_arguments(form.cleaned_data) # lgtm[py/init-calls-subclass]
# LGTM Note: as this is the last statement in the __init__
# call this is unproblematic, however this should probably
# still be separated in the future (TODO)
2018-11-08 19:45:21 +00:00
@property
def name(self):
return dict(COMMANDLINE_TOOLS).get(self.tool)
@property
def form(self):
return self.Form()
@property
def description(self):
return self.tool
@property
def pretty_result(self):
if not self.result:
return ""
r = []
for line in self.result.split("\n"):
if line.find("[error]") > -1:
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
r.append(f'<div class="error">{line}</div>')
2018-11-08 19:45:21 +00:00
elif line.find("[warning]") > -1:
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
r.append(f'<div class="warning">{line}</div>')
2018-11-08 19:45:21 +00:00
else:
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
r.append(f'<div class="info">{line}</div>')
2018-11-08 19:45:21 +00:00
return "\n".join(r)
def set_arguments(self, form_data):
pass
2019-02-15 17:46:20 +00:00
def validate(self):
pass
def _run(self, command, commit=False, user=None):
r = io.StringIO()
2019-01-04 10:02:28 +00:00
if self.maintenance and commit:
maintenance.on()
2018-11-08 19:45:21 +00:00
try:
2019-02-15 17:46:20 +00:00
self.validate()
2018-11-08 19:45:21 +00:00
if commit:
2019-12-05 16:57:52 +00:00
call_command(
self.tool, *self.args, commit=True, stdout=r, **self.kwargs
)
2018-11-08 19:45:21 +00:00
else:
call_command(self.tool, *self.args, stdout=r, **self.kwargs)
self.result = r.getvalue()
except Exception as inst:
traceback.print_exc()
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
self.result = f"[error] {inst}"
2018-11-08 19:45:21 +00:00
self.status = 1
2019-01-04 10:02:28 +00:00
finally:
if self.maintenance and commit:
maintenance.off()
2018-11-08 19:45:21 +00:00
if commit:
if self.queue:
# if command was processed through the queue, update the existing
# command instance
command.description = self.description
command.status = "done"
command.result = self.result
command.save()
else:
# if command was processed in line with the http request it still
# needs to be persisted to the database
CommandLineTool.objects.create(
user=user,
tool=self.tool,
description=self.description,
status="done",
arguments=json.dumps({"args": self.args, "kwargs": self.kwargs}),
result=self.result,
)
2018-11-08 19:45:21 +00:00
return self.result
@transaction.atomic
2019-01-04 10:02:28 +00:00
def run(self, user, commit=False):
if self.queue and commit:
2019-12-05 16:57:52 +00:00
if (
CommandLineTool.objects.filter(tool=self.tool)
.exclude(status="done")
.count()
>= self.queue
):
self.result = "[error] {}".format(
_(
"This command is already waiting / running - please wait for it to finish before executing it again"
)
)
2019-01-04 10:02:28 +00:00
return self.result
self.cmd_instance = CommandLineTool.objects.create(
2019-12-05 16:57:52 +00:00
user=user,
tool=self.tool,
description=self.description,
status="waiting",
arguments=json.dumps({"args": self.args, "kwargs": self.kwargs}),
result="",
)
self.result = "[warn] {}".format(
_(
"This command takes a while to complete and will be queued and ran in the "
"background. No output log can be provided at this point in time. You may "
"review once the command has finished."
)
)
2019-01-04 10:02:28 +00:00
return self.result
else:
with reversion.create_revision():
return self._run(None, commit=commit, user=user)
2019-01-04 10:02:28 +00:00
def download_link(self):
return None
2018-11-08 19:45:21 +00:00
# TOOL: RENUMBER LAN
@register_tool
class ToolRenumberLans(CommandLineToolWrapper):
"""
This tools runs the pdb_renumber_lans command to
Renumber IP Spaces in an Exchange.
2018-11-08 19:45:21 +00:00
"""
tool = "pdb_renumber_lans"
class Form(forms.Form):
exchange = forms.ModelChoiceField(
queryset=InternetExchange.handleref.undeleted().order_by("name"),
2019-12-05 16:57:52 +00:00
widget=autocomplete.ModelSelect2(url="/autocomplete/ix/json"),
)
2018-11-08 19:45:21 +00:00
old_prefix = forms.CharField(
2019-12-05 16:57:52 +00:00
help_text=_(
"Old prefix - renumber all netixlans that fall into this prefix"
)
)
2018-11-08 19:45:21 +00:00
new_prefix = forms.CharField(
2019-12-05 16:57:52 +00:00
help_text=_(
"New prefix - needs to be the same protocol and length as old prefix"
)
)
2018-11-08 19:45:21 +00:00
@property
def description(self):
"""Provide a human readable description of the command that was run."""
try:
return "{}: {} to {}".format(
2019-12-05 16:57:52 +00:00
InternetExchange.objects.get(id=self.args[0]),
self.args[1],
self.args[2],
)
except Exception:
# if a version of this command was run before, we still need to able
# to display a somewhat useful discription, so fall back to this basic
# display
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
return f"(Legacy) {self.args}"
2018-11-08 19:45:21 +00:00
def set_arguments(self, form_data):
2019-12-05 16:57:52 +00:00
self.args = [
form_data.get("exchange", EmptyId()).id,
form_data.get("old_prefix"),
form_data.get("new_prefix"),
]
2018-11-08 19:45:21 +00:00
@register_tool
class ToolMergeFacilities(CommandLineToolWrapper):
"""
This tool runs the pdb_fac_merge command to
merge two facilities.
2018-11-08 19:45:21 +00:00
"""
tool = "pdb_fac_merge"
class Form(forms.Form):
other = forms.ModelChoiceField(
queryset=Facility.handleref.undeleted().order_by("name"),
widget=autocomplete.ModelSelect2(url="/autocomplete/fac/json"),
2019-12-05 16:57:52 +00:00
help_text=_("Merge this facility - it will be deleted"),
)
2018-11-08 19:45:21 +00:00
target = forms.ModelChoiceField(
queryset=Facility.handleref.undeleted().order_by("name"),
widget=autocomplete.ModelSelect2(url="/autocomplete/fac/json"),
2019-12-05 16:57:52 +00:00
help_text=_("Target facility"),
)
2018-11-08 19:45:21 +00:00
@property
def description(self):
"""Provide a human readable description of the command that was run."""
2018-11-08 19:45:21 +00:00
return "{} into {}".format(
Facility.objects.get(id=self.kwargs["ids"]),
2019-12-05 16:57:52 +00:00
Facility.objects.get(id=self.kwargs["target"]),
)
2018-11-08 19:45:21 +00:00
def set_arguments(self, form_data):
self.kwargs = {
"ids": str(form_data.get("other", EmptyId()).id),
2019-12-05 16:57:52 +00:00
"target": str(form_data.get("target", EmptyId()).id),
2018-11-08 19:45:21 +00:00
}
@register_tool
class ToolMergeFacilitiesUndo(CommandLineToolWrapper):
"""
This tool runs the pdb_fac_merge_undo command to
undo a facility merge.
2018-11-08 19:45:21 +00:00
"""
tool = "pdb_fac_merge_undo"
class Form(forms.Form):
merge = forms.ModelChoiceField(
2019-12-05 16:57:52 +00:00
queryset=CommandLineTool.objects.filter(tool="pdb_fac_merge").order_by(
"-created"
),
2018-11-08 19:45:21 +00:00
widget=autocomplete.ModelSelect2(
2019-12-05 16:57:52 +00:00
url="/autocomplete/admin/clt-history/pdb_fac_merge/"
),
help_text=_("Undo this merge"),
)
2018-11-08 19:45:21 +00:00
@property
def description(self):
"""Provide a human readable description of the command that was run."""
2018-11-08 19:45:21 +00:00
# in order to make a useful description we need to collect the arguments
# from the merge command that was undone
kwargs = json.loads(
2019-12-05 16:57:52 +00:00
CommandLineTool.objects.get(id=self.kwargs["clt"]).arguments
).get("kwargs")
2018-11-08 19:45:21 +00:00
return "Undo: {} into {}".format(
Facility.objects.get(id=kwargs["ids"]),
2019-12-05 16:57:52 +00:00
Facility.objects.get(id=kwargs["target"]),
)
2018-11-08 19:45:21 +00:00
def set_arguments(self, form_data):
self.kwargs = {"clt": form_data.get("merge", EmptyId()).id}
2019-01-04 10:02:28 +00:00
2019-12-05 16:57:52 +00:00
2019-01-04 10:02:28 +00:00
@register_tool
class ToolReset(CommandLineToolWrapper):
tool = "pdb_wipe"
queue = 1
maintenance = True
class Form(forms.Form):
2019-12-05 16:57:52 +00:00
keep_users = forms.BooleanField(
required=False,
help_text=_(
"Don't delete users. Note that superuser accounts are always kept - regardless of this setting."
),
)
load_data = forms.BooleanField(
required=False, initial=True, help_text=_("Load data from peeringdb API")
)
load_data_url = forms.CharField(
required=False, initial="https://www.peeringdb.com/api"
)
2019-01-04 10:02:28 +00:00
@property
def description(self):
return "Reset environment"
def set_arguments(self, form_data):
self.kwargs = form_data
2019-02-15 17:46:20 +00:00
@register_tool
class ToolUndelete(CommandLineToolWrapper):
"""
Allows restoration of an object object and it's child objects.
2019-02-15 17:46:20 +00:00
"""
2019-12-05 16:57:52 +00:00
2019-02-15 17:46:20 +00:00
tool = "pdb_undelete"
# These are the reftags that are currently supported by this
# tool.
2019-12-05 16:57:52 +00:00
supported_reftags = ["ixlan", "fac"]
2019-02-15 17:46:20 +00:00
class Form(forms.Form):
version = forms.ModelChoiceField(
queryset=Version.objects.all().order_by("-revision_id"),
2019-12-05 16:57:52 +00:00
widget=autocomplete.ModelSelect2(url="/autocomplete/admin/deletedversions"),
help_text=_("Restore this object - search by [reftag] [id]"),
)
2019-02-15 17:46:20 +00:00
@property
def description(self):
return "{reftag} {id}".format(**self.kwargs)
def set_arguments(self, form_data):
version = form_data.get("version")
if not version:
return
reftag = version.content_type.model_class().HandleRef.tag
2019-12-05 16:57:52 +00:00
self.kwargs = {
"reftag": reftag,
"id": version.object_id,
"version_id": version.id,
}
2019-02-15 17:46:20 +00:00
def validate(self):
if self.kwargs.get("reftag") not in self.supported_reftags:
2019-12-05 16:57:52 +00:00
raise ValueError(
_(
"Only {} type objects may be restored "
"through this interface at this point"
).format(",".join(self.supported_reftags))
)
obj = REFTAG_MAP[self.kwargs.get("reftag")].objects.get(
id=self.kwargs.get("id")
)
2019-02-15 17:46:20 +00:00
if obj.status != "deleted":
July updates (#762) * Change label from primary ASN to ASN * Raise validation error when trying to update ASN * first steps for dotf importer procotol (#697) * migrations (#697) * Add translation to error meessage * Make ASN readonly in table * Add test now that ASN should not be able to update * Set fac.rencode to '' for all entries and make it readonly in serializer * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * Add unique constraints to network ixlan ip addresses * Add migration to null out duplicate ipaddresses for deleted netixlans * remove old migrations (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf_member_data migrations (#697) * fix table name (#697) * importer protocol (#697) * fix netixlan ipaddr dedupe migration (#268) add netixlan ipaddr unique constraint migration (#268) * ixf proposed changes notifications (#697) * Delete repeated query * Add a test to show rencode is readonly * Blank out rencode when mocking data * Remove validator now that constraint exists * Add back unique field validator w Check Deleted true * conflict resolving (#697) * UniqueFieldValidator raise error with code "unique" (#268) * conflict resolution (#697) * Add fixme comment to tests * conflict resolution (#697) * Remove now invalid undelete tests * UniqueFieldValidator raise error with code "unique" (#268) * delete admin tools for duplicate ip addresses * Make migration to delete duplicateipnetworkixlan * Add ixlan-ixpfx status matching validation, add corresponding test * delete redundant checking in test * resolve conflict ui (#697) * fix migrations hierarchy * squash migrations for ixf member data * clean up preview and post-mortem tools * remove non-sensical permission check when undeleting soft-deleted objects through unique integrity error handling * only include the ix-f data url in notifications to admincom (#697) * resolve on --skip-import (#697) * ac conflict resolution (#697) * Define more accurately the incompatible statuses for ixlan and ixpfx * Add another status test * Preventing disrupting changes (#697) * fix tests (#697) * Stop allow_ixp_update from being write only and add a global stat for automated networks * Add tests for global stats that appear in footer * Change how timezone is called with datetime, to get test_stats.py/test_generate_for_current_date to pass * test for protected entities (#697) * admincom conflict resolution refine readonly fields (#697) network notifications only if the problem is actually actionable by the network (#697) * ixp / ac notifcation when ix-f source cannot be parsed (#697) fix issue with ixlan prefix protection (#697) * migrations (#697) * code documentation (#697) * ux tweaks (#697) * UX tweaks (#697) * Fix typo * fix netixlan returned in IXFMemberData.apply when adding a new one (#697) * fix import log incosistencies (#697) * Add IXFMemberData to test * Update test data * Add protocol tests * Add tests for views * always persist changes to remote data on set_conflict (#697) * More tests * always persist changes to remote data on set_conflict (#697) * suggest-add test * net_present_at_ix should check status (#697) * Add more protocol tests * Edit language of some tests * django-peeringdb to 2.1.1 relock pipfile, pin django-ratelimit to <3 as it breaks stuff * Add net_count_ixf field to ix object (#683) * Add the IX-F Member Export URL to the ixlan API endpoint (#249) * Lock some objects from being deleted by the owner (#696) * regenerate api docs (#249) * always persist changes to remote data on set_add and set_update (#697) * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * always persist changes to remote data on set_add and set_update (#697) * Fix suggest-add tests * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * IXFMemberData: always persist remote data changes during set_add and set_update, also allow for saving without touching the updated field * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * fix test_suggest_delete_local_ixf_no_flag (#697 tests) * fix issue with deletion when ixfmemberdata for entry existed previously (#697) * invalid ips get logged and notified to the ix via notify_error (#697) * Fix more tests * issue with previous_data when running without save (#697) properly track speed errors (#697) * reset errors on ixfmemberdata that go into pending_save (#697) * add remote_data to admin view (#697) * fix error reset inconsistency (#697) * Refine invalid data tests * remove debug output * for notifications to ac include contact points for net and ix in the message (#697) * settings to toggle ix-f tickets / emails (#697) * allow turning off ix-f notifications for net and ix separately (#697) * add jsonschema test * Add idempotent tests to updater * remove old ixf member tests * Invalid data tests when ixp_updates are enabled * fix speed error validation (#697) * fix issue with rollback (#697) * fix migration hierarchy * fix ixfmemberdata _email * django-peeringdb to 2.2 and relock * add ixf rollback tests * ixf email notifications off by default * black formatted * pyupgrade Co-authored-by: egfrank <egfrank@20c.com> Co-authored-by: Stefan Pratter <stefan@20c.com>
2020-07-15 02:07:01 -05:00
raise ValueError(f"{obj} is not currently marked as deleted")
Dotf fixes (#781) * fix issue where ix-f import would raise suggestions ipaddresses not that ixlan (#764) * IX-F Suggestions: Leaving the editor and returning to it via back button issues (#765) * IX-F importer: Clicking "Preview" (IXP Update Tools) on /net/ page resulted in 170 ticket resolutions (#769) More robust testing * black formatting (was lost after pyupgrade) * Add regex searching to deskpro ticket subjects * Change operational error * IX-F suggestions: consolidate delete+add (#770) * Add reset functions to commandline tool * Fix commandline tool bugs * Fix reset commandline tool bugs * add commandline tool * Ixlan needs to be set for import commandline tool * Add email model * Add admin view to emails * Allow network and ix to be null * save emails as part of ixf import * Add email model * Add email delete * add iregex search and better comments * fix ixlan selection for import * redefine migration dependencies for this branch * only enable search w start and end char * Add caption to regex search * Remove delete all ixfmemberdata option * [beta] IX-F importer: don't bother about missing IPv{4,6} address when network is not doing IPv{4,6} (#771) * Add cmdline tests * Resolve email conflicts * Add cmd tool reset tests * add autocomplete to commandline tool * Fix email bugs * Fix email migrations * Fix typos * [beta] IX-F importer: prevent Admin Committee overload by initially limiting importer to IXes enabled by AC (#772) * Finalize regex search for emails and deskprotickets * Fix keyword bug * fix typo * protocol-conflict will now be handled in the notification consolidation 771 changes where if the network indicates neither ipv4 nor ipv6 support, it is handled as supporting both (eg the network didnt configure these at all) realised that the importer command re instantiates the `Importer` class for each ixlan it processes, so moved the sending of consolidated notifications (#772) out of the `update` function and into the command itself after its done processing all the ixlans. This means for tests you will need to call `importer.notify_proposals` after `importer.update` to test the consolidated notifications. fixed several MultipleObjectsReturned errors when network switch protocol support in between imports * should be checking for "ix" in the form data (#773) * Fix cmd ixf tests * fix issue in log_peer * Add commit check for reset tool * fix importer bugs * remove dupe IXFImportEmail definition * ixfimportemail support ix__name and net__name searching * ticket resolution responses * Add commit to command ixf import changes * fix modify entry header * remove whitespace in notification about remote data changes * Begin updating tests * ixf-import command line tool to queue * refactor conflict inserts * Update import protocol tests, including tests for 770 * More test edits * Change cmd tests * better ixfmemberdata error handling and fix some test data * dont reset the same ixfmemberdata requirement * fix many bugs add many tests * remove debug message * fix bug during import when consolidating delete+add * fix perfomance issue in IXFMemberData listing * dont show reset flags on prod env * Add regex search tests * Add 772 tests * remove debug output * fix `test_resolve_deskpro_ticket` test * black formatting * remove dupe import * fix issue with unique constraint error handling * add test for ixp / network ip protocol notification * add missing test data Co-authored-by: Stefan Pratter <stefan@20c.com> Co-authored-by: Elliot Frank <elliot@20c.com>
2020-07-26 23:36:27 -05:00
@register_tool
class ToolIXFIXPMemberImport(CommandLineToolWrapper):
"""
Allows resets for various parts of the ix-f member data import protocol.
And import ix-f member data for a single Ixlan at a time.
"""
tool = "pdb_ixf_ixp_member_import"
queue = 1
class Form(forms.Form):
ix = forms.ModelChoiceField(
queryset=InternetExchange.objects.all(),
widget=autocomplete.ModelSelect2(url="/autocomplete/ix/json"),
help_text=_(
"Select an Internet Exchange to perform an ix-f memberdata import"
),
)
if settings.RELEASE_ENV != "prod":
# reset toggles are not available on production
# environment
reset = forms.BooleanField(
required=False, initial=False, help_text=_("Reset all")
)
reset_hints = forms.BooleanField(
required=False, initial=False, help_text=_("Reset hints")
)
reset_dismisses = forms.BooleanField(
required=False, initial=False, help_text=_("Reset dismisses")
)
reset_email = forms.BooleanField(
required=False, initial=False, help_text=_("Reset email")
)
reset_tickets = forms.BooleanField(
required=False, initial=False, help_text=_("Reset tickets")
)
@property
def description(self):
return "IX-F Member Import Tool"
def set_arguments(self, form_data):
for key in [
"reset",
"reset_hints",
"reset_dismisses",
"reset_email",
"reset_tickets",
]:
self.kwargs[key] = form_data.get(key, False)
if form_data.get("ix"):
self.kwargs["ixlan"] = [form_data.get("ix").id]
@register_tool
class ToolValidateData(CommandLineToolWrapper):
"""
Validate data in the database.
"""
tool = "pdb_validate_data"
queue = 10
class Form(forms.Form):
models = []
for tag, model in REFTAG_MAP.items():
models.append((tag, model._meta.verbose_name.title()))
handleref_tag = forms.ChoiceField(
choices=models,
help_text=_("Select a handleref tag to validate"),
label=_("Object type"),
)
field_name = forms.CharField(
required=True,
help_text=_("Enter a field name to validate"),
)
exclude = forms.ChoiceField(
choices=(
(None, "--"),
("valid", _("Valid")),
("invalid", _("Invalid")),
),
required=False,
help_text=_("Exclude data based on validation result"),
)
verbose = forms.BooleanField(
required=False, initial=False, help_text=_("Verbose output")
)
@property
def description(self):
return f"Validate data: {self.args[0]}.{self.args[1]}"
def set_arguments(self, form_data):
self.args = [
form_data.get("handleref_tag"),
form_data.get("field_name"),
]
self.kwargs = {
"exclude": form_data.get("exclude", None),
"verbose": form_data.get("verbose", False),
}
self.form_data = form_data
def download_link(self):
handleref_tag = self.args[0]
field_name = self.args[1]
return (
f"/pdb_validate_data/export/pdb_validate_data_{handleref_tag}_{field_name}.csv",
"Download validation results (CSV) (This file has an expiry date!)",
)