From a25a27f31f4d9971db5121d8a4270afb2e665e8a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 9 Aug 2019 12:33:33 -0400 Subject: [PATCH 01/63] Initial work on custom scripts (#3415) --- .gitignore | 2 + netbox/extras/forms.py | 15 ++ netbox/extras/scripts.py | 143 ++++++++++++++++++ netbox/extras/templatetags/log_levels.py | 37 +++++ netbox/extras/urls.py | 10 +- netbox/extras/views.py | 54 ++++++- netbox/netbox/settings.py | 1 + netbox/scripts/__init__.py | 0 netbox/templates/extras/script.html | 77 ++++++++++ netbox/templates/extras/script_list.html | 40 +++++ .../extras/templatetags/log_level.html | 1 + 11 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 netbox/extras/scripts.py create mode 100644 netbox/extras/templatetags/log_levels.py create mode 100644 netbox/scripts/__init__.py create mode 100644 netbox/templates/extras/script.html create mode 100644 netbox/templates/extras/script_list.html create mode 100644 netbox/templates/extras/templatetags/log_level.html diff --git a/.gitignore b/.gitignore index d859bad28..36c6d3fa8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /netbox/netbox/ldap_config.py /netbox/reports/* !/netbox/reports/__init__.py +/netbox/scripts/* +!/netbox/scripts/__init__.py /netbox/static .idea /*.sh diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 261822d28..fad5a7ac2 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -380,3 +380,18 @@ class ObjectChangeFilterForm(BootstrapMixin, forms.Form): widget=ContentTypeSelect(), label='Object Type' ) + + +# +# Scripts +# + +class ScriptForm(BootstrapMixin, forms.Form): + + def __init__(self, vars, *args, **kwargs): + + super().__init__(*args, **kwargs) + + # Dynamically populate fields for variables + for name, var in vars: + self.fields[name] = var.as_field() diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py new file mode 100644 index 000000000..7ba7edbf0 --- /dev/null +++ b/netbox/extras/scripts.py @@ -0,0 +1,143 @@ +from collections import OrderedDict +import inspect +import pkgutil + +from django import forms +from django.conf import settings + +from .constants import LOG_DEFAULT, LOG_FAILURE, LOG_INFO, LOG_SUCCESS, LOG_WARNING +from .forms import ScriptForm + + +# +# Script variables +# + +class ScriptVariable: + form_field = forms.CharField + + def __init__(self, label='', description=''): + + # Default field attributes + if not hasattr(self, 'field_attrs'): + self.field_attrs = {} + if label: + self.field_attrs['label'] = label + if description: + self.field_attrs['help_text'] = description + + def as_field(self): + """ + Render the variable as a Django form field. + """ + return self.form_field(**self.field_attrs) + + +class StringVar(ScriptVariable): + pass + + +class IntegerVar(ScriptVariable): + form_field = forms.IntegerField + + +class BooleanVar(ScriptVariable): + form_field = forms.BooleanField + field_attrs = { + 'required': False + } + + +class ObjectVar(ScriptVariable): + form_field = forms.ModelChoiceField + + def __init__(self, queryset, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.field_attrs['queryset'] = queryset + + +class Script: + """ + Custom scripts inherit this object. + """ + + def __init__(self): + + # Initiate the log + self.log = [] + + # Grab some info about the script + self.filename = inspect.getfile(self.__class__) + self.source = inspect.getsource(self.__class__) + + def __str__(self): + if hasattr(self, 'name'): + return self.name + return self.__class__.__name__ + + def _get_vars(self): + # TODO: This should preserve var ordering + return inspect.getmembers(self, is_variable) + + def run(self, context): + raise NotImplementedError("The script must define a run() method.") + + def as_form(self, data=None): + """ + Return a Django form suitable for populating the context data required to run this Script. + """ + vars = self._get_vars() + form = ScriptForm(vars, data) + + return form + + # Logging + + def log_debug(self, message): + self.log.append((LOG_DEFAULT, message)) + + def log_success(self, message): + self.log.append((LOG_SUCCESS, message)) + + def log_info(self, message): + self.log.append((LOG_INFO, message)) + + def log_warning(self, message): + self.log.append((LOG_WARNING, message)) + + def log_failure(self, message): + self.log.append((LOG_FAILURE, message)) + + +# +# Functions +# + +def is_script(obj): + """ + Returns True if the object is a Script. + """ + return obj in Script.__subclasses__() + + +def is_variable(obj): + """ + Returns True if the object is a ScriptVariable. + """ + return isinstance(obj, ScriptVariable) + + +def get_scripts(): + scripts = OrderedDict() + + # Iterate through all modules within the reports path. These are the user-created files in which reports are + # defined. + for importer, module_name, _ in pkgutil.iter_modules([settings.SCRIPTS_ROOT]): + module = importer.find_module(module_name).load_module(module_name) + module_scripts = OrderedDict() + for name, cls in inspect.getmembers(module, is_script): + module_scripts[name] = cls + scripts[module_name] = module_scripts + + return scripts diff --git a/netbox/extras/templatetags/log_levels.py b/netbox/extras/templatetags/log_levels.py new file mode 100644 index 000000000..f1a545cb9 --- /dev/null +++ b/netbox/extras/templatetags/log_levels.py @@ -0,0 +1,37 @@ +from django import template + +from extras.constants import LOG_DEFAULT, LOG_FAILURE, LOG_INFO, LOG_SUCCESS, LOG_WARNING + + +register = template.Library() + + +@register.inclusion_tag('extras/templatetags/log_level.html') +def log_level(level): + """ + Display a label indicating a syslog severity (e.g. info, warning, etc.). + """ + levels = { + LOG_DEFAULT: { + 'name': 'Default', + 'class': 'default' + }, + LOG_SUCCESS: { + 'name': 'Success', + 'class': 'success', + }, + LOG_INFO: { + 'name': 'Info', + 'class': 'info' + }, + LOG_WARNING: { + 'name': 'Warning', + 'class': 'warning' + }, + LOG_FAILURE: { + 'name': 'Failure', + 'class': 'danger' + } + } + + return levels[level] diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index ad6eabe1e..7de0faf91 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -28,13 +28,17 @@ urlpatterns = [ path(r'image-attachments//edit/', views.ImageAttachmentEditView.as_view(), name='imageattachment_edit'), path(r'image-attachments//delete/', views.ImageAttachmentDeleteView.as_view(), name='imageattachment_delete'), + # Change logging + path(r'changelog/', views.ObjectChangeListView.as_view(), name='objectchange_list'), + path(r'changelog//', views.ObjectChangeView.as_view(), name='objectchange'), + # Reports path(r'reports/', views.ReportListView.as_view(), name='report_list'), path(r'reports//', views.ReportView.as_view(), name='report'), path(r'reports//run/', views.ReportRunView.as_view(), name='report_run'), - # Change logging - path(r'changelog/', views.ObjectChangeListView.as_view(), name='objectchange_list'), - path(r'changelog//', views.ObjectChangeView.as_view(), name='objectchange'), + # Scripts + path(r'scripts/', views.ScriptListView.as_view(), name='script_list'), + path(r'scripts///', views.ScriptView.as_view(), name='script'), ] diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 6f4751619..8f9f2d282 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -1,8 +1,9 @@ from django import template from django.conf import settings from django.contrib import messages -from django.contrib.auth.mixins import PermissionRequiredMixin +from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin from django.contrib.contenttypes.models import ContentType +from django.db import transaction from django.db.models import Count, Q from django.http import Http404 from django.shortcuts import get_object_or_404, redirect, render @@ -20,6 +21,7 @@ from .forms import ( ) from .models import ConfigContext, ImageAttachment, ObjectChange, ReportResult, Tag, TaggedItem from .reports import get_report, get_reports +from .scripts import get_scripts from .tables import ConfigContextTable, ObjectChangeTable, TagTable, TaggedItemTable @@ -355,3 +357,53 @@ class ReportRunView(PermissionRequiredMixin, View): messages.success(request, mark_safe(msg)) return redirect('extras:report', name=report.full_name) + + +# +# Scripts +# + +class ScriptListView(LoginRequiredMixin, View): + + def get(self, request): + + return render(request, 'extras/script_list.html', { + 'scripts': get_scripts(), + }) + + +class ScriptView(LoginRequiredMixin, View): + + def _get_script(self, module, name): + scripts = get_scripts() + try: + return scripts[module][name]() + except KeyError: + raise Http404 + + def get(self, request, module, name): + + script = self._get_script(module, name) + form = script.as_form() + + return render(request, 'extras/script.html', { + 'module': module, + 'script': script, + 'form': form, + }) + + def post(self, request, module, name): + + script = self._get_script(module, name) + form = script.as_form(request.POST) + + if form.is_valid(): + + with transaction.atomic(): + script.run(form.cleaned_data) + + return render(request, 'extras/script.html', { + 'module': module, + 'script': script, + 'form': form, + }) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 090122e37..014b623cd 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -85,6 +85,7 @@ NAPALM_USERNAME = getattr(configuration, 'NAPALM_USERNAME', '') PAGINATE_COUNT = getattr(configuration, 'PAGINATE_COUNT', 50) PREFER_IPV4 = getattr(configuration, 'PREFER_IPV4', False) REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') +SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/') SESSION_FILE_PATH = getattr(configuration, 'SESSION_FILE_PATH', None) SHORT_DATE_FORMAT = getattr(configuration, 'SHORT_DATE_FORMAT', 'Y-m-d') SHORT_DATETIME_FORMAT = getattr(configuration, 'SHORT_DATETIME_FORMAT', 'Y-m-d H:i') diff --git a/netbox/scripts/__init__.py b/netbox/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html new file mode 100644 index 000000000..8b4065613 --- /dev/null +++ b/netbox/templates/extras/script.html @@ -0,0 +1,77 @@ +{% extends '_base.html' %} +{% load helpers %} +{% load form_helpers %} +{% load log_levels %} + +{% block title %}{{ script }}{% endblock %} + +{% block content %} +
+
+
    +
  1. Scripts
    2. +
  2. {{ module|bettertitle }}
    3. +
  3. {{ script }}
    4. +
+
+
+

{{ script }}

+

{{ script.description }}

+ +
+
+ {% if script.log %} +
+
+
+
+ Script Output +
+ + + + + + + {% for level, message in script.log %} + + + + + + {% endfor %} +
LineLevelMessage
{{ forloop.counter }}{% log_level level %}{{ message }}
+
+
+
+ {% endif %} +
+
+
+ {% csrf_token %} + {% if form %} + {% render_form form %} + {% else %} +

This script does not require any input to run.

+ {% endif %} +
+ + Cancel +
+
+
+
+
+
+ {{ script.filename }} + 
{{ script.source }}
+
+
+{% endblock %} diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html new file mode 100644 index 000000000..0189ef755 --- /dev/null +++ b/netbox/templates/extras/script_list.html @@ -0,0 +1,40 @@ +{% extends '_base.html' %} +{% load helpers %} + +{% block content %} +

{% block title %}Scripts{% endblock %}

+
+
+ {% if scripts %} + {% for module, module_scripts in scripts.items %} +

{{ module|bettertitle }}

+ + + + + + + + + + {% for class_name, script in module_scripts.items %} + + + + + + {% endfor %} + +
NameDescription
+ {{ script }} + {{ script.description }}
+ {% endfor %} + {% else %} +
+

No scripts found.

+

Reports should be saved to {{ settings.SCRIPTS_ROOT }}. (This path can be changed by setting SCRIPTS_ROOT in NetBox's configuration.)

+
+ {% endif %} +
+
+{% endblock %} diff --git a/netbox/templates/extras/templatetags/log_level.html b/netbox/templates/extras/templatetags/log_level.html new file mode 100644 index 000000000..0787c2d46 --- /dev/null +++ b/netbox/templates/extras/templatetags/log_level.html @@ -0,0 +1 @@ + \ No newline at end of file From 9d054fb345b360257124a9fd3dadfb01b6969257 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 9 Aug 2019 13:56:37 -0400 Subject: [PATCH 02/63] Add options for script vars; include script output --- netbox/extras/scripts.py | 69 ++++++++++++++++++++++++----- netbox/extras/views.py | 4 +- netbox/templates/extras/script.html | 6 +++ 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 7ba7edbf0..a4900b9a2 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -4,27 +4,37 @@ import pkgutil from django import forms from django.conf import settings +from django.core.validators import RegexValidator from .constants import LOG_DEFAULT, LOG_FAILURE, LOG_INFO, LOG_SUCCESS, LOG_WARNING from .forms import ScriptForm +class OptionalBooleanField(forms.BooleanField): + required = False + + # # Script variables # class ScriptVariable: + """ + Base model for script variables + """ form_field = forms.CharField - def __init__(self, label='', description=''): + def __init__(self, label='', description='', default=None, required=True): # Default field attributes - if not hasattr(self, 'field_attrs'): - self.field_attrs = {} + self.field_attrs = { + 'help_text': description, + 'required': required + } if label: self.field_attrs['label'] = label - if description: - self.field_attrs['help_text'] = description + if default: + self.field_attrs['initial'] = default def as_field(self): """ @@ -34,26 +44,62 @@ class ScriptVariable: class StringVar(ScriptVariable): - pass + """ + Character string representation. Can enforce minimum/maximum length and/or regex validation. + """ + def __init__(self, min_length=None, max_length=None, regex=None, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Optional minimum/maximum lengths + if min_length: + self.field_attrs['min_length'] = min_length + if max_length: + self.field_attrs['max_length'] = max_length + + # Optional regular expression validation + if regex: + self.field_attrs['validators'] = [ + RegexValidator( + regex=regex, + message='Invalid value. Must match regex: {}'.format(regex), + code='invalid' + ) + ] class IntegerVar(ScriptVariable): + """ + Integer representation. Can enforce minimum/maximum values. + """ form_field = forms.IntegerField + def __init__(self, min_value=None, max_value=None, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Optional minimum/maximum values + if min_value: + self.field_attrs['min_value'] = min_value + if max_value: + self.field_attrs['max_value'] = max_value + class BooleanVar(ScriptVariable): - form_field = forms.BooleanField - field_attrs = { - 'required': False - } + """ + Boolean representation (true/false). Renders as a checkbox. + """ + form_field = OptionalBooleanField class ObjectVar(ScriptVariable): + """ + NetBox object representation. The provided QuerySet will determine the choices available. + """ form_field = forms.ModelChoiceField def __init__(self, queryset, *args, **kwargs): super().__init__(*args, **kwargs) + # Queryset for field choices self.field_attrs['queryset'] = queryset @@ -61,7 +107,6 @@ class Script: """ Custom scripts inherit this object. """ - def __init__(self): # Initiate the log @@ -80,7 +125,7 @@ class Script: # TODO: This should preserve var ordering return inspect.getmembers(self, is_variable) - def run(self, context): + def run(self, data): raise NotImplementedError("The script must define a run() method.") def as_form(self, data=None): diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 8f9f2d282..21aed1471 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -396,14 +396,16 @@ class ScriptView(LoginRequiredMixin, View): script = self._get_script(module, name) form = script.as_form(request.POST) + output = None if form.is_valid(): with transaction.atomic(): - script.run(form.cleaned_data) + output = script.run(form.cleaned_data) return render(request, 'extras/script.html', { 'module': module, 'script': script, 'form': form, + 'output': output, }) diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index 8b4065613..bbd949098 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -21,6 +21,9 @@ +
  • Source
    • @@ -69,6 +72,9 @@ +
    + 
    {{ output }}
    +
    {{ script.filename }} 
    {{ script.source }}
    From 4fc19742ec74b8e45a0beae65c64b9ee2ede2729 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 9 Aug 2019 15:00:06 -0400 Subject: [PATCH 03/63] Added documentation for custom scripts --- docs/additional-features/custom-scripts.md | 149 +++++++++++++++++++++ docs/configuration/optional-settings.md | 8 ++ 2 files changed, 157 insertions(+) create mode 100644 docs/additional-features/custom-scripts.md diff --git a/docs/additional-features/custom-scripts.md b/docs/additional-features/custom-scripts.md new file mode 100644 index 000000000..c97137a41 --- /dev/null +++ b/docs/additional-features/custom-scripts.md @@ -0,0 +1,149 @@ +# Custom Scripts + +Custom scripting was introduced in NetBox v2.7 to provide a way for users to execute custom logic from within the NetBox UI. Custom scripts enable the user to directly and conveniently manipulate NetBox data in a prescribed fashion. They can be used to accomplish myriad tasks, such as: + +* Automatically populate new devices and cables in preparation for a new site deployment +* Create a range of new reserved prefixes or IP addresses +* Fetch data from an external source and import it to NetBox + +Custom scripts are Python code and exist outside of the official NetBox code base, so they can be updated and changed without interfering with the core NetBox installation. And because they're written from scratch, a custom script can be used to accomplish just about anything. + +## Writing Custom Scripts + +All custom scripts must inherit from the `extras.scripts.Script` base class. This class provides the functionality necessary to generate forms and log activity. + +``` +from extras.scripts import Script + +class MyScript(Script): + .. +``` + +Scripts comprise two core components: variables and a `run()` method. Variables allow your script to accept user input via the NetBox UI. The `run()` method is where your script's execution logic lives. (Note that your script can have as many methods as needed: this is merely the point of invocation for NetBox.) + +``` +class MyScript(Script): + var1 = StringVar(...) + var2 = IntegerVar(...) + var3 = ObjectVar(...) + + def run(self, data): + ... +``` + +The `run()` method is passed a single argument: a dictionary containing all of the variable data passed via the web form. Your script can reference this data during execution. + +Defining variables is optional: You may create a script with only a `run()` method if no user input is needed. + +Returning output from your script is optional. Any raw output generated by the script will be displayed under the "output" tab in the UI. + +## Logging + +The Script object provides a set of convenient functions for recording messages at different severity levels: + +* `log_debug` +* `log_success` +* `log_info` +* `log_warning` +* `log_failure` + +Log messages are returned to the user upon execution of the script. + +## Variable Reference + +### StringVar + +Stores a string of characters (i.e. a line of text). Options include: + +* `min_length` - Minimum number of characters +* `max_length` - Maximum number of characters +* `regex` - A regular expression against which the provided value must match + +Note: `min_length` and `max_length` can be set to the same number to effect a fixed-length field. + +### IntegerVar + +Stored a numeric integer. Options include: + +* `min_value:` - Minimum value +* `max_value` - Maximum value + +### BooleanVar + +A true/false flag. This field has no options beyond the defaults. + +### ObjectVar + +A NetBox object. The list of available objects is defined by the queryset parameter. Each instance of this variable is limited to a single object type. + +* `queryset` - A [Django queryset](https://docs.djangoproject.com/en/stable/topics/db/queries/) + +### Default Options + +All variables support the following default options: + +* `label` - The name of the form field +* `description` - A brief description of the field +* `default` - The field's default value +* `required` - Indicates whether the field is mandatory (default: true) + +## Example + +Below is an example script that creates new objects for a planned site. The user is prompted for three variables: + +* The name of the new site +* The device model (a filtered list of defined device types) +* The number of access switches to create + +These variables are presented as a web form to be completed by the user. Once submitted, the script's `run()` method is called to create the appropriate objects. + +``` +from django.utils.text import slugify + +from dcim.constants import * +from dcim.models import Device, DeviceRole, DeviceType, Site +from extras.scripts import Script, IntegerVar, ObjectVar, StringVar + + +class NewBranchScript(Script): + name = "New Branch" + description = "Provision a new branch site" + + site_name = StringVar( + description="Name of the new site" + ) + switch_count = IntegerVar( + description="Number of access switches to create" + ) + switch_model = ObjectVar( + description="Access switch model", + queryset = DeviceType.objects.filter( + manufacturer__name='Cisco', + model__in=['Catalyst 3560X-48T', 'Catalyst 3750X-48T'] + ) + ) + + def run(self, data): + + # Create the new site + site = Site( + name=data['site_name'], + slug=slugify(data['site_name']), + status=SITE_STATUS_PLANNED + ) + site.save() + self.log_success("Created new site: {}".format(site)) + + # Create access switches + switch_role = DeviceRole.objects.get(name='Access Switch') + for i in range(1, data['switch_count'] + 1): + switch = Device( + device_type=data['switch_model'], + name='{}-switch{}'.format(site.slug, i), + site=site, + status=DEVICE_STATUS_PLANNED, + device_role=switch_role + ) + switch.save() + self.log_success("Created new switch: {}".format(switch)) +``` diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index 4ebb56290..b532c9757 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -277,6 +277,14 @@ The file path to the location where custom reports will be kept. By default, thi --- +## SCRIPTS_ROOT + +Default: $BASE_DIR/netbox/scripts/ + +The file path to the location where custom scripts will be kept. By default, this is the `netbox/scripts/` directory within the base NetBox installation path. + +--- + ## SESSION_FILE_PATH Default: None From 3f7f3f88f3ab8409e7b6c290047a718ef7ed5995 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 9 Aug 2019 16:34:01 -0400 Subject: [PATCH 04/63] Fix form field ordering --- docs/additional-features/custom-scripts.md | 23 ++++++++++++++++-- netbox/extras/forms.py | 2 +- netbox/extras/scripts.py | 28 ++++++++++++++++++---- netbox/templates/extras/script.html | 2 +- netbox/templates/extras/script_list.html | 2 +- 5 files changed, 48 insertions(+), 9 deletions(-) diff --git a/docs/additional-features/custom-scripts.md b/docs/additional-features/custom-scripts.md index c97137a41..b4e5852e0 100644 --- a/docs/additional-features/custom-scripts.md +++ b/docs/additional-features/custom-scripts.md @@ -37,6 +37,24 @@ Defining variables is optional: You may create a script with only a `run()` meth Returning output from your script is optional. Any raw output generated by the script will be displayed under the "output" tab in the UI. +## Script Attributes + +### script_name + +This is the human-friendly names of your script. If omitted, the class name will be used. + +### script_description + +A human-friendly description of what your script does (optional). + +### script_fields + +The order in which the variable fields should appear. This is optional, however on Python 3.5 and earlier the fields will appear in random order. (Declarative ordering is preserved on Python 3.6 and above.) For example: + +``` +script_fields = ['var1', 'var2', 'var3'] +``` + ## Logging The Script object provides a set of convenient functions for recording messages at different severity levels: @@ -106,8 +124,9 @@ from extras.scripts import Script, IntegerVar, ObjectVar, StringVar class NewBranchScript(Script): - name = "New Branch" - description = "Provision a new branch site" + script_name = "New Branch" + script_description = "Provision a new branch site" + script_fields = ['site_name', 'switch_count', 'switch_model'] site_name = StringVar( description="Name of the new site" diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index fad5a7ac2..15c91a880 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -393,5 +393,5 @@ class ScriptForm(BootstrapMixin, forms.Form): super().__init__(*args, **kwargs) # Dynamically populate fields for variables - for name, var in vars: + for name, var in vars.items(): self.fields[name] = var.as_field() diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index a4900b9a2..7ef3dde2f 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -10,6 +10,15 @@ from .constants import LOG_DEFAULT, LOG_FAILURE, LOG_INFO, LOG_SUCCESS, LOG_WARN from .forms import ScriptForm +__all__ = [ + 'Script', + 'StringVar', + 'IntegerVar', + 'BooleanVar', + 'ObjectVar', +] + + class OptionalBooleanField(forms.BooleanField): required = False @@ -117,13 +126,24 @@ class Script: self.source = inspect.getsource(self.__class__) def __str__(self): - if hasattr(self, 'name'): - return self.name + if hasattr(self, 'script_name'): + return self.script_name return self.__class__.__name__ def _get_vars(self): - # TODO: This should preserve var ordering - return inspect.getmembers(self, is_variable) + vars = OrderedDict() + + # Infer order from script_fields (Python 3.5 and lower) + if hasattr(self, 'script_fields'): + for name in self.script_fields: + vars[name] = getattr(self, name) + + # Default to order of declaration on class + for name, attr in self.__class__.__dict__.items(): + if name not in vars and issubclass(attr.__class__, ScriptVariable): + vars[name] = attr + + return vars def run(self, data): raise NotImplementedError("The script must define a run() method.") diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index bbd949098..66beeb852 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -16,7 +16,7 @@

    {{ script }}

    -

    {{ script.description }}

    +

    {{ script.script_description }}

      + + Scripts + Reports From 116c395948c5034dc069cf233a2ad319e623757c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 12 Aug 2019 11:57:48 -0400 Subject: [PATCH 09/63] Fixes #3422: Prevent navigation menu from overlapping page content --- CHANGELOG.md | 1 + netbox/project-static/css/base.css | 8 ++-- netbox/scripts/examples.py | 66 ++++++++++++++++++++++++++++++ netbox/scripts/myscripts.py | 54 ++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 netbox/scripts/examples.py create mode 100644 netbox/scripts/myscripts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf8cd930..6e049b645 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ v2.6.3 (FUTURE) ## Bug Fixes * [#3405](https://github.com/netbox-community/netbox/issues/3405) - Fix population of power port/outlet details on device creation +* [#3422](https://github.com/netbox-community/netbox/issues/3422) - Prevent navigation menu from overlapping page content --- diff --git a/netbox/project-static/css/base.css b/netbox/project-static/css/base.css index fcee05e12..d625b9d68 100644 --- a/netbox/project-static/css/base.css +++ b/netbox/project-static/css/base.css @@ -42,8 +42,8 @@ footer p { } } -/* Hide the search bar in the navigation menu on displays less than 1200px wide */ -@media (max-width: 1199px) { +/* Hide the search bar in the navigation menu on displays less than 1250px wide */ +@media (max-width: 1249px) { #navbar_search { display: none; } @@ -62,8 +62,8 @@ footer p { } } -/* Collapse the nav menu on displays less than 960px wide */ -@media (max-width: 959px) { +/* Collapse the nav menu on displays less than 980px wide */ +@media (max-width: 979px) { .navbar-header { float: none; } diff --git a/netbox/scripts/examples.py b/netbox/scripts/examples.py new file mode 100644 index 000000000..b2adf8da4 --- /dev/null +++ b/netbox/scripts/examples.py @@ -0,0 +1,66 @@ +from django.utils.text import slugify + +from dcim.constants import * +from dcim.models import Device, DeviceRole, DeviceType, Site +from extras.scripts import * + + +class NewBranchScript(Script): + script_name = "New Branch" + script_description = "Provision a new branch site" + script_fields = ['site_name', 'switch_count', 'switch_model'] + + site_name = StringVar( + description="Name of the new site" + ) + switch_count = IntegerVar( + description="Number of access switches to create" + ) + switch_model = ObjectVar( + description="Access switch model", + queryset=DeviceType.objects.filter( + manufacturer__name='Cisco', + model__in=['Catalyst 3560X-48T', 'Catalyst 3750X-48T'] + ) + ) + x = BooleanVar( + description="Check me out" + ) + + def run(self, data): + + # Create the new site + site = Site( + name=data['site_name'], + slug=slugify(data['site_name']), + status=SITE_STATUS_PLANNED + ) + site.save() + self.log_success("Created new site: {}".format(site)) + + # Create access switches + switch_role = DeviceRole.objects.get(name='Access Switch') + for i in range(1, data['switch_count'] + 1): + switch = Device( + device_type=data['switch_model'], + name='{}-switch{}'.format(site.slug, i), + site=site, + status=DEVICE_STATUS_PLANNED, + device_role=switch_role + ) + switch.save() + self.log_success("Created new switch: {}".format(switch)) + + # Generate a CSV table of new devices + output = [ + 'name,make,model' + ] + for switch in Device.objects.filter(site=site): + attrs = [ + switch.name, + switch.device_type.manufacturer.name, + switch.device_type.model + ] + output.append(','.join(attrs)) + + return '\n'.join(output) diff --git a/netbox/scripts/myscripts.py b/netbox/scripts/myscripts.py new file mode 100644 index 000000000..f3542c368 --- /dev/null +++ b/netbox/scripts/myscripts.py @@ -0,0 +1,54 @@ +from dcim.models import Site +from extras.scripts import Script, BooleanVar, IntegerVar, ObjectVar, StringVar + + +class NoInputScript(Script): + description = "This script does not require any input" + + def run(self, data): + + self.log_debug("This a debug message.") + self.log_info("This an info message.") + self.log_success("This a success message.") + self.log_warning("This a warning message.") + self.log_failure("This a failure message.") + + +class DemoScript(Script): + name = "Script Demo" + description = "A quick demonstration of the available field types" + + my_string1 = StringVar( + description="Input a string between 3 and 10 characters", + min_length=3, + max_length=10 + ) + my_string2 = StringVar( + description="This field enforces a regex: three letters followed by three numbers", + regex=r'[a-z]{3}\d{3}' + ) + my_number = IntegerVar( + description="Pick a number between 1 and 255 (inclusive)", + min_value=1, + max_value=255 + ) + my_boolean = BooleanVar( + description="Use the checkbox to toggle true/false" + ) + my_object = ObjectVar( + description="Select a NetBox site", + queryset=Site.objects.all() + ) + + def run(self, data): + + self.log_info("Your string was {}".format(data['my_string1'])) + self.log_info("Your second string was {}".format(data['my_string2'])) + self.log_info("Your number was {}".format(data['my_number'])) + if data['my_boolean']: + self.log_info("You ticked the checkbox") + else: + self.log_info("You did not tick the checkbox") + self.log_info("You chose the sites {}".format(data['my_object'])) + + return "Here's some output" From dd4dafa7befcf0cc40a98d66b95a23f36c5b8d31 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 12 Aug 2019 12:10:36 -0400 Subject: [PATCH 10/63] Closes #3420: Serial number filter for racks, devices, and inventory items is now case-insensitive --- CHANGELOG.md | 4 ++++ netbox/dcim/filters.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e049b645..ac8b460c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ v2.6.3 (FUTURE) ## Bug Fixes +* [#3420](https://github.com/netbox-community/netbox/issues/3420) - Serial number filter for racks, devices, and inventory items is now case-insensitive + +## Enhancements + * [#3405](https://github.com/netbox-community/netbox/issues/3405) - Fix population of power port/outlet details on device creation * [#3422](https://github.com/netbox-community/netbox/issues/3422) - Prevent navigation menu from overlapping page content diff --git a/netbox/dcim/filters.py b/netbox/dcim/filters.py index a063a6b83..be1b02703 100644 --- a/netbox/dcim/filters.py +++ b/netbox/dcim/filters.py @@ -160,12 +160,15 @@ class RackFilter(TenancyFilterSet, CustomFieldFilterSet): to_field_name='slug', label='Role (slug)', ) + serial = django_filters.CharFilter( + lookup_expr='iexact' + ) tag = TagFilter() class Meta: model = Rack fields = [ - 'id', 'name', 'facility_id', 'serial', 'asset_tag', 'type', 'width', 'u_height', 'desc_units', + 'id', 'name', 'facility_id', 'asset_tag', 'type', 'width', 'u_height', 'desc_units', 'outer_width', 'outer_depth', 'outer_unit', ] @@ -519,6 +522,9 @@ class DeviceFilter(TenancyFilterSet, CustomFieldFilterSet): field_name='interfaces__mac_address', label='MAC address', ) + serial = django_filters.CharFilter( + lookup_expr='iexact' + ) has_primary_ip = django_filters.BooleanFilter( method='_has_primary_ip', label='Has a primary IP', @@ -560,7 +566,7 @@ class DeviceFilter(TenancyFilterSet, CustomFieldFilterSet): class Meta: model = Device - fields = ['id', 'name', 'serial', 'asset_tag', 'face', 'position', 'vc_position', 'vc_priority'] + fields = ['id', 'name', 'asset_tag', 'face', 'position', 'vc_position', 'vc_priority'] def search(self, queryset, name, value): if not value.strip(): @@ -847,10 +853,13 @@ class InventoryItemFilter(DeviceComponentFilterSet): to_field_name='slug', label='Manufacturer (slug)', ) + serial = django_filters.CharFilter( + lookup_expr='iexact' + ) class Meta: model = InventoryItem - fields = ['id', 'name', 'part_id', 'serial', 'asset_tag', 'discovered'] + fields = ['id', 'name', 'part_id', 'asset_tag', 'discovered'] def search(self, queryset, name, value): if not value.strip(): From 19eb4c510c4737110e9944ddbc3d842ddb5ae70c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 12 Aug 2019 13:16:18 -0400 Subject: [PATCH 11/63] Move script attributes under a Meta class --- docs/additional-features/custom-scripts.md | 20 ++++++++++++-------- netbox/extras/scripts.py | 15 ++++++++------- netbox/templates/extras/script.html | 2 +- netbox/templates/extras/script_list.html | 2 +- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/additional-features/custom-scripts.md b/docs/additional-features/custom-scripts.md index ebec6b921..5caeb66a3 100644 --- a/docs/additional-features/custom-scripts.md +++ b/docs/additional-features/custom-scripts.md @@ -39,20 +39,22 @@ Returning output from your script is optional. Any raw output generated by the s ## Script Attributes -### script_name +Script attributes are defined under a class named `Meta` within the script. These are optional, but encouraged. + +### `name` This is the human-friendly names of your script. If omitted, the class name will be used. -### script_description +### `description` -A human-friendly description of what your script does (optional). +A human-friendly description of what your script does. -### script_fields +### `fields` The order in which the variable fields should appear. This is optional, however on Python 3.5 and earlier the fields will appear in random order. (Declarative ordering is preserved on Python 3.6 and above.) For example: ``` -script_fields = ['var1', 'var2', 'var3'] +fields = ['var1', 'var2', 'var3'] ``` ## Logging @@ -124,9 +126,11 @@ from extras.scripts import * class NewBranchScript(Script): - script_name = "New Branch" - script_description = "Provision a new branch site" - script_fields = ['site_name', 'switch_count', 'switch_model'] + + class Meta: + name = "New Branch" + description = "Provision a new branch site" + fields = ['site_name', 'switch_count', 'switch_model'] site_name = StringVar( description="Name of the new site" diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 396240677..489ee7633 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -118,6 +118,9 @@ class Script: """ Custom scripts inherit this object. """ + class Meta: + pass + def __init__(self): # Initiate the log @@ -128,17 +131,15 @@ class Script: self.source = inspect.getsource(self.__class__) def __str__(self): - if hasattr(self, 'script_name'): - return self.script_name - return self.__class__.__name__ + return getattr(self.Meta, 'name', self.__class__.__name__) def _get_vars(self): vars = OrderedDict() - # Infer order from script_fields (Python 3.5 and lower) - if hasattr(self, 'script_fields'): - for name in self.script_fields: - vars[name] = getattr(self, name) + # Infer order from Meta.fields (Python 3.5 and lower) + fields = getattr(self.Meta, 'fields') + for name in fields: + vars[name] = getattr(self, name) # Default to order of declaration on class for name, attr in self.__class__.__dict__.items(): diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index 240e54a51..87f389c75 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -16,7 +16,7 @@

      {{ script }}

      -

      {{ script.script_description }}

      +

      {{ script.Meta.description }}

      - {% if script.log %} + {% if execution_time %}
      - Script Output + Script Log
      @@ -49,8 +49,17 @@ + {% empty %} + + + {% endfor %}
      {% log_level level %} {{ message }}
      + No log output +
      +
      + Exec time: {{ execution_time|floatformat:3 }}s +
      From 9c079ead4c0bc9706b5b6ac50fdf1b45a5a0db11 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 10:18:25 -0400 Subject: [PATCH 19/63] Fix notice when form does not require user input --- netbox/extras/forms.py | 7 +++++++ netbox/templates/extras/script.html | 10 ++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 32ffc1396..5432e34be 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -404,3 +404,10 @@ class ScriptForm(BootstrapMixin, forms.Form): # Move _commit to the end of the form self.fields.move_to_end('_commit', True) + + @property + def requires_input(self): + """ + A boolean indicating whether the form requires user input (ignore the _commit field). + """ + return bool(len(self.fields) > 1) diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index c2b9bd205..06b2dfb15 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -74,11 +74,13 @@ {% endif %} {% csrf_token %} - {% if form %} - {% render_form form %} - {% else %} -

      This script does not require any input to run.

      + {% if not form.requires_input %} +
      + + This script does not require any input to run. +
      {% endif %} + {% render_form form %}
      Cancel From 11e5e1c4908024d28c5fa8c67d6a07cb59969ee7 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 12:19:36 -0400 Subject: [PATCH 20/63] Show script log when an exception occurs --- netbox/templates/extras/script.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index 06b2dfb15..7a9ddb665 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -30,7 +30,7 @@
    - {% if execution_time %} + {% if execution_time or script.log %}
    From 7f65e009a8b42e2611583a5eb35ef3a80239548a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 13:08:21 -0400 Subject: [PATCH 21/63] Add convenience functions for loading YAML/JSON data from file --- netbox/extras/scripts.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index fac44a530..47bd8284c 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -1,7 +1,10 @@ from collections import OrderedDict import inspect +import json +import os import pkgutil import time +import yaml from django import forms from django.conf import settings @@ -196,6 +199,28 @@ class Script: def log_failure(self, message): self.log.append((LOG_FAILURE, message)) + # Convenience functions + + def load_yaml(self, filename): + """ + Return data from a YAML file + """ + file_path = os.path.join(settings.SCRIPTS_ROOT, filename) + with open(file_path, 'r') as datafile: + data = yaml.load(datafile) + + return data + + def load_json(self, filename): + """ + Return data from a JSON file + """ + file_path = os.path.join(settings.SCRIPTS_ROOT, filename) + with open(file_path, 'r') as datafile: + data = json.load(datafile) + + return data + # # Functions From 8bd1fad7d0022db84cce9d137960cc977dee9f7c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 14:03:11 -0400 Subject: [PATCH 22/63] Use TreeNodeChoiceField for MPTT objects --- netbox/extras/scripts.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 47bd8284c..cffb5e59d 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -10,6 +10,8 @@ from django import forms from django.conf import settings from django.core.validators import RegexValidator from django.db import transaction +from mptt.forms import TreeNodeChoiceField +from mptt.models import MPTTModel from ipam.formfields import IPFormField from utilities.exceptions import AbortTransaction @@ -124,6 +126,10 @@ class ObjectVar(ScriptVariable): # Queryset for field choices self.field_attrs['queryset'] = queryset + # Update form field for MPTT (nested) objects + if issubclass(queryset.model, MPTTModel): + self.form_field = TreeNodeChoiceField + class IPNetworkVar(ScriptVariable): """ From 434e656e277fc4f29fe8908f92c25f4225594bb8 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 14:26:13 -0400 Subject: [PATCH 23/63] Include stack trace when catching an exception --- netbox/extras/scripts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index cffb5e59d..206a53ec4 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -3,7 +3,9 @@ import inspect import json import os import pkgutil +import sys import time +import traceback import yaml from django import forms @@ -265,8 +267,9 @@ def run_script(script, data, commit=True): except AbortTransaction: pass except Exception as e: + stacktrace = traceback.format_exc() script.log_failure( - "An exception occurred. {}: {}".format(type(e).__name__, e) + "An exception occurred. {}: {}\n```{}```".format(type(e).__name__, e, stacktrace) ) commit = False finally: From f8326ef6df07544b5ed8e7085824524db2c325a2 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 14:38:11 -0400 Subject: [PATCH 24/63] Add markdown rendering for log mesages --- netbox/extras/scripts.py | 2 +- netbox/project-static/css/base.css | 3 +++ netbox/templates/extras/script.html | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 206a53ec4..156d0a4bc 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -269,7 +269,7 @@ def run_script(script, data, commit=True): except Exception as e: stacktrace = traceback.format_exc() script.log_failure( - "An exception occurred. {}: {}\n```{}```".format(type(e).__name__, e, stacktrace) + "An exception occurred: `{}: {}`\n```\n{}\n```".format(type(e).__name__, e, stacktrace) ) commit = False finally: diff --git a/netbox/project-static/css/base.css b/netbox/project-static/css/base.css index fcee05e12..93e2188ba 100644 --- a/netbox/project-static/css/base.css +++ b/netbox/project-static/css/base.css @@ -529,6 +529,9 @@ table.report th a { border-top: 1px solid #dddddd; padding: 8px; } +.rendered-markdown :last-child { + margin-bottom: 0; +} /* AJAX loader */ .loading { diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index 7a9ddb665..ae1f89b49 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -47,7 +47,7 @@ {{ forloop.counter }} {% log_level level %} - {{ message }} + {{ message|gfm }} {% empty %} From 47d60dbb20584d0c6b8dbdb047f88befb6c77941 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 15:46:08 -0400 Subject: [PATCH 25/63] Fix table column widths --- netbox/templates/extras/script_list.html | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index 3230ab714..5e115fba2 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -4,16 +4,15 @@ {% block content %}

    {% block title %}Scripts{% endblock %}

    -
    +
    {% if scripts %} {% for module, module_scripts in scripts.items %}

    {{ module|bettertitle }}

    - - - + + @@ -23,7 +22,6 @@ {{ script }} - {% endfor %} From cb0dbc0769c22b41d440d2b2cc50a7acfa23bf01 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 14 Aug 2019 16:20:52 -0400 Subject: [PATCH 26/63] Add TextVar for large text entry --- netbox/extras/scripts.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 156d0a4bc..2a0c0db7b 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -3,7 +3,6 @@ import inspect import json import os import pkgutil -import sys import time import traceback import yaml @@ -24,6 +23,7 @@ from .forms import ScriptForm __all__ = [ 'Script', 'StringVar', + 'TextVar', 'IntegerVar', 'BooleanVar', 'ObjectVar', @@ -87,6 +87,18 @@ class StringVar(ScriptVariable): ] +class TextVar(ScriptVariable): + """ + Free-form text data. Renders as a
    NameDescriptionNameDescription
    {{ script.Meta.description }}