mirror/netbox-community-netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2024-05-10 07:54:54 +00:00
Code Activity

Closes #11890: Sync/upload reports & scripts (#12059)

Browse Source 
* Initial work on #11890

* Consolidate get_scripts() and get_reports() functions

* Introduce proxy models for script & report modules

* Add add/delete views for reports & scripts

* Add deletion links for modules

* Enable resolving scripts/reports from module class

* Remove get_modules() utility function

* Show results in report/script lists

* Misc cleanup

* Fix file uploads

* Support automatic migration for submodules

* Fix module child ordering

* Template cleanup

* Remove ManagedFile views

* Move is_script(), is_report() into extras.utils

* Fix URLs for nested reports & scripts

* Misc cleanup
This commit is contained in:
Jeremy Stretch
2023-03-24 21:00:36 -04:00
committed by GitHub
parent 9c5f4163af
commit f7a2eb8aef
23 changed files with 659 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
netbox/core/choices.py
View File

@ -20,7 +20,6 @@ class DataSourceTypeChoices(ChoiceSet):
class DataSourceStatusChoices(ChoiceSet):
NEW = 'new'
QUEUED = 'queued'
SYNCING = 'syncing'
@ -34,3 +33,17 @@ class DataSourceStatusChoices(ChoiceSet):
(COMPLETED, _('Completed'), 'green'),
(FAILED, _('Failed'), 'red'),
)
#
# Managed files
#
class ManagedFileRootPathChoices(ChoiceSet):
SCRIPTS = 'scripts' # settings.SCRIPTS_ROOT
REPORTS = 'reports' # settings.REPORTS_ROOT
CHOICES = (
(SCRIPTS, _('Scripts')),
(REPORTS, _('Reports')),
)

36
netbox/core/forms/model_forms.py
View File

@ -3,12 +3,14 @@ import copy
from django import forms
from core.models import *
from extras.forms.mixins import SyncedDataMixin
from netbox.forms import NetBoxModelForm
from netbox.registry import registry
from utilities.forms import CommentField, get_field_value
__all__ = (
'DataSourceForm',
'ManagedFileForm',
)
@ -73,3 +75,37 @@ class DataSourceForm(NetBoxModelForm):
self.instance.parameters = parameters
return super().save(*args, **kwargs)
class ManagedFileForm(SyncedDataMixin, NetBoxModelForm):
upload_file = forms.FileField(
required=False
)
fieldsets = (
('File Upload', ('upload_file',)),
('Data Source', ('data_source', 'data_file')),
)
class Meta:
model = ManagedFile
fields = ('data_source', 'data_file')
def clean(self):
super().clean()
if self.cleaned_data.get('upload_file') and self.cleaned_data.get('data_file'):
raise forms.ValidationError("Cannot upload a file and sync from an existing file")
if not self.cleaned_data.get('upload_file') and not self.cleaned_data.get('data_file'):
raise forms.ValidationError("Must upload a file or select a data file to sync")
return self.cleaned_data
def save(self, *args, **kwargs):
# If a file was uploaded, save it to disk
if self.cleaned_data['upload_file']:
self.instance.file_path = self.cleaned_data['upload_file'].name
with open(self.instance.full_path, 'wb+') as new_file:
new_file.write(self.cleaned_data['upload_file'].read())
return super().save(*args, **kwargs)

39
netbox/core/migrations/0002_managedfile.py Normal file
View File

@ -0,0 +1,39 @@
# Generated by Django 4.1.7 on 2023-03-23 17:35
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('core', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='ManagedFile',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
('data_path', models.CharField(blank=True, editable=False, max_length=1000)),
('data_synced', models.DateTimeField(blank=True, editable=False, null=True)),
('created', models.DateTimeField(auto_now_add=True)),
('last_updated', models.DateTimeField(blank=True, editable=False, null=True)),
('file_root', models.CharField(max_length=1000)),
('file_path', models.FilePathField(editable=False)),
('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')),
('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')),
],
options={
'ordering': ('file_root', 'file_path'),
},
),
migrations.AddIndex(
model_name='managedfile',
index=models.Index(fields=['file_root', 'file_path'], name='core_managedfile_root_path'),
),
migrations.AddConstraint(
model_name='managedfile',
constraint=models.UniqueConstraint(fields=('file_root', 'file_path'), name='core_managedfile_unique_root_path'),
),
]

1
netbox/core/models/__init__.py
View File

@ -1 +1,2 @@
from .data import *
from .files import *

14
netbox/core/models/data.py
View File

@ -14,7 +14,6 @@ from django.utils import timezone
from django.utils.module_loading import import_string
from django.utils.translation import gettext as _
from extras.models import JobResult
from netbox.models import PrimaryModel
from netbox.registry import registry
from utilities.files import sha256_hash
@ -113,6 +112,8 @@ class DataSource(PrimaryModel):
"""
Enqueue a background job to synchronize the DataSource by calling sync().
"""
from extras.models import JobResult
# Set the status to "syncing"
self.status = DataSourceStatusChoices.QUEUED
DataSource.objects.filter(pk=self.pk).update(status=self.status)
@ -314,3 +315,14 @@ class DataFile(models.Model):
self.data = f.read()
return is_modified
def write_to_disk(self, path, overwrite=False):
"""
Write the object's data to disk at the specified path
"""
# Check whether file already exists
if os.path.isfile(path) and not overwrite:
raise FileExistsError()
with open(path, 'wb+') as new_file:
new_file.write(self.data)

88
netbox/core/models/files.py Normal file
View File

@ -0,0 +1,88 @@
import logging
import os
from django.conf import settings
from django.db import models
from django.urls import reverse
from django.utils.translation import gettext as _
from ..choices import ManagedFileRootPathChoices
from netbox.models.features import SyncedDataMixin
from utilities.querysets import RestrictedQuerySet
__all__ = (
'ManagedFile',
)
logger = logging.getLogger('netbox.core.files')
class ManagedFile(SyncedDataMixin, models.Model):
"""
Database representation for a file on disk. This class is typically wrapped by a proxy class (e.g. ScriptModule)
to provide additional functionality.
"""
created = models.DateTimeField(
auto_now_add=True
)
last_updated = models.DateTimeField(
editable=False,
blank=True,
null=True
)
file_root = models.CharField(
max_length=1000,
choices=ManagedFileRootPathChoices
)
file_path = models.FilePathField(
editable=False,
help_text=_("File path relative to the designated root path")
)
objects = RestrictedQuerySet.as_manager()
class Meta:
ordering = ('file_root', 'file_path')
constraints = (
models.UniqueConstraint(
fields=('file_root', 'file_path'),
name='%(app_label)s_%(class)s_unique_root_path'
),
)
indexes = [
models.Index(fields=('file_root', 'file_path'), name='core_managedfile_root_path'),
]
def __str__(self):
return self.name
def get_absolute_url(self):
return reverse('core:managedfile', args=[self.pk])
@property
def name(self):
return self.file_path
@property
def full_path(self):
return os.path.join(self._resolve_root_path(), self.file_path)
def _resolve_root_path(self):
return {
'scripts': settings.SCRIPTS_ROOT,
'reports': settings.REPORTS_ROOT,
}[self.file_root]
def sync_data(self):
if self.data_file:
self.file_path = os.path.basename(self.data_path)
self.data_file.write_to_disk(self.full_path, overwrite=True)
def delete(self, *args, **kwargs):
# Delete file from disk
try:
os.remove(self.full_path)
except FileNotFoundError:
pass
return super().delete(*args, **kwargs)

28
netbox/extras/api/views.py
View File

@ -16,8 +16,8 @@ from extras import filtersets
from extras.choices import JobResultStatusChoices
from extras.models import *
from extras.models import CustomField
from extras.reports import get_report, get_reports, run_report
from extras.scripts import get_script, get_scripts, run_script
from extras.reports import get_report, run_report
from extras.scripts import get_script, run_script
from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
from netbox.api.features import SyncedDataMixin
from netbox.api.metadata import ContentTypeMetadata
@ -27,7 +27,6 @@ from utilities.exceptions import RQWorkerNotRunningException
from utilities.utils import copy_safe_request, count_related
from . import serializers
from .mixins import ConfigTemplateRenderMixin
from .nested_serializers import NestedConfigTemplateSerializer
class ExtrasRootView(APIRootView):
@ -189,7 +188,6 @@ class ReportViewSet(ViewSet):
"""
Compile all reports and their related results (if any). Result data is deferred in the list view.
"""
report_list = []
report_content_type = ContentType.objects.get(app_label='extras', model='report')
results = {
r.name: r
@ -199,13 +197,13 @@ class ReportViewSet(ViewSet):
).order_by('name', '-created').distinct('name').defer('data')
}
# Iterate through all available Reports.
for module_name, reports in get_reports().items():
for report in reports.values():
report_list = []
for report_module in ReportModule.objects.restrict(request.user):
report_list.extend([report() for report in report_module.reports.values()])
# Attach the relevant JobResult (if any) to each Report.
report.result = results.get(report.full_name, None)
report_list.append(report)
# Attach JobResult objects to each report (if any)
for report in report_list:
report.result = results.get(report.full_name, None)
serializer = serializers.ReportSerializer(report_list, many=True, context={
'request': request,
@ -296,15 +294,15 @@ class ScriptViewSet(ViewSet):
).order_by('name', '-created').distinct('name').defer('data')
}
flat_list = []
for script_list in get_scripts().values():
flat_list.extend(script_list.values())
script_list = []
for script_module in ScriptModule.objects.restrict(request.user):
script_list.extend(script_module.scripts.values())
# Attach JobResult objects to each script (if any)
for script in flat_list:
for script in script_list:
script.result = results.get(script.full_name, None)
serializer = serializers.ScriptSerializer(flat_list, many=True, context={'request': request})
serializer = serializers.ScriptSerializer(script_list, many=True, context={'request': request})
return Response(serializer.data)

14
netbox/extras/management/commands/runreport.py
View File

@ -5,8 +5,8 @@ from django.core.management.base import BaseCommand
from django.utils import timezone
from extras.choices import JobResultStatusChoices
from extras.models import JobResult
from extras.reports import get_reports, run_report
from extras.models import JobResult, ReportModule
from extras.reports import run_report
class Command(BaseCommand):
@ -17,13 +17,9 @@ class Command(BaseCommand):
def handle(self, *args, **options):
# Gather all available reports
reports = get_reports()
# Run reports
for module_name, report_list in reports.items():
for report in report_list.values():
if module_name in options['reports'] or report.full_name in options['reports']:
for module in ReportModule.objects.all():
for report in module.reports.values():
if module.name in options['reports'] or report.full_name in options['reports']:
# Run the report and create a new JobResult
self.stdout.write(

86
netbox/extras/migrations/0091_create_managedfiles.py Normal file
View File

@ -0,0 +1,86 @@
import os
import pkgutil
from django.conf import settings
from django.db import migrations, models
import extras.models.models
def create_files(cls, root_name, root_path):
path_tree = [
path for path, _, _ in os.walk(root_path)
if os.path.basename(path)[0] not in ('_', '.')
]
modules = list(pkgutil.iter_modules(path_tree))
filenames = []
for importer, module_name, is_pkg in modules:
if is_pkg:
continue
try:
module = importer.find_module(module_name).load_module(module_name)
rel_path = os.path.relpath(module.__file__, root_path)
filenames.append(rel_path)
except ImportError:
pass
managed_files = [
cls(file_root=root_name, file_path=filename)
for filename in filenames
]
cls.objects.bulk_create(managed_files)
def replicate_scripts(apps, schema_editor):
ScriptModule = apps.get_model('extras', 'ScriptModule')
create_files(ScriptModule, 'scripts', settings.SCRIPTS_ROOT)
def replicate_reports(apps, schema_editor):
ReportModule = apps.get_model('extras', 'ReportModule')
create_files(ReportModule, 'reports', settings.REPORTS_ROOT)
class Migration(migrations.Migration):
dependencies = [
('core', '0002_managedfile'),
('extras', '0090_objectchange_index_request_id'),
]
operations = [
# Create proxy models
migrations.CreateModel(
name='ReportModule',
fields=[
],
options={
'proxy': True,
'indexes': [],
'constraints': [],
},
bases=(extras.models.models.PythonModuleMixin, 'core.managedfile', models.Model),
),
migrations.CreateModel(
name='ScriptModule',
fields=[
],
options={
'proxy': True,
'indexes': [],
'constraints': [],
},
bases=(extras.models.models.PythonModuleMixin, 'core.managedfile', models.Model),
),
# Instantiate ManagedFiles to represent scripts & reports
migrations.RunPython(
code=replicate_scripts,
reverse_code=migrations.RunPython.noop
),
migrations.RunPython(
code=replicate_reports,
reverse_code=migrations.RunPython.noop
),
]

2
netbox/extras/models/__init__.py
View File

@ -23,8 +23,10 @@ __all__ = (
'JournalEntry',
'ObjectChange',
'Report',
'ReportModule',
'SavedFilter',
'Script',
'ScriptModule',
'StagedChange',
'Tag',
'TaggedItem',

119
netbox/extras/models/models.py
View File

@ -1,6 +1,11 @@
import inspect
import json
import os
import uuid
from functools import cached_property
from pkgutil import ModuleInfo, get_importer
import django_rq
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
@ -16,12 +21,13 @@ from django.utils import timezone
from django.utils.formats import date_format
from django.utils.translation import gettext as _
from rest_framework.utils.encoders import JSONEncoder
import django_rq
from core.choices import ManagedFileRootPathChoices
from core.models import ManagedFile
from extras.choices import *
from extras.constants import *
from extras.conditions import ConditionSet
from extras.utils import FeatureQuery, image_upload
from extras.constants import *
from extras.utils import FeatureQuery, image_upload, is_report, is_script
from netbox.config import get_config
from netbox.constants import RQ_QUEUE_DEFAULT
from netbox.models import ChangeLoggedModel
@ -41,8 +47,10 @@ __all__ = (
'JobResult',
'JournalEntry',
'Report',
'ReportModule',
'SavedFilter',
'Script',
'ScriptModule',
'Webhook',
)
@ -814,6 +822,27 @@ class ConfigRevision(models.Model):
# Custom scripts & reports
#
class PythonModuleMixin:
@property
def path(self):
return os.path.splitext(self.file_path)[0]
def get_module_info(self):
path = os.path.dirname(self.full_path)
module_name = os.path.basename(self.path)
return ModuleInfo(
module_finder=get_importer(path),
name=module_name,
ispkg=False
)
def get_module(self):
importer, module_name, _ = self.get_module_info()
module = importer.find_module(module_name).load_module(module_name)
return module
class Script(JobResultsMixin, WebhooksMixin, models.Model):
"""
Dummy model used to generate permissions for custom scripts. Does not exist in the database.
@ -822,6 +851,48 @@ class Script(JobResultsMixin, WebhooksMixin, models.Model):
managed = False
class ScriptModuleManager(models.Manager.from_queryset(RestrictedQuerySet)):
def get_queryset(self):
return super().get_queryset().filter(file_root=ManagedFileRootPathChoices.SCRIPTS)
class ScriptModule(PythonModuleMixin, ManagedFile):
"""
Proxy model for script module files.
"""
objects = ScriptModuleManager()
class Meta:
proxy = True
def get_absolute_url(self):
return reverse('extras:script_list')
@cached_property
def scripts(self):
def _get_name(cls):
# For child objects in submodules use the full import path w/o the root module as the name
return cls.full_name.split(".", maxsplit=1)[1]
module = self.get_module()
scripts = {}
ordered = getattr(module, 'script_order', [])
for cls in ordered:
scripts[_get_name(cls)] = cls
for name, cls in inspect.getmembers(module, is_script):
if cls not in ordered:
scripts[_get_name(cls)] = cls
return scripts
def save(self, *args, **kwargs):
self.file_root = ManagedFileRootPathChoices.SCRIPTS
return super().save(*args, **kwargs)
#
# Reports
#
@ -832,3 +903,45 @@ class Report(JobResultsMixin, WebhooksMixin, models.Model):
"""
class Meta:
managed = False
class ReportModuleManager(models.Manager.from_queryset(RestrictedQuerySet)):
def get_queryset(self):
return super().get_queryset().filter(file_root=ManagedFileRootPathChoices.REPORTS)
class ReportModule(PythonModuleMixin, ManagedFile):
"""
Proxy model for report module files.
"""
objects = ReportModuleManager()
class Meta:
proxy = True
def get_absolute_url(self):
return reverse('extras:report_list')
@cached_property
def reports(self):
def _get_name(cls):
# For child objects in submodules use the full import path w/o the root module as the name
return cls.full_name.split(".", maxsplit=1)[1]
module = self.get_module()
reports = {}
ordered = getattr(module, 'report_order', [])
for cls in ordered:
reports[_get_name(cls)] = cls
for name, cls in inspect.getmembers(module, is_report):
if cls not in ordered:
reports[_get_name(cls)] = cls
return reports
def save(self, *args, **kwargs):
self.file_root = ManagedFileRootPathChoices.REPORTS
return super().save(*args, **kwargs)

84
netbox/extras/reports.py
View File

@ -1,75 +1,23 @@
import inspect
import logging
import pkgutil
import traceback
from datetime import timedelta
from django.conf import settings
from django.utils import timezone
from django.utils.functional import classproperty
from django_rq import job
from .choices import JobResultStatusChoices, LogLevelChoices
from .models import JobResult
from .models import JobResult, ReportModule
logger = logging.getLogger(__name__)
def is_report(obj):
"""
Returns True if the given object is a Report.
"""
return obj in Report.__subclasses__()
def get_report(module_name, report_name):
"""
Return a specific report from within a module.
"""
reports = get_reports()
module = reports.get(module_name)
if module is None:
return None
report = module.get(report_name)
if report is None:
return None
return report
def get_reports():
"""
Compile a list of all reports available across all modules in the reports path. Returns a list of tuples:
[
(module_name, (report, report, report, ...)),
(module_name, (report, report, report, ...)),
...
]
"""
module_list = {}
# Iterate through all modules within the reports path. These are the user-created files in which reports are
# defined.
for importer, module_name, _ in pkgutil.iter_modules([settings.REPORTS_ROOT]):
module = importer.find_module(module_name).load_module(module_name)
report_order = getattr(module, "report_order", ())
ordered_reports = [cls() for cls in report_order if is_report(cls)]
unordered_reports = [cls() for _, cls in inspect.getmembers(module, is_report) if cls not in report_order]
module_reports = {}
for cls in [*ordered_reports, *unordered_reports]:
# For reports in submodules use the full import path w/o the root module as the name
report_name = cls.full_name.split(".", maxsplit=1)[1]
module_reports[report_name] = cls
if module_reports:
module_list[module_name] = module_reports
return module_list
module = ReportModule.objects.get(file_path=f'{module_name}.py')
return module.reports.get(report_name)
@job('default')
@ -79,7 +27,7 @@ def run_report(job_result, *args, **kwargs):
method for queueing into the background processor.
"""
module_name, report_name = job_result.name.split('.', 1)
report = get_report(module_name, report_name)
report = get_report(module_name, report_name)()
try:
job_result.start()
@ -136,7 +84,7 @@ class Report(object):
self.active_test = None
self.failed = False
self.logger = logging.getLogger(f"netbox.reports.{self.full_name}")
self.logger = logging.getLogger(f"netbox.reports.{self.__module__}.{self.__class__.__name__}")
# Compile test methods and initialize results skeleton
test_methods = []
@ -154,13 +102,17 @@ class Report(object):
raise Exception("A report must contain at least one test method.")
self.test_methods = test_methods
@property
@classproperty
def module(self):
return self.__module__
@property
@classproperty
def class_name(self):
return self.__class__.__name__
return self.__name__
@classproperty
def full_name(self):
return f'{self.module}.{self.class_name}'
@property
def name(self):
@ -169,9 +121,9 @@ class Report(object):
"""
return self.class_name
@property
def full_name(self):
return f'{self.module}.{self.class_name}'
#
# Logging methods
#
def _log(self, obj, message, level=LogLevelChoices.LOG_DEFAULT):
"""
@ -228,6 +180,10 @@ class Report(object):
self.logger.info(f"Failure | {obj}: {message}")
self.failed = True
#
# Run methods
#
def run(self, job_result):
"""
Run the report and save its results. Each test method will be executed in order.

95
netbox/extras/scripts.py
View File

@ -2,9 +2,6 @@ import inspect
import json
import logging
import os
import pkgutil
import sys
import threading
import traceback
from datetime import timedelta
@ -17,7 +14,7 @@ from django.utils.functional import classproperty
from extras.api.serializers import ScriptOutputSerializer
from extras.choices import JobResultStatusChoices, LogLevelChoices
from extras.models import JobResult
from extras.models import JobResult, ScriptModule
from extras.signals import clear_webhooks
from ipam.formfields import IPAddressFormField, IPNetworkFormField
from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator
@ -43,8 +40,6 @@ __all__ = [
'TextVar',
]
lock = threading.Lock()
#
# Script variables
@ -272,7 +267,7 @@ class BaseScript:
def __init__(self):
# Initiate the log
self.logger = logging.getLogger(f"netbox.scripts.{self.module()}.{self.__class__.__name__}")
self.logger = logging.getLogger(f"netbox.scripts.{self.__module__}.{self.__class__.__name__}")
self.log = []
# Declare the placeholder for the current request
@ -285,22 +280,26 @@ class BaseScript:
def __str__(self):
return self.name
@classproperty
def module(self):
return self.__module__
@classproperty
def class_name(self):
return self.__name__
@classproperty
def full_name(self):
return f'{self.module}.{self.class_name}'
@classproperty
def name(self):
return getattr(self.Meta, 'name', self.__name__)
@classproperty
def full_name(self):
return '.'.join([self.__module__, self.__name__])
@classproperty
def description(self):
return getattr(self.Meta, 'description', '')
@classmethod
def module(cls):
return cls.__module__
@classmethod
def root_module(cls):
return cls.__module__.split(".")[0]
@ -427,15 +426,6 @@ class Script(BaseScript):
# Functions
#
def is_script(obj):
"""
Returns True if the object is a Script.
"""
try:
return issubclass(obj, Script) and obj != Script
except TypeError:
return False
def is_variable(obj):
"""
@ -452,10 +442,10 @@ def run_script(data, request, commit=True, *args, **kwargs):
job_result = kwargs.pop('job_result')
job_result.start()
module, script_name = job_result.name.split('.', 1)
script = get_script(module, script_name)()
module_name, script_name = job_result.name.split('.', 1)
script = get_script(module_name, script_name)()
logger = logging.getLogger(f"netbox.scripts.{module}.{script_name}")
logger = logging.getLogger(f"netbox.scripts.{module_name}.{script_name}")
logger.info(f"Running script (commit={commit})")
# Add files to form data
@ -522,56 +512,9 @@ def run_script(data, request, commit=True, *args, **kwargs):
)
def get_scripts(use_names=False):
"""
Return a dict of dicts mapping all scripts to their modules. Set use_names to True to use each module's human-
defined name in place of the actual module name.
"""
scripts = {}
# Get all modules within the scripts path. These are the user-created files in which scripts are
# defined.
modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT]))
modules_bases = set([name.split(".")[0] for _, name, _ in modules])
# Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is
# removed from sys.modules while another thread is importing
with lock:
for module_name in list(sys.modules.keys()):
# Everything sharing a base module path with a module in the script folder is removed.
# We also remove all modules with a base module called "scripts". This allows modifying imported
# non-script modules without having to reload the RQ worker.
module_base = module_name.split(".")[0]
if module_base == "scripts" or module_base in modules_bases:
del sys.modules[module_name]
for importer, module_name, _ in modules:
module = importer.find_module(module_name).load_module(module_name)
if use_names and hasattr(module, 'name'):
module_name = module.name
module_scripts = {}
script_order = getattr(module, "script_order", ())
ordered_scripts = [cls for cls in script_order if is_script(cls)]
unordered_scripts = [cls for _, cls in inspect.getmembers(module, is_script) if cls not in script_order]
for cls in [*ordered_scripts, *unordered_scripts]:
# For scripts in submodules use the full import path w/o the root module as the name
script_name = cls.full_name.split(".", maxsplit=1)[1]
module_scripts[script_name] = cls
if module_scripts:
scripts[module_name] = module_scripts
return scripts
def get_script(module_name, script_name):
"""
Retrieve a script class by module and name. Returns None if the script does not exist.
"""
scripts = get_scripts()
module = scripts.get(module_name)
if module:
return module.get(script_name)
module = ScriptModule.objects.get(file_path=f'{module_name}.py')
return module.scripts.get(script_name)

16
netbox/extras/urls.py
View File

@ -94,19 +94,23 @@ urlpatterns = [
# Reports
path('reports/', views.ReportListView.as_view(), name='report_list'),
path('reports/add/', views.ReportModuleCreateView.as_view(), name='reportmodule_add'),
path('reports/results/<int:job_result_pk>/', views.ReportResultView.as_view(), name='report_result'),
re_path(r'^reports/(?P<module>.([^.]+)).(?P<name>.(.+))/', views.ReportView.as_view(), name='report'),
path('reports/<int:pk>/', include(get_model_urls('extras', 'reportmodule'))),
path('reports/<path:module>.<str:name>/', views.ReportView.as_view(), name='report'),
# Scripts
path('scripts/', views.ScriptListView.as_view(), name='script_list'),
path('scripts/add/', views.ScriptModuleCreateView.as_view(), name='scriptmodule_add'),
path('scripts/results/<int:job_result_pk>/', views.ScriptResultView.as_view(), name='script_result'),
path('scripts/<int:pk>/', include(get_model_urls('extras', 'scriptmodule'))),
path('scripts/<path:module>.<str:name>/', views.ScriptView.as_view(), name='script'),
# Job results
path('job-results/', views.JobResultListView.as_view(), name='jobresult_list'),
path('job-results/delete/', views.JobResultBulkDeleteView.as_view(), name='jobresult_bulk_delete'),
path('job-results/<int:pk>/delete/', views.JobResultDeleteView.as_view(), name='jobresult_delete'),
# Scripts
path('scripts/', views.ScriptListView.as_view(), name='script_list'),
path('scripts/results/<int:job_result_pk>/', views.ScriptResultView.as_view(), name='script_result'),
re_path(r'^scripts/(?P<module>.([^.]+)).(?P<name>.(.+))/', views.ScriptView.as_view(), name='script'),
# Markdown
path('render/markdown/', views.RenderMarkdownView.as_view(), name="render_markdown")
]

22
netbox/extras/utils.py
View File

@ -66,3 +66,25 @@ def register_features(model, features):
raise KeyError(
f"{feature} is not a valid model feature! Valid keys are: {registry['model_features'].keys()}"
)
def is_script(obj):
"""
Returns True if the object is a Script.
"""
from .scripts import Script
try:
return issubclass(obj, Script) and obj != Script
except TypeError:
return False
def is_report(obj):
"""
Returns True if the given object is a Report.
"""
from .reports import Report
try:
return issubclass(obj, Report) and obj != Report
except TypeError:
return False

108
netbox/extras/views.py
View File

@ -7,6 +7,8 @@ from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.views.generic import View
from core.choices import ManagedFileRootPathChoices
from core.forms import ManagedFileForm
from extras.dashboard.forms import DashboardWidgetAddForm, DashboardWidgetForm
from extras.dashboard.utils import get_widget_class
from netbox.views import generic
@ -20,8 +22,8 @@ from . import filtersets, forms, tables
from .choices import JobResultStatusChoices
from .forms.reports import ReportForm
from .models import *
from .reports import get_report, get_reports, run_report
from .scripts import get_scripts, run_script
from .reports import get_report, run_report
from .scripts import run_script
#
@ -790,18 +792,34 @@ class DashboardWidgetDeleteView(LoginRequiredMixin, View):
# Reports
#
@register_model_view(ReportModule, 'edit')
class ReportModuleCreateView(generic.ObjectEditView):
queryset = ReportModule.objects.all()
form = ManagedFileForm
def alter_object(self, obj, *args, **kwargs):
obj.file_root = ManagedFileRootPathChoices.REPORTS
return obj
@register_model_view(ReportModule, 'delete')
class ReportModuleDeleteView(generic.ObjectDeleteView):
queryset = ReportModule.objects.all()
default_return_url = 'extras:report_list'
class ReportListView(ContentTypePermissionRequiredMixin, View):
"""
Retrieve all of the available reports from disk and the recorded JobResult (if any) for each.
Retrieve all the available reports from disk and the recorded JobResult (if any) for each.
"""
def get_required_permission(self):
return 'extras.view_report'
def get(self, request):
report_modules = ReportModule.objects.restrict(request.user)
reports = get_reports()
report_content_type = ContentType.objects.get(app_label='extras', model='report')
results = {
job_results = {
r.name: r
for r in JobResult.objects.filter(
obj_type=report_content_type,
@ -809,17 +827,10 @@ class ReportListView(ContentTypePermissionRequiredMixin, View):
).order_by('name', '-created').distinct('name').defer('data')
}
ret = []
for module, report_list in reports.items():
module_reports = []
for report in report_list.values():
report.result = results.get(report.full_name, None)
module_reports.append(report)
ret.append((module, module_reports))
return render(request, 'extras/report_list.html', {
'reports': ret,
'model': ReportModule,
'report_modules': report_modules,
'job_results': job_results,
})
@ -831,10 +842,8 @@ class ReportView(ContentTypePermissionRequiredMixin, View):
return 'extras.view_report'
def get(self, request, module, name):
report = get_report(module, name)
if report is None:
raise Http404
module = get_object_or_404(ReportModule.objects.restrict(request.user), file_path=f'{module}.py')
report = module.reports[name]()
report_content_type = ContentType.objects.get(app_label='extras', model='report')
report.result = JobResult.objects.filter(
@ -844,20 +853,17 @@ class ReportView(ContentTypePermissionRequiredMixin, View):
).first()
return render(request, 'extras/report.html', {
'module': module,
'report': report,
'form': ReportForm(),
})
def post(self, request, module, name):
# Permissions check
if not request.user.has_perm('extras.run_report'):
return HttpResponseForbidden()
report = get_report(module, name)
if report is None:
raise Http404
module = get_object_or_404(ReportModule.objects.restrict(request.user), file_path=f'{module}.py')
report = module.reports[name]()
form = ReportForm(request.POST)
if form.is_valid():
@ -883,6 +889,7 @@ class ReportView(ContentTypePermissionRequiredMixin, View):
return redirect('extras:report_result', job_result_pk=job_result.pk)
return render(request, 'extras/report.html', {
'module': module,
'report': report,
'form': form,
})
@ -924,15 +931,20 @@ class ReportResultView(ContentTypePermissionRequiredMixin, View):
# Scripts
#
class GetScriptMixin:
def _get_script(self, name, module=None):
if module is None:
module, name = name.split('.', 1)
scripts = get_scripts()
try:
return scripts[module][name]()
except KeyError:
raise Http404
@register_model_view(ScriptModule, 'edit')
class ScriptModuleCreateView(generic.ObjectEditView):
queryset = ScriptModule.objects.all()
form = ManagedFileForm
def alter_object(self, obj, *args, **kwargs):
obj.file_root = ManagedFileRootPathChoices.SCRIPTS
return obj
@register_model_view(ScriptModule, 'delete')
class ScriptModuleDeleteView(generic.ObjectDeleteView):
queryset = ScriptModule.objects.all()
default_return_url = 'extras:script_list'
class ScriptListView(ContentTypePermissionRequiredMixin, View):
@ -941,10 +953,10 @@ class ScriptListView(ContentTypePermissionRequiredMixin, View):
return 'extras.view_script'
def get(self, request):
script_modules = ScriptModule.objects.restrict(request.user)
scripts = get_scripts(use_names=True)
script_content_type = ContentType.objects.get(app_label='extras', model='script')
results = {
job_results = {
r.name: r
for r in JobResult.objects.filter(
obj_type=script_content_type,
@ -952,22 +964,21 @@ class ScriptListView(ContentTypePermissionRequiredMixin, View):
).order_by('name', '-created').distinct('name').defer('data')
}
for _scripts in scripts.values():
for script in _scripts.values():
script.result = results.get(script.full_name)
return render(request, 'extras/script_list.html', {
'scripts': scripts,
'model': ScriptModule,
'script_modules': script_modules,
'job_results': job_results,
})
class ScriptView(ContentTypePermissionRequiredMixin, GetScriptMixin, View):
class ScriptView(ContentTypePermissionRequiredMixin, View):
def get_required_permission(self):
return 'extras.view_script'
def get(self, request, module, name):
script = self._get_script(name, module)
module = get_object_or_404(ScriptModule.objects.restrict(request.user), file_path=f'{module}.py')
script = module.scripts[name]()
form = script.as_form(initial=normalize_querydict(request.GET))
# Look for a pending JobResult (use the latest one by creation timestamp)
@ -985,12 +996,11 @@ class ScriptView(ContentTypePermissionRequiredMixin, GetScriptMixin, View):
})
def post(self, request, module, name):
# Permissions check
if not request.user.has_perm('extras.run_script'):
return HttpResponseForbidden()
script = self._get_script(name, module)
module = get_object_or_404(ScriptModule.objects.restrict(request.user), file_path=f'{module}.py')
script = module.scripts[name]()
form = script.as_form(request.POST, request.FILES)
# Allow execution only if RQ worker process is running
@ -1020,7 +1030,7 @@ class ScriptView(ContentTypePermissionRequiredMixin, GetScriptMixin, View):
})
class ScriptResultView(ContentTypePermissionRequiredMixin, GetScriptMixin, View):
class ScriptResultView(ContentTypePermissionRequiredMixin, View):
def get_required_permission(self):
return 'extras.view_script'
@ -1031,7 +1041,9 @@ class ScriptResultView(ContentTypePermissionRequiredMixin, GetScriptMixin, View)
if result.obj_type != script_content_type:
raise Http404
script = self._get_script(result.name)
module_name, script_name = result.name.split('.', 1)
module = get_object_or_404(ScriptModule.objects.restrict(request.user), file_path=f'{module_name}.py')
script = module.scripts[script_name]()
# If this is an HTMX request, return only the result HTML
if is_htmx(request):

2
netbox/templates/extras/htmx/report_result.html
View File

@ -12,7 +12,7 @@
{% if result.completed %}
Duration: <strong>{{ result.duration }}</strong>
{% endif %}
<span id="pending-result-label">{% include 'extras/inc/job_label.html' %}</span>
<span id="pending-result-label">{% badge result.get_status_display result.get_status_color %}</span>
</p>
{% if result.completed %}
<div class="card">

2
netbox/templates/extras/htmx/script_result.html
View File

@ -13,7 +13,7 @@
{% if result.completed %}
Duration: <strong>{{ result.duration }}</strong>
{% endif %}
<span id="pending-result-label">{% include 'extras/inc/job_label.html' %}</span>
<span id="pending-result-label">{% badge result.get_status_display result.get_status_color %}</span>
</p>
{% if result.completed %}
<div class="card mb-3">

15
netbox/templates/extras/inc/job_label.html
View File

@ -1,15 +0,0 @@
{% if result.status == 'failed' %}
<span class="badge bg-danger">Failed</span>
{% elif result.status == 'errored' %}
<span class="badge bg-danger">Errored</span>
{% elif result.status == 'pending' %}
<span class="badge bg-info">Pending</span>
{% elif result.status == 'scheduled' %}
<span class="badge bg-info">Scheduled</span>
{% elif result.status == 'running' %}
<span class="badge bg-warning">Running</span>
{% elif result.status == 'completed' %}
<span class="badge bg-success">Completed</span>
{% else %}
<span class="badge bg-secondary">N/A</span>
{% endif %}

2
netbox/templates/extras/report.html
View File

@ -10,7 +10,7 @@
{% block breadcrumbs %}
<li class="breadcrumb-item"><a href="{% url 'extras:report_list' %}">Reports</a></li>
<li class="breadcrumb-item"><a href="{% url 'extras:report_list' %}#module.{{ report.module }}">{{ report.module|bettertitle }}</a></li>
<li class="breadcrumb-item"><a href="{% url 'extras:report_list' %}#module{{ module.pk }}">{{ report.module|bettertitle }}</a></li>
{% endblock breadcrumbs %}
{% block subtitle %}

99
netbox/templates/extras/report_list.html
View File

@ -1,5 +1,7 @@
{% extends 'base/layout.html' %}
{% load buttons %}
{% load helpers %}
{% load perms %}
{% block title %}Reports{% endblock %}
@ -11,50 +13,67 @@
</ul>
{% endblock tabs %}
{% block controls %}
<div class="controls">
<div class="control-group">
{% block extra_controls %}{% endblock %}
{% add_button model %}
</div>
</div>
{% endblock controls %}
{% block content-wrapper %}
<div class="tab-content">
{% if reports %}
{% for module, module_reports in reports %}
<div class="card">
<h5 class="card-header">
<a name="module.{{ module }}"></a>
<i class="mdi mdi-file-document-outline"></i> {{ module|bettertitle }}
</h5>
<div class="card-body">
<table class="table table-hover table-headings reports">
<thead>
<tr>
<th width="250">Name</th>
<th width="110">Status</th>
<th>Description</th>
<th width="150" class="text-end">Last Run</th>
<th width="120"></th>
</tr>
</thead>
<tbody>
{% for report in module_reports %}
{% for module in report_modules %}
<div class="card">
<h5 class="card-header" id="module{{ module.pk }}">
{% if perms.extras.delete_reportmodule %}
<div class="float-end">
<a href="{% url 'extras:reportmodule_delete' pk=module.pk %}" class="btn btn-danger btn-sm">
<i class="mdi mdi-trash-can-outline" aria-hidden="true"></i> Delete
</a>
</div>
{% endif %}
<i class="mdi mdi-file-document-outline"></i> {{ module.name|bettertitle }}
</h5>
<div class="card-body">
{% include 'inc/sync_warning.html' with object=module %}
<table class="table table-hover table-headings reports">
<thead>
<tr>
<th width="250">Name</th>
<th>Description</th>
<th>Last Run</th>
<th>Status</th>
<th width="120"></th>
</tr>
</thead>
<tbody>
{% for report_name, report in module.reports.items %}
{% with last_result=job_results|get_key:report.full_name %}
<tr>
<td>
<a href="{% url 'extras:report' module=report.module name=report.class_name %}" id="{{ report.module }}.{{ report.class_name }}">{{ report.name }}</a>
</td>
<td>
{% include 'extras/inc/job_label.html' with result=report.result %}
<a href="{% url 'extras:report' module=module.path name=report.class_name %}" id="{{ report.module }}.{{ report.class_name }}">{{ report.name }}</a>
</td>
<td>{{ report.description|markdown|placeholder }}</td>
<td class="text-end">
{% if report.result %}
<a href="{% url 'extras:report_result' job_result_pk=report.result.pk %}">{{ report.result.created|annotated_date }}</a>
{% else %}
<span class="text-muted">Never</span>
{% endif %}
</td>
{% if last_result %}
<td>
<a href="{% url 'extras:report_result' job_result_pk=last_result.pk %}">{{ last_result.created|annotated_date }}</a>
</td>
<td>
{% badge last_result.get_status_display last_result.get_status_color %}
</td>
{% else %}
<td class="text-muted">Never</td>
<td>{{ ''|placeholder }}</td>
{% endif %}
<td>
{% if perms.extras.run_report %}
<div class="float-end noprint">
<form action="{% url 'extras:report' module=report.module name=report.class_name %}" method="post">
{% csrf_token %}
<button type="submit" name="_run" class="btn btn-primary btn-sm" style="width: 110px">
{% if report.result %}
{% if last_result %}
<i class="mdi mdi-replay"></i> Run Again
{% else %}
<i class="mdi mdi-play"></i> Run Report
@ -65,7 +84,7 @@
{% endif %}
</td>
</tr>
{% for method, stats in report.result.data.items %}
{% for method, stats in last_result.data.items %}
<tr>
<td colspan="4" class="method">
<span class="ps-3">{{ method }}</span>
@ -78,19 +97,19 @@
</td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
</div>
{% endwith %}
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
{% else %}
</div>
{% empty %}
<div class="alert alert-info" role="alert">
<h4 class="alert-heading">No Reports Found</h4>
Reports should be saved to <code>{{ settings.REPORTS_ROOT }}</code>.
<hr/>
<small>This path can be changed by setting <code>REPORTS_ROOT</code> in NetBox's configuration.</small>
</div>
{% endif %}
{% endfor %}
</div>
{% endblock content-wrapper %}

2
netbox/templates/extras/script.html
View File

@ -11,7 +11,7 @@
{% block breadcrumbs %}
<li class="breadcrumb-item"><a href="{% url 'extras:script_list' %}">Scripts</a></li>
<li class="breadcrumb-item"><a href="{% url 'extras:script_list' %}#module.{{ module }}">{{ module|bettertitle }}</a></li>
<li class="breadcrumb-item"><a href="{% url 'extras:script_list' %}#module{{ module.pk }}">{{ module|bettertitle }}</a></li>
{% endblock breadcrumbs %}
{% block subtitle %}

86
netbox/templates/extras/script_list.html
View File

@ -1,8 +1,18 @@
{% extends 'base/layout.html' %}
{% load buttons %}
{% load helpers %}
{% block title %}Scripts{% endblock %}
{% block controls %}
<div class="controls">
<div class="control-group">
{% block extra_controls %}{% endblock %}
{% add_button model %}
</div>
</div>
{% endblock controls %}
{% block tabs %}
<ul class="nav nav-tabs px-3">
<li class="nav-item" role="presentation">
@ -13,56 +23,64 @@
{% block content-wrapper %}
<div class="tab-content">
{% if scripts %}
{% for module, module_scripts in scripts.items %}
<div class="card">
<h5 class="card-header">
<a name="module.{{ module }}"></a>
<i class="mdi mdi-file-document-outline"></i> {{ module|bettertitle }}
</h5>
<div class="card-body">
<table class="table table-hover table-headings reports">
<thead>
<tr>
<th width="250">Name</th>
<th width="110">Status</th>
<th>Description</th>
<th class="text-end">Last Run</th>
</tr>
</thead>
<tbody>
{% for class_name, script in module_scripts.items %}
{% for module in script_modules %}
<div class="card">
<h5 class="card-header" id="module{{ module.pk }}">
{% if perms.extras.delete_scriptmodule %}
<div class="float-end">
<a href="{% url 'extras:scriptmodule_delete' pk=module.pk %}" class="btn btn-danger btn-sm">
<i class="mdi mdi-trash-can-outline" aria-hidden="true"></i> Delete
</a>
</div>
{% endif %}
<i class="mdi mdi-file-document-outline"></i> {{ module.name|bettertitle }}
</h5>
<div class="card-body">
{% include 'inc/sync_warning.html' with object=module %}
<table class="table table-hover table-headings reports">
<thead>
<tr>
<th width="250">Name</th>
<th>Description</th>
<th>Last Run</th>
<th class="text-end">Status</th>
</tr>
</thead>
<tbody>
{% for script_name, script_class in module.scripts.items %}
{% with last_result=job_results|get_key:script_class.full_name %}
<tr>
<td>
<a href="{% url 'extras:script' module=script.root_module name=class_name %}" name="script.{{ class_name }}">{{ script.name }}</a>
<a href="{% url 'extras:script' module=module.path name=script_name %}" name="script.{{ script_name }}">{{ script_class.name }}</a>
</td>
<td>
{% include 'extras/inc/job_label.html' with result=script.result %}
{{ script_class.Meta.description|markdown|placeholder }}
</td>
<td>
{{ script.Meta.description|markdown|placeholder }}
</td>
{% if script.result %}
{% if last_result %}
<td>
<a href="{% url 'extras:script_result' job_result_pk=last_result.pk %}">{{ last_result.created|annotated_date }}</a>
</td>
<td class="text-end">
<a href="{% url 'extras:script_result' job_result_pk=script.result.pk %}">{{ script.result.created|annotated_date }}</a>
{% badge last_result.get_status_display last_result.get_status_color %}
</td>
{% else %}
<td class="text-end text-muted">Never</td>
<td class="text-muted">Never</td>
<td class="text-end">{{ ''|placeholder }}</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endwith %}
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
{% else %}
</div>
{% empty %}
<div class="alert alert-info">
<h4 class="alert-heading">No Scripts Found</h4>
Scripts should be saved to <code>{{ settings.SCRIPTS_ROOT }}</code>.
<hr/>
This path can be changed by setting <code>SCRIPTS_ROOT</code> in NetBox's configuration.
</div>
{% endif %}
{% endfor %}
</div>
{% endblock content-wrapper %}