1
0
mirror of https://github.com/netbox-community/netbox.git synced 2024-05-10 07:54:54 +00:00

Merge pull request #1544 from digitalocean/reports

Closes #1511: Implemented reports
This commit is contained in:
Jeremy Stretch
2017-09-28 17:16:10 -04:00
committed by GitHub
21 changed files with 815 additions and 8 deletions

View File

@@ -7,7 +7,7 @@ from rest_framework import serializers
from dcim.api.serializers import NestedDeviceSerializer, NestedRackSerializer, NestedSiteSerializer
from dcim.models import Device, Rack, Site
from extras.models import (
ACTION_CHOICES, ExportTemplate, Graph, GRAPH_TYPE_CHOICES, ImageAttachment, TopologyMap, UserAction,
ACTION_CHOICES, ExportTemplate, Graph, GRAPH_TYPE_CHOICES, ImageAttachment, ReportResult, TopologyMap, UserAction,
)
from users.api.serializers import NestedUserSerializer
from utilities.api import ChoiceFieldSerializer, ContentTypeFieldSerializer, ValidatedModelSerializer
@@ -127,6 +127,41 @@ class WritableImageAttachmentSerializer(ValidatedModelSerializer):
return data
#
# Reports
#
class ReportResultSerializer(serializers.ModelSerializer):
class Meta:
model = ReportResult
fields = ['created', 'user', 'failed', 'data']
class NestedReportResultSerializer(serializers.ModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='extras-api:report-detail',
lookup_field='report',
lookup_url_kwarg='pk'
)
class Meta:
model = ReportResult
fields = ['url', 'created', 'user', 'failed']
class ReportSerializer(serializers.Serializer):
module = serializers.CharField(max_length=255)
name = serializers.CharField(max_length=255)
description = serializers.CharField(max_length=255, required=False)
test_methods = serializers.ListField(child=serializers.CharField(max_length=255))
result = NestedReportResultSerializer()
class ReportDetailSerializer(ReportSerializer):
result = ReportResultSerializer()
#
# User actions
#

View File

@@ -28,6 +28,9 @@ router.register(r'topology-maps', views.TopologyMapViewSet)
# Image attachments
router.register(r'image-attachments', views.ImageAttachmentViewSet)
# Reports
router.register(r'reports', views.ReportViewSet, base_name='report')
# Recent activity
router.register(r'recent-activity', views.RecentActivityViewSet)

View File

@@ -1,14 +1,17 @@
from __future__ import unicode_literals
from rest_framework.decorators import detail_route
from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet
from rest_framework.exceptions import PermissionDenied
from rest_framework.response import Response
from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet, ViewSet
from django.contrib.contenttypes.models import ContentType
from django.http import HttpResponse
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404
from extras import filters
from extras.models import ExportTemplate, Graph, ImageAttachment, TopologyMap, UserAction
from extras.models import ExportTemplate, Graph, ImageAttachment, ReportResult, TopologyMap, UserAction
from extras.reports import get_report, get_reports
from utilities.api import WritableSerializerMixin
from . import serializers
@@ -88,6 +91,77 @@ class ImageAttachmentViewSet(WritableSerializerMixin, ModelViewSet):
write_serializer_class = serializers.WritableImageAttachmentSerializer
class ReportViewSet(ViewSet):
_ignore_model_permissions = True
exclude_from_schema = True
lookup_value_regex = '[^/]+' # Allow dots
def _retrieve_report(self, pk):
# Read the PK as "<module>.<report>"
if '.' not in pk:
raise Http404
module_name, report_name = pk.split('.', 1)
# Raise a 404 on an invalid Report module/name
report = get_report(module_name, report_name)
if report is None:
raise Http404
return report
def list(self, request):
"""
Compile all reports and their related results (if any). Result data is deferred in the list view.
"""
report_list = []
# Iterate through all available Reports.
for module_name, reports in get_reports():
for report in reports:
# Attach the relevant ReportResult (if any) to each Report.
report.result = ReportResult.objects.filter(report=report.full_name).defer('data').first()
report_list.append(report)
serializer = serializers.ReportSerializer(report_list, many=True, context={
'request': request,
})
return Response(serializer.data)
def retrieve(self, request, pk):
"""
Retrieve a single Report identified as "<module>.<report>".
"""
# Retrieve the Report and ReportResult, if any.
report = self._retrieve_report(pk)
report.result = ReportResult.objects.filter(report=report.full_name).first()
serializer = serializers.ReportDetailSerializer(report)
return Response(serializer.data)
@detail_route(methods=['post'])
def run(self, request, pk):
"""
Run a Report and create a new ReportResult, overwriting any previous result for the Report.
"""
# Check that the user has permission to run reports.
if not request.user.has_perm('extras.add_reportresult'):
raise PermissionDenied("This user does not have permission to run reports.")
# Retrieve and run the Report. This will create a new ReportResult.
report = self._retrieve_report(pk)
report.run()
serializer = serializers.ReportDetailSerializer(report)
return Response(serializer.data)
class RecentActivityViewSet(ReadOnlyModelViewSet):
"""
List all UserActions to provide a log of recent activity.

View File

@@ -62,3 +62,17 @@ ACTION_CHOICES = (
(ACTION_DELETE, 'deleted'),
(ACTION_BULK_DELETE, 'bulk deleted'),
)
# Report logging levels
LOG_DEFAULT = 0
LOG_SUCCESS = 10
LOG_INFO = 20
LOG_WARNING = 30
LOG_FAILURE = 40
LOG_LEVEL_CODES = {
LOG_DEFAULT: 'default',
LOG_SUCCESS: 'success',
LOG_INFO: 'info',
LOG_WARNING: 'warning',
LOG_FAILURE: 'failure',
}

View File

@@ -0,0 +1,48 @@
from __future__ import unicode_literals
from django.core.management.base import BaseCommand
from django.utils import timezone
from extras.models import ReportResult
from extras.reports import get_reports
class Command(BaseCommand):
help = "Run a report to validate data in NetBox"
def add_arguments(self, parser):
parser.add_argument('reports', nargs='+', help="Report(s) to run")
# parser.add_argument('--verbose', action='store_true', default=False, help="Print all logs")
def handle(self, *args, **options):
# Gather all available reports
reports = get_reports()
# Run reports
for module_name, report_list in reports:
for report in report_list:
if module_name in options['reports'] or report.full_namel in options['reports']:
# Run the report and create a new ReportResult
self.stdout.write(
"[{:%H:%M:%S}] Running {}...".format(timezone.now(), report.full_name)
)
report.run()
# Report on success/failure
status = self.style.ERROR('FAILED') if report.failed else self.style.SUCCESS('SUCCESS')
for test_name, attrs in report.result.data.items():
self.stdout.write(
"\t{}: {} success, {} info, {} warning, {} failure".format(
test_name, attrs['success'], attrs['info'], attrs['warning'], attrs['failure']
)
)
self.stdout.write(
"[{:%H:%M:%S}] {}: {}".format(timezone.now(), report.full_name, status)
)
# Wrap things up
self.stdout.write(
"[{:%H:%M:%S}] Finished".format(timezone.now())
)

View File

@@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.4 on 2017-09-26 21:25
from __future__ import unicode_literals
from django.conf import settings
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('extras', '0007_unicode_literals'),
]
operations = [
migrations.CreateModel(
name='ReportResult',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('report', models.CharField(max_length=255, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
('failed', models.BooleanField()),
('data', django.contrib.postgres.fields.jsonb.JSONField()),
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['report'],
},
),
]

View File

@@ -6,6 +6,7 @@ import graphviz
from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.contrib.postgres.fields import JSONField
from django.core.validators import ValidationError
from django.db import models
from django.db.models import Q
@@ -388,6 +389,24 @@ class ImageAttachment(models.Model):
return None
#
# Report results
#
class ReportResult(models.Model):
"""
This model stores the results from running a user-defined report.
"""
report = models.CharField(max_length=255, unique=True)
created = models.DateTimeField(auto_now_add=True)
user = models.ForeignKey(User, on_delete=models.SET_NULL, related_name='+', blank=True, null=True)
failed = models.BooleanField()
data = JSONField()
class Meta:
ordering = ['report']
#
# User actions
#

178
netbox/extras/reports.py Normal file
View File

@@ -0,0 +1,178 @@
from collections import OrderedDict
import importlib
import inspect
import pkgutil
from django.utils import timezone
from .constants import LOG_DEFAULT, LOG_FAILURE, LOG_INFO, LOG_LEVEL_CODES, LOG_SUCCESS, LOG_WARNING
from .models import ReportResult
import reports as custom_reports
def is_report(obj):
"""
Returns True if the given object is a Report.
"""
if obj in Report.__subclasses__():
return True
return False
def get_report(module_name, report_name):
"""
Return a specific report from within a module.
"""
module = importlib.import_module('reports.{}'.format(module_name))
report = getattr(module, report_name, None)
if report is None:
return None
return report()
def get_reports():
"""
Compile a list of all reports available across all modules in the reports path. Returns a list of tuples:
[
(module_name, (report, report, report, ...)),
(module_name, (report, report, report, ...)),
...
]
"""
module_list = []
# Iterate through all modules within the reports path. These are the user-defined files in which reports are
# defined.
for importer, module_name, is_pkg in pkgutil.walk_packages(custom_reports.__path__):
module = importlib.import_module('reports.{}'.format(module_name))
report_list = [cls() for _, cls in inspect.getmembers(module, is_report)]
module_list.append((module_name, report_list))
return module_list
class Report(object):
"""
NetBox users can extend this object to write custom reports to be used for validating data within NetBox. Each
report must have one or more test methods named `test_*`.
The `_results` attribute of a completed report will take the following form:
{
'test_bar': {
'failures': 42,
'log': [
(<datetime>, <level>, <object>, <message>),
...
]
},
'test_foo': {
'failures': 0,
'log': [
(<datetime>, <level>, <object>, <message>),
...
]
}
}
"""
description = None
def __init__(self):
self._results = OrderedDict()
self.active_test = None
self.failed = False
# Compile test methods and initialize results skeleton
test_methods = []
for method in dir(self):
if method.startswith('test_') and callable(getattr(self, method)):
test_methods.append(method)
self._results[method] = OrderedDict([
('success', 0),
('info', 0),
('warning', 0),
('failure', 0),
('log', []),
])
if not test_methods:
raise Exception("A report must contain at least one test method.")
self.test_methods = test_methods
@property
def module(self):
return self.__module__.rsplit('.', 1)[1]
@property
def name(self):
return self.__class__.__name__
@property
def full_name(self):
return '.'.join([self.module, self.name])
def _log(self, obj, message, level=LOG_DEFAULT):
"""
Log a message from a test method. Do not call this method directly; use one of the log_* wrappers below.
"""
if level not in LOG_LEVEL_CODES:
raise Exception("Unknown logging level: {}".format(level))
self._results[self.active_test]['log'].append((
timezone.now().isoformat(),
LOG_LEVEL_CODES.get(level),
str(obj) if obj else None,
obj.get_absolute_url() if getattr(obj, 'get_absolute_url', None) else None,
message,
))
def log(self, message):
"""
Log a message which is not associated with a particular object.
"""
self._log(None, message, level=LOG_DEFAULT)
def log_success(self, obj, message=None):
"""
Record a successful test against an object. Logging a message is optional.
"""
if message:
self._log(obj, message, level=LOG_SUCCESS)
self._results[self.active_test]['success'] += 1
def log_info(self, obj, message):
"""
Log an informational message.
"""
self._log(obj, message, level=LOG_INFO)
self._results[self.active_test]['info'] += 1
def log_warning(self, obj, message):
"""
Log a warning.
"""
self._log(obj, message, level=LOG_WARNING)
self._results[self.active_test]['warning'] += 1
def log_failure(self, obj, message):
"""
Log a failure. Calling this method will automatically mark the report as failed.
"""
self._log(obj, message, level=LOG_FAILURE)
self._results[self.active_test]['failure'] += 1
self.failed = True
def run(self):
"""
Run the report and return its results. Each test method will be executed in order.
"""
for method_name in self.test_methods:
self.active_test = method_name
test_method = getattr(self, method_name)
test_method()
# Delete any previous ReportResult and create a new one to record the result.
ReportResult.objects.filter(report=self.full_name).delete()
result = ReportResult(report=self.full_name, failed=self.failed, data=self._results)
result.save()
self.result = result

View File

@@ -12,4 +12,9 @@ urlpatterns = [
url(r'^image-attachments/(?P<pk>\d+)/edit/$', views.ImageAttachmentEditView.as_view(), name='imageattachment_edit'),
url(r'^image-attachments/(?P<pk>\d+)/delete/$', views.ImageAttachmentDeleteView.as_view(), name='imageattachment_delete'),
# Reports
url(r'^reports/$', views.ReportListView.as_view(), name='report_list'),
url(r'^reports/(?P<name>[^/]+\.[^/]+)/$', views.ReportView.as_view(), name='report'),
url(r'^reports/(?P<name>[^/]+\.[^/]+)/run/$', views.ReportRunView.as_view(), name='report_run'),
]

View File

@@ -1,13 +1,23 @@
from __future__ import unicode_literals
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.shortcuts import get_object_or_404
from django.contrib import messages
from django.http import Http404
from django.shortcuts import get_object_or_404, redirect, render
from django.utils.safestring import mark_safe
from django.views.generic import View
from utilities.forms import ConfirmationForm
from utilities.views import ObjectDeleteView, ObjectEditView
from .forms import ImageAttachmentForm
from .models import ImageAttachment
from .models import ImageAttachment, ReportResult, UserAction
from .reports import get_report, get_reports
#
# Image attachments
#
class ImageAttachmentEditView(PermissionRequiredMixin, ObjectEditView):
permission_required = 'extras.change_imageattachment'
model = ImageAttachment
@@ -30,3 +40,79 @@ class ImageAttachmentDeleteView(PermissionRequiredMixin, ObjectDeleteView):
def get_return_url(self, request, imageattachment):
return imageattachment.parent.get_absolute_url()
#
# Reports
#
class ReportListView(View):
"""
Retrieve all of the available reports from disk and the recorded ReportResult (if any) for each.
"""
def get(self, request):
reports = get_reports()
results = {r.report: r for r in ReportResult.objects.all()}
ret = []
for module, report_list in reports:
module_reports = []
for report in report_list:
report.result = results.get(report.full_name, None)
module_reports.append(report)
ret.append((module, module_reports))
return render(request, 'extras/report_list.html', {
'reports': ret,
})
class ReportView(View):
"""
Display a single Report and its associated ReportResult (if any).
"""
def get(self, request, name):
# Retrieve the Report by "<module>.<report>"
module_name, report_name = name.split('.')
report = get_report(module_name, report_name)
if report is None:
raise Http404
# Attach the ReportResult (if any)
report.result = ReportResult.objects.filter(report=report.full_name).first()
return render(request, 'extras/report.html', {
'report': report,
'run_form': ConfirmationForm(),
})
class ReportRunView(PermissionRequiredMixin, View):
"""
Run a Report and record a new ReportResult.
"""
permission_required = 'extras.add_reportresult'
def post(self, request, name):
# Retrieve the Report by "<module>.<report>"
module_name, report_name = name.split('.')
report = get_report(module_name, report_name)
if report is None:
raise Http404
form = ConfirmationForm(request.POST)
if form.is_valid():
# Run the Report. A new ReportResult is created.
report.run()
result = 'failed' if report.failed else 'passed'
msg = "Ran report {} ({})".format(report.full_name, result)
messages.success(request, mark_safe(msg))
UserAction.objects.log_create(request.user, report.result, msg)
return redirect('extras:report', name=report.full_name)

View File

@@ -339,6 +339,18 @@ table.component-list td.subtable td {
padding-top: 6px;
}
/* Reports */
table.reports td.method {
font-family: monospace;
padding-left: 30px;
}
table.reports td.stats label {
display: inline-block;
line-height: 14px;
margin-bottom: 0;
min-width: 40px;
}
/* AJAX loader */
.loading {
position: fixed;

View File

View File

@@ -28,7 +28,7 @@
<div id="navbar" class="navbar-collapse collapse">
{% if request.user.is_authenticated or not settings.LOGIN_REQUIRED %}
<ul class="nav navbar-nav">
<li class="dropdown{% if request.path|contains:'/dcim/sites/,/dcim/regions/,/tenancy/' %} active{% endif %}">
<li class="dropdown{% if request.path|contains:'/dcim/sites/,/dcim/regions/,/tenancy/,/extras/reports/' %} active{% endif %}">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Organization <span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{% url 'dcim:site_list' %}"><strong>Sites</strong></a></li>
@@ -52,6 +52,8 @@
{% if perms.tenancy.add_tenantgroup %}
<li class="subnav"><a href="{% url 'tenancy:tenantgroup_add' %}"><i class="fa fa-plus"></i> Add a Tenant Group</a></li>
{% endif %}
<li class="divider"></li>
<li><a href="{% url 'extras:report_list' %}"><strong>Reports</strong></a></li>
</ul>
</li>
<li class="dropdown{% if request.path|contains:'/dcim/rack' %} active{% endif %}">

View File

@@ -0,0 +1,7 @@
{% if report.result.failed %}
<label class="label label-danger">Failed</label>
{% elif report.result %}
<label class="label label-success">Passed</label>
{% else %}
<label class="label label-default">N/A</label>
{% endif %}

View File

@@ -0,0 +1,92 @@
{% extends '_base.html' %}
{% load helpers %}
{% block title %}{{ report.name }}{% endblock %}
{% block content %}
<div class="row">
<div class="col-md-12">
<ol class="breadcrumb">
<li><a href="{% url 'extras:report_list' %}">Reports</a></li>
<li><a href="{% url 'extras:report_list' %}#module.{{ report.module }}">{{ report.module|bettertitle }}</a></li>
<li>{{ report.name }}</li>
</ol>
</div>
</div>
{% if perms.extras.add_reportresult %}
<div class="pull-right">
<form action="{% url 'extras:report_run' name=report.full_name %}" method="post">
{% csrf_token %}
{{ run_form }}
<button type="submit" name="_run" class="btn btn-primary"><i class="fa fa-play"></i> Run Report</button>
</form>
</div>
{% endif %}
<h1>{{ report.name }}{% include 'extras/inc/report_label.html' %}</h1>
<div class="row">
<div class="col-md-12">
{% if report.description %}
<p class="lead">{{ report.description }}</p>
{% endif %}
{% if report.result %}
<p>Last run: {{ report.result.created }}</p>
{% else %}
<p class="text-muted">Last run: Never</p>
{% endif %}
</div>
<div class="col-md-9">
{% if report.result %}
<table class="table table-hover">
<thead>
<tr>
<th>Time</th>
<th>Level</th>
<th>Object</th>
<th>Message</th>
</tr>
</thead>
{% for method, data in report.result.data.items %}
<tr>
<th colspan="4"><a name="{{ method }}"></a>{{ method }}</th>
</tr>
{% for time, level, obj, url, message in data.log %}
<tr class="{% if level == 'failure' %}danger{% elif level %}{{ level }}{% endif %}">
<td>{{ time }}</td>
<td>
<label class="label label-{% if level == 'failure' %}danger{% else %}{{ level }}{% endif %}">{{ level|title }}</label>
</td>
<td>
{% if obj and url %}
<a href="{{ url }}">{{ obj }}</a>
{% elif obj %}
{{ obj }}
{% endif %}
</td>
<td>{{ message }}</td>
</tr>
{% endfor %}
{% endfor %}
</table>
{% else %}
<div class="well">No results are available for this report. Please run the report first.</div>
{% endif %}
</div>
<div class="col-md-3">
{% if report.result %}
<div class="panel panel-default">
<div class="panel-heading">
<strong>Methods</strong>
</div>
<ul class="list-group">
{% for method, data in report.result.data.items %}
<li class="list-group-item">
<a href="#{{ method }}">{{ method }}</a>
<span class="badge">{{ data.log|length }}</span>
</li>
{% endfor %}
</ul>
</div>
{% endif %}
</div>
</div>
{% endblock %}

View File

@@ -0,0 +1,73 @@
{% extends '_base.html' %}
{% load helpers %}
{% block content %}
<h1>{% block title %}Reports{% endblock %}</h1>
<div class="row">
<div class="col-md-9">
{% for module, module_reports in reports %}
<h3><a name="module.{{ module }}"></a>{{ module|bettertitle }}</h3>
<table class="table table-hover table-headings reports">
<thead>
<tr>
<th>Name</th>
<th>Status</th>
<th>Description</th>
<th class="text-right">Last Run</th>
</tr>
</thead>
<tbody>
{% for report in module_reports %}
<tr>
<td>
<a href="{% url 'extras:report' name=report.full_name %}" name="report.{{ report.name }}"><strong>{{ report.name }}</strong></a>
</td>
<td>
{% include 'extras/inc/report_label.html' %}
</td>
<td>{{ report.description|default:"" }}</td>
{% if report.result %}
<td class="text-right">{{ report.result.created }}</td>
{% else %}
<td class="text-right text-muted">Never</td>
{% endif %}
</tr>
{% for method, stats in report.result.data.items %}
<tr>
<td colspan="3" class="method">
{{ method }}
</td>
<td class="text-right stats">
<label class="label label-success">{{ stats.success }}</label>
<label class="label label-info">{{ stats.info }}</label>
<label class="label label-warning">{{ stats.warning }}</label>
<label class="label label-danger">{{ stats.failure }}</label>
</td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
{% endfor %}
</div>
<div class="col-md-3">
<div class="panel panel-default">
{% for module, module_reports in reports %}
<div class="panel-heading">
<strong>{{ module|bettertitle }}</strong>
</div>
<ul class="list-group">
{% for report in module_reports %}
<a href="#report.{{ report.name }}" class="list-group-item">
<i class="fa fa-list-alt"></i> {{ report.name }}
<div class="pull-right">
{% include 'extras/inc/report_label.html' %}
</div>
</a>
{% endfor %}
</ul>
{% endfor %}
</div>
</div>
</div>
{% endblock %}