mirror/netbox-community-netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2024-05-10 07:54:54 +00:00
Code Activity

Merge branch 'develop' into develop-2.8

Browse Source
This commit is contained in:
Jeremy Stretch
2020-03-03 13:20:00 -05:00
parent 28e3b7af18 74821c2c17
commit c85bcbcf31
77 changed files with 1331 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
netbox/extras/admin.py
View File

@@ -26,7 +26,7 @@ class WebhookForm(forms.ModelForm):
class Meta:
model = Webhook
exclude = []
exclude = ()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -38,13 +38,35 @@ class WebhookForm(forms.ModelForm):
@admin.register(Webhook, site=admin_site)
class WebhookAdmin(admin.ModelAdmin):
list_display = [
'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update',
'type_delete', 'ssl_verification',
'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update', 'type_delete',
'ssl_verification',
]
list_filter = [
'enabled', 'type_create', 'type_update', 'type_delete', 'obj_type',
]
form = WebhookForm
fieldsets = (
(None, {
'fields': (
'name', 'obj_type', 'enabled',
)
}),
('Events', {
'fields': (
'type_create', 'type_update', 'type_delete',
)
}),
('HTTP Request', {
'fields': (
'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret',
)
}),
('SSL', {
'fields': (
'ssl_verification', 'ca_file_path',
)
})
)
def models(self, obj):
return ', '.join([ct.name for ct in obj.obj_type.all()])

7
netbox/extras/api/views.py
View File

@@ -14,7 +14,7 @@ from extras.models import (
ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, Tag,
)
from extras.reports import get_report, get_reports
from extras.scripts import get_script, get_scripts
from extras.scripts import get_script, get_scripts, run_script
from utilities.api import FieldChoicesViewSet, IsAuthenticatedOrLoginNotRequired, ModelViewSet
from . import serializers
@@ -265,8 +265,9 @@ class ScriptViewSet(ViewSet):
input_serializer = serializers.ScriptInputSerializer(data=request.data)
if input_serializer.is_valid():
output = script.run(input_serializer.data['data'])
script.output = output
data = input_serializer.data['data']
commit = input_serializer.data['commit']
script.output, execution_time = run_script(script, data, request, commit)
output_serializer = serializers.ScriptOutputSerializer(script)
return Response(output_serializer.data)

21
netbox/extras/choices.py
View File

@@ -124,17 +124,18 @@ class TemplateLanguageChoices(ChoiceSet):
# Webhooks
#
class WebhookContentTypeChoices(ChoiceSet):
class WebhookHttpMethodChoices(ChoiceSet):
CONTENTTYPE_JSON = 'application/json'
CONTENTTYPE_FORMDATA = 'application/x-www-form-urlencoded'
METHOD_GET = 'GET'
METHOD_POST = 'POST'
METHOD_PUT = 'PUT'
METHOD_PATCH = 'PATCH'
METHOD_DELETE = 'DELETE'
CHOICES = (
(CONTENTTYPE_JSON, 'JSON'),
(CONTENTTYPE_FORMDATA, 'Form data'),
(METHOD_GET, 'GET'),
(METHOD_POST, 'POST'),
(METHOD_PUT, 'PUT'),
(METHOD_PATCH, 'PATCH'),
(METHOD_DELETE, 'DELETE'),
)
LEGACY_MAP = {
CONTENTTYPE_JSON: 1,
CONTENTTYPE_FORMDATA: 2,
}

2
netbox/extras/constants.py
View File

@@ -138,6 +138,8 @@ LOG_LEVEL_CODES = {
LOG_FAILURE: 'failure',
}
HTTP_CONTENT_TYPE_JSON = 'application/json'
# Models which support registered webhooks
WEBHOOK_MODELS = Q(
Q(app_label='circuits', model__in=[

19
netbox/extras/middleware.py
View File

@@ -5,11 +5,14 @@ from copy import deepcopy
from datetime import timedelta
from django.conf import settings
from django.contrib import messages
from django.db.models.signals import pre_delete, post_save
from django.utils import timezone
from django_prometheus.models import model_deletes, model_inserts, model_updates
from redis.exceptions import RedisError
from extras.utils import is_taggable
from utilities.api import is_api_request
from utilities.querysets import DummyQuerySet
from .choices import ObjectChangeActionChoices
from .models import ObjectChange
@@ -98,7 +101,12 @@ class ObjectChangeMiddleware(object):
if not _thread_locals.changed_objects:
return response
# Disconnect our receivers from the post_save and post_delete signals.
post_save.disconnect(handle_changed_object, dispatch_uid='handle_changed_object')
pre_delete.disconnect(handle_deleted_object, dispatch_uid='handle_deleted_object')
# Create records for any cached objects that were changed.
redis_failed = False
for instance, action in _thread_locals.changed_objects:
# Refresh cached custom field values
@@ -114,7 +122,16 @@ class ObjectChangeMiddleware(object):
objectchange.save()
# Enqueue webhooks
enqueue_webhooks(instance, request.user, request.id, action)
try:
enqueue_webhooks(instance, request.user, request.id, action)
except RedisError as e:
if not redis_failed and not is_api_request(request):
messages.error(
request,
"There was an error processing webhooks for this request. Check that the Redis service is "
"running and reachable. The full error details were: {}".format(e)
)
redis_failed = True
# Increment metric counters
if action == ObjectChangeActionChoices.ACTION_CREATE:

48
netbox/extras/migrations/0038_webhook_template_support.py Normal file
View File

@@ -0,0 +1,48 @@
import json
from django.db import migrations, models
def json_to_text(apps, schema_editor):
"""
Convert a JSON representation of HTTP headers to key-value pairs (one header per line)
"""
Webhook = apps.get_model('extras', 'Webhook')
for webhook in Webhook.objects.exclude(additional_headers=''):
data = json.loads(webhook.additional_headers)
headers = ['{}: {}'.format(k, v) for k, v in data.items()]
Webhook.objects.filter(pk=webhook.pk).update(additional_headers='\n'.join(headers))
class Migration(migrations.Migration):
dependencies = [
('extras', '0037_configcontexts_clusters'),
]
operations = [
migrations.AddField(
model_name='webhook',
name='http_method',
field=models.CharField(default='POST', max_length=30),
),
migrations.AddField(
model_name='webhook',
name='body_template',
field=models.TextField(blank=True),
),
migrations.AlterField(
model_name='webhook',
name='additional_headers',
field=models.TextField(blank=True, default=''),
preserve_default=False,
),
migrations.AlterField(
model_name='webhook',
name='http_content_type',
field=models.CharField(default='application/json', max_length=100),
),
migrations.RunPython(
code=json_to_text
),
]

71
netbox/extras/models.py
View File

@@ -1,3 +1,4 @@
import json
from collections import OrderedDict
from datetime import date
@@ -12,6 +13,7 @@ from django.http import HttpResponse
from django.template import Template, Context
from django.urls import reverse
from django.utils.text import slugify
from rest_framework.utils.encoders import JSONEncoder
from taggit.models import TagBase, GenericTaggedItemBase
from utilities.fields import ColorField
@@ -52,7 +54,6 @@ class Webhook(models.Model):
delete in NetBox. The request will contain a representation of the object, which the remote application can act on.
Each Webhook can be limited to firing only on certain actions or certain object types.
"""
obj_type = models.ManyToManyField(
to=ContentType,
related_name='webhooks',
@@ -81,17 +82,33 @@ class Webhook(models.Model):
verbose_name='URL',
help_text="A POST will be sent to this URL when the webhook is called."
)
http_content_type = models.CharField(
max_length=50,
choices=WebhookContentTypeChoices,
default=WebhookContentTypeChoices.CONTENTTYPE_JSON,
verbose_name='HTTP content type'
enabled = models.BooleanField(
default=True
)
additional_headers = JSONField(
null=True,
http_method = models.CharField(
max_length=30,
choices=WebhookHttpMethodChoices,
default=WebhookHttpMethodChoices.METHOD_POST,
verbose_name='HTTP method'
)
http_content_type = models.CharField(
max_length=100,
default=HTTP_CONTENT_TYPE_JSON,
verbose_name='HTTP content type',
help_text='The complete list of official content types is available '
'<a href="https://www.iana.org/assignments/media-types/media-types.xhtml">here</a>.'
)
additional_headers = models.TextField(
blank=True,
help_text="User supplied headers which should be added to the request in addition to the HTTP content type. "
"Headers are supplied as key/value pairs in a JSON object."
help_text="User-supplied HTTP headers to be sent with the request in addition to the HTTP content type. "
"Headers should be defined in the format <code>Name: Value</code>. Jinja2 template processing is "
"support with the same context as the request body (below)."
)
body_template = models.TextField(
blank=True,
help_text='Jinja2 template for a custom request body. If blank, a JSON object representing the change will be '
'included. Available context data includes: <code>event</code>, <code>model</code>, '
'<code>timestamp</code>, <code>username</code>, <code>request_id</code>, and <code>data</code>.'
)
secret = models.CharField(
max_length=255,
@@ -101,9 +118,6 @@ class Webhook(models.Model):
"the secret as the key. The secret is not transmitted in "
"the request."
)
enabled = models.BooleanField(
default=True
)
ssl_verification = models.BooleanField(
default=True,
verbose_name='SSL verification',
@@ -126,9 +140,6 @@ class Webhook(models.Model):
return self.name
def clean(self):
"""
Validate model
"""
if not self.type_create and not self.type_delete and not self.type_update:
raise ValidationError(
"You must select at least one type: create, update, and/or delete."
@@ -136,14 +147,30 @@ class Webhook(models.Model):
if not self.ssl_verification and self.ca_file_path:
raise ValidationError({
'ca_file_path': 'Do not specify a CA certificate file if SSL verification is dissabled.'
'ca_file_path': 'Do not specify a CA certificate file if SSL verification is disabled.'
})
# Verify that JSON data is provided as an object
if self.additional_headers and type(self.additional_headers) is not dict:
raise ValidationError({
'additional_headers': 'Header JSON data must be in object form. Example: {"X-API-KEY": "abc123"}'
})
def render_headers(self, context):
"""
Render additional_headers and return a dict of Header: Value pairs.
"""
if not self.additional_headers:
return {}
ret = {}
data = render_jinja2(self.additional_headers, context)
for line in data.splitlines():
header, value = line.split(':')
ret[header.strip()] = value.strip()
return ret
def render_body(self, context):
"""
Render the body template, if defined. Otherwise, jump the context as a JSON object.
"""
if self.body_template:
return render_jinja2(self.body_template, context)
else:
return json.dumps(context, cls=JSONEncoder)
#

21
netbox/extras/scripts.py
View File

@@ -63,10 +63,6 @@ class ScriptVariable:
self.field_attrs['widget'] = widget
self.field_attrs['required'] = required
# Initialize the list of optional validators if none have already been defined
if 'validators' not in self.field_attrs:
self.field_attrs['validators'] = []
def as_field(self):
"""
Render the variable as a Django form field.
@@ -227,14 +223,12 @@ class IPNetworkVar(ScriptVariable):
An IPv4 or IPv6 prefix.
"""
form_field = IPNetworkFormField
field_attrs = {
'validators': [prefix_validator]
}
def __init__(self, min_prefix_length=None, max_prefix_length=None, *args, **kwargs):
super().__init__(*args, **kwargs)
# Optional minimum/maximum prefix lengths
# Set prefix validator and optional minimum/maximum prefix lengths
self.field_attrs['validators'] = [prefix_validator]
if min_prefix_length is not None:
self.field_attrs['validators'].append(
MinPrefixLengthValidator(min_prefix_length)
@@ -292,7 +286,7 @@ class BaseScript:
return vars
def run(self, data):
def run(self, data, commit):
raise NotImplementedError("The script must define a run() method.")
def as_form(self, data=None, files=None, initial=None):
@@ -389,10 +383,17 @@ def run_script(script, data, request, commit=True):
# Add the current request as a property of the script
script.request = request
# Determine whether the script accepts a 'commit' argument (this was introduced in v2.7.8)
kwargs = {
'data': data
}
if 'commit' in inspect.signature(script.run).parameters:
kwargs['commit'] = commit
try:
with transaction.atomic():
start_time = time.time()
output = script.run(data)
output = script.run(**kwargs)
end_time = time.time()
if not commit:
raise AbortTransaction()

2
netbox/extras/tests/test_api.py
View File

@@ -582,7 +582,7 @@ class ScriptTest(APITestCase):
var2 = IntegerVar()
var3 = BooleanVar()
def run(self, data):
def run(self, data, commit=True):
self.log_info(data['var1'])
self.log_success(data['var2'])

2
netbox/extras/tests/test_webhooks.py
View File

@@ -34,7 +34,7 @@ class WebhookTest(APITestCase):
DUMMY_SECRET = "LOOKATMEIMASECRETSTRING"
webhooks = Webhook.objects.bulk_create((
Webhook(name='Site Create Webhook', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers={'X-Foo': 'Bar'}),
Webhook(name='Site Create Webhook', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'),
Webhook(name='Site Update Webhook', type_update=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET),
Webhook(name='Site Delete Webhook', type_delete=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET),
))

1
netbox/extras/webhooks.py
View File

@@ -1,4 +1,3 @@
import datetime
import hashlib
import hmac

55
netbox/extras/webhooks_worker.py
View File

@@ -1,19 +1,21 @@
import json
import logging
import requests
from django_rq import job
from rest_framework.utils.encoders import JSONEncoder
from jinja2.exceptions import TemplateError
from .choices import ObjectChangeActionChoices, WebhookContentTypeChoices
from .choices import ObjectChangeActionChoices
from .webhooks import generate_signature
logger = logging.getLogger('netbox.webhooks_worker')
@job('default')
def process_webhook(webhook, data, model_name, event, timestamp, username, request_id):
"""
Make a POST request to the defined Webhook
"""
payload = {
context = {
'event': dict(ObjectChangeActionChoices)[event].lower(),
'timestamp': timestamp,
'model': model_name,
@@ -21,29 +23,48 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque
'request_id': request_id,
'data': data
}
# Build the headers for the HTTP request
headers = {
'Content-Type': webhook.http_content_type,
}
if webhook.additional_headers:
headers.update(webhook.additional_headers)
try:
headers.update(webhook.render_headers(context))
except (TemplateError, ValueError) as e:
logger.error("Error parsing HTTP headers for webhook {}: {}".format(webhook, e))
raise e
# Render the request body
try:
body = webhook.render_body(context)
except TemplateError as e:
logger.error("Error rendering request body for webhook {}: {}".format(webhook, e))
raise e
# Prepare the HTTP request
params = {
'method': 'POST',
'method': webhook.http_method,
'url': webhook.payload_url,
'headers': headers
'headers': headers,
'data': body,
}
logger.info(
"Sending {} request to {} ({} {})".format(
params['method'], params['url'], context['model'], context['event']
)
)
logger.debug(params)
try:
prepared_request = requests.Request(**params).prepare()
except requests.exceptions.RequestException as e:
logger.error("Error forming HTTP request: {}".format(e))
raise e
if webhook.http_content_type == WebhookContentTypeChoices.CONTENTTYPE_JSON:
params.update({'data': json.dumps(payload, cls=JSONEncoder)})
elif webhook.http_content_type == WebhookContentTypeChoices.CONTENTTYPE_FORMDATA:
params.update({'data': payload})
prepared_request = requests.Request(**params).prepare()
# If a secret key is defined, sign the request with a hash of the key and its content
if webhook.secret != '':
# Sign the request with a hash of the secret key and its content.
prepared_request.headers['X-Hook-Signature'] = generate_signature(prepared_request.body, webhook.secret)
# Send the request
with requests.Session() as session:
session.verify = webhook.ssl_verification
if webhook.ca_file_path:
@@ -51,8 +72,10 @@ def process_webhook(webhook, data, model_name, event, timestamp, username, reque
response = session.send(prepared_request)
if 200 <= response.status_code <= 299:
logger.info("Request succeeded; response status {}".format(response.status_code))
return 'Status {} returned, webhook successfully processed.'.format(response.status_code)
else:
logger.warning("Request failed; response status {}: {}".format(response.status_code, response.content))
raise requests.exceptions.RequestException(
"Status {} returned with content '{}', webhook FAILED to process.".format(
response.status_code, response.content