From de7207de558d27934d3e46910da07d42278936f5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 16 Apr 2019 18:02:52 -0400 Subject: [PATCH 1/3] Enable dictionary specification of related objects in API --- CHANGELOG.md | 31 ++++++++++++++++++++++++++++ docs/api/overview.md | 35 +++++++++++++++++++++---------- netbox/utilities/api.py | 43 ++++++++++++++++++++++++++++++++++----- netbox/utilities/utils.py | 32 +++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7c43aa73..64b624f0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,36 @@ REDIS = { } ``` +### API Support for Specifying Related Objects by Attributes([#3077](https://github.com/digitalocean/netbox/issues/3077)) + +Previously, referencing a related object in an API request required knowing the primary key (integer ID) of that object. +For example, when creating a new device, its rack would be specified as an integer: + +``` +{ + "name": "MyNewDevice", + "rack": 123, + ... +} +``` + +The NetBox API now supports referencing related objects by a set of sufficiently unique attrbiutes: + +``` +{ + "name": "MyNewDevice", + "rack": { + "site": { + "name": "Equinix DC6" + }, + "name": "R204" + }, + ... +} +``` + +Note that if the provided parameters do not return exactly one object, a validation error is raised. + ### API Device/VM Config Context Included by Default ([#2350](https://github.com/digitalocean/netbox/issues/2350)) The rendered Config Context for Devices and VMs is now included by default in all API results (list and detail views). @@ -112,6 +142,7 @@ to now use "Extras | Tag." ## API Changes +* ForeignKey fields now accept either the related object PK or a dictionary of attributes describing the related object. * dcim.Interface: `form_factor` has been renamed to `type`. Backward-compatibile support for `form_factor` will be maintained until NetBox v2.7. * dcim.Interface: The `type` filter has been renamed to `kind`. * dcim.DeviceType: `instance_count` has been renamed to `device_count`. diff --git a/docs/api/overview.md b/docs/api/overview.md index 00ff9c27e..6b9a1a429 100644 --- a/docs/api/overview.md +++ b/docs/api/overview.md @@ -104,24 +104,37 @@ The base serializer is used to represent the default view of a model. This inclu } ``` -Related objects (e.g. `ForeignKey` fields) are represented using a nested serializer. A nested serializer provides a minimal representation of an object, including only its URL and enough information to construct its name. When performing write api actions (`POST`, `PUT`, and `PATCH`), any `ForeignKey` relationships do not use the nested serializer, instead you will pass just the integer ID of the related model. +## Related Objects -When a base serializer includes one or more nested serializers, the hierarchical structure precludes it from being used for write operations. Thus, a flat representation of an object may be provided using a writable serializer. This serializer includes only raw database values and is not typically used for retrieval, except as part of the response to the creation or updating of an object. +Related objects (e.g. `ForeignKey` fields) are represented using a nested serializer. A nested serializer provides a minimal representation of an object, including only its URL and enough information to display the object to a user. When performing write API actions (`POST`, `PUT`, and `PATCH`), related objects may be specified by either numeric ID (primary key), or by a set of attributes sufficiently unique to return the desired object. + +For example, when creating a new device, its rack can be specified by NetBox ID (PK): ``` { - "id": 1201, - "site": 7, - "group": 4, - "vid": 102, - "name": "Users-Floor2", - "tenant": null, - "status": 1, - "role": 9, - "description": "" + "name": "MyNewDevice", + "rack": 123, + ... } ``` +Or by a set of nested attributes used to identify the rack: + +``` +{ + "name": "MyNewDevice", + "rack": { + "site": { + "name": "Equinix DC6" + }, + "name": "R204" + }, + ... +} +``` + +Note that if the provided parameters do not return exactly one object, a validation error is raised. + ## Brief Format Most API endpoints support an optional "brief" format, which returns only a minimal representation of each object in the response. This is useful when you need only a list of the objects themselves without any related data, such as when populating a drop-down list in a form. diff --git a/netbox/utilities/api.py b/netbox/utilities/api.py index 4068b7741..f49018242 100644 --- a/netbox/utilities/api.py +++ b/netbox/utilities/api.py @@ -3,7 +3,7 @@ from collections import OrderedDict import pytz from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import FieldError, MultipleObjectsReturned, ObjectDoesNotExist from django.db.models import ManyToManyField from django.http import Http404 from django.utils.decorators import method_decorator @@ -15,7 +15,7 @@ from rest_framework.response import Response from rest_framework.serializers import Field, ModelSerializer, ValidationError from rest_framework.viewsets import ModelViewSet as _ModelViewSet, ViewSet -from .utils import dynamic_import +from .utils import dict_to_filter_params, dynamic_import class ServiceUnavailable(APIException): @@ -202,15 +202,48 @@ class WritableNestedSerializer(ModelSerializer): """ Returns a nested representation of an object on read, but accepts only a primary key on write. """ + def to_internal_value(self, data): + if data is None: return None + + # Dictionary of related object attributes + if isinstance(data, dict): + params = dict_to_filter_params(data) + try: + return self.Meta.model.objects.get(**params) + except ObjectDoesNotExist: + raise ValidationError( + "Related object not found using the provided attributes: {}".format(params) + ) + except MultipleObjectsReturned: + raise ValidationError( + "Multiple objects match the provided attributes: {}".format(params) + ) + except FieldError as e: + raise ValidationError(e) + + # Integer PK of related object + if isinstance(data, int): + pk = data + else: + try: + # PK might have been mistakenly passed as a string + pk = int(data) + except (TypeError, ValueError): + raise ValidationError( + "Related objects must be referenced by numeric ID or by dictionary of attributes. Received an " + "unrecognized value: {}".format(data) + ) + + # Look up object by PK try: return self.Meta.model.objects.get(pk=int(data)) - except (TypeError, ValueError): - raise ValidationError("Primary key must be an integer") except ObjectDoesNotExist: - raise ValidationError("Invalid ID") + raise ValidationError( + "Related object not found using the provided numeric ID: {}".format(pk) + ) # diff --git a/netbox/utilities/utils.py b/netbox/utilities/utils.py index 1d1f12ddb..c323bf473 100644 --- a/netbox/utilities/utils.py +++ b/netbox/utilities/utils.py @@ -85,6 +85,38 @@ def serialize_object(obj, extra=None): return data +def dict_to_filter_params(d, prefix=''): + """ + Translate a dictionary of attributes to a nested set of parameters suitable for QuerySet filtering. For example: + + { + "name": "Foo", + "rack": { + "facility_id": "R101" + } + } + + Becomes: + + { + "name": "Foo", + "rack__facility_id": "R101" + } + + And can be employed as filter parameters: + + Device.objects.filter(**dict_to_filter(attrs_dict)) + """ + params = {} + for key, val in d.items(): + k = prefix + key + if isinstance(val, dict): + params.update(dict_to_filter_params(val, k + '__')) + else: + params[k] = val + return params + + def deepmerge(original, new): """ Deep merge two dictionaries (new into original) and return a new dict From a10880753410abcbb788843037474a14b109a3e5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 17 Apr 2019 10:54:50 -0400 Subject: [PATCH 2/3] Add tests for WritableNestedSerializer --- netbox/utilities/tests/test_api.py | 119 +++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 netbox/utilities/tests/test_api.py diff --git a/netbox/utilities/tests/test_api.py b/netbox/utilities/tests/test_api.py new file mode 100644 index 000000000..3ff4b3876 --- /dev/null +++ b/netbox/utilities/tests/test_api.py @@ -0,0 +1,119 @@ +from django.urls import reverse +from rest_framework import status + +from dcim.models import Region, Site +from ipam.models import VLAN +from utilities.testing import APITestCase + + +class WritableNestedSerializerTest(APITestCase): + """ + Test the operation of WritableNestedSerializer using VLANSerializer as our test subject. + """ + + def setUp(self): + + super().setUp() + + self.region_a = Region.objects.create(name='Region A', slug='region-a') + self.site1 = Site.objects.create(region=self.region_a, name='Site 1', slug='site-1') + self.site2 = Site.objects.create(region=self.region_a, name='Site 2', slug='site-2') + + def test_related_by_pk(self): + + data = { + 'vid': 100, + 'name': 'Test VLAN 100', + 'site': self.site1.pk, + } + + url = reverse('ipam-api:vlan-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(response.data['site']['id'], self.site1.pk) + vlan = VLAN.objects.get(pk=response.data['id']) + self.assertEqual(vlan.site, self.site1) + + def test_related_by_pk_no_match(self): + + data = { + 'vid': 100, + 'name': 'Test VLAN 100', + 'site': 999, + } + + url = reverse('ipam-api:vlan-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST) + self.assertEqual(VLAN.objects.count(), 0) + self.assertTrue(response.data['site'][0].startswith("Related object not found")) + + def test_related_by_attributes(self): + + data = { + 'vid': 100, + 'name': 'Test VLAN 100', + 'site': { + 'name': 'Site 1' + }, + } + + url = reverse('ipam-api:vlan-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(response.data['site']['id'], self.site1.pk) + vlan = VLAN.objects.get(pk=response.data['id']) + self.assertEqual(vlan.site, self.site1) + + def test_related_by_attributes_no_match(self): + + data = { + 'vid': 100, + 'name': 'Test VLAN 100', + 'site': { + 'name': 'Site X' + }, + } + + url = reverse('ipam-api:vlan-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST) + self.assertEqual(VLAN.objects.count(), 0) + self.assertTrue(response.data['site'][0].startswith("Related object not found")) + + def test_related_by_attributes_multiple_matches(self): + + data = { + 'vid': 100, + 'name': 'Test VLAN 100', + 'site': { + 'region': { + "name": "Region A", + }, + }, + } + + url = reverse('ipam-api:vlan-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST) + self.assertEqual(VLAN.objects.count(), 0) + self.assertTrue(response.data['site'][0].startswith("Multiple objects match")) + + def test_related_by_invalid(self): + + data = { + 'vid': 100, + 'name': 'Test VLAN 100', + 'site': 'XXX', + } + + url = reverse('ipam-api:vlan-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST) + self.assertEqual(VLAN.objects.count(), 0) From fd4c5031c7afa66eb02411a180d36dbc6ee7102b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 17 Apr 2019 11:19:59 -0400 Subject: [PATCH 3/3] Add test for dict_to_filter_params --- netbox/utilities/tests/test_utils.py | 39 ++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/netbox/utilities/tests/test_utils.py b/netbox/utilities/tests/test_utils.py index 4e0fec1ba..5d9a98ad5 100644 --- a/netbox/utilities/tests/test_utils.py +++ b/netbox/utilities/tests/test_utils.py @@ -1,13 +1,48 @@ from django.test import TestCase -from utilities.utils import deepmerge +from utilities.utils import deepmerge, dict_to_filter_params + + +class DictToFilterParamsTest(TestCase): + """ + Validate the operation of dict_to_filter_params(). + """ + def setUp(self): + return + + def test_dict_to_filter_params(self): + + input = { + 'a': True, + 'foo': { + 'bar': 123, + 'baz': 456, + }, + 'x': { + 'y': { + 'z': False + } + } + } + + output = { + 'a': True, + 'foo__bar': 123, + 'foo__baz': 456, + 'x__y__z': False, + } + + self.assertEqual(dict_to_filter_params(input), output) + + input['x']['y']['z'] = True + + self.assertNotEqual(dict_to_filter_params(input), output) class DeepMergeTest(TestCase): """ Validate the behavior of the deepmerge() utility. """ - def setUp(self): return