Merge pull request #3078 from digitalocean/3077-nested-api-writes

Enable dictionary specification of related objects in API

CHANGELOG.md

@@ -85,6 +85,36 @@ REDIS = {
 }
 ```
 
+### API Support for Specifying Related Objects by Attributes([#3077](https://github.com/digitalocean/netbox/issues/3077))
+
+Previously, referencing a related object in an API request required knowing the primary key (integer ID) of that object.
+For example, when creating a new device, its rack would be specified as an integer:
+
+```
+{
+    "name": "MyNewDevice",
+    "rack": 123,
+    ...
+}
+```
+
+The NetBox API now supports referencing related objects by a set of sufficiently unique attrbiutes:
+
+```
+{
+    "name": "MyNewDevice",
+    "rack": {
+        "site": {
+            "name": "Equinix DC6"
+        },
+        "name": "R204"
+    },
+    ...
+}
+```
+
+Note that if the provided parameters do not return exactly one object, a validation error is raised.
+
 ### API Device/VM Config Context Included by Default ([#2350](https://github.com/digitalocean/netbox/issues/2350))
 
 The rendered config context for devices and VMs is now included by default in all API results (list and detail views).
@@ -115,6 +145,7 @@ functionality provided by the front end UI.
 
 * New API endpoints for power modeling: `/api/dcim/power-panels` and `/api/dcim/power-feeds/`
 * New API endpoint for custom field choices: `/api/extras/_custom_field_choices/`
+* ForeignKey fields now accept either the related object PK or a dictionary of attributes describing the related object.
 * Organizational objects now include child object counts. For example, the Role serializer includes `prefix_count` and `vlan_count`.
 * Added a `description` field for all device components.
 * dcim.Device: The devices list endpoint now includes rendered context data.

docs/api/overview.md

@@ -104,24 +104,37 @@ The base serializer is used to represent the default view of a model. This inclu
 }
 ```
 
-Related objects (e.g. `ForeignKey` fields) are represented using a nested serializer. A nested serializer provides a minimal representation of an object, including only its URL and enough information to construct its name. When performing write api actions (`POST`, `PUT`, and `PATCH`), any `ForeignKey` relationships do not use the nested serializer, instead you will pass just the integer ID of the related model.
+## Related Objects
 
-When a base serializer includes one or more nested serializers, the hierarchical structure precludes it from being used for write operations. Thus, a flat representation of an object may be provided using a writable serializer. This serializer includes only raw database values and is not typically used for retrieval, except as part of the response to the creation or updating of an object.
+Related objects (e.g. `ForeignKey` fields) are represented using a nested serializer. A nested serializer provides a minimal representation of an object, including only its URL and enough information to display the object to a user. When performing write API actions (`POST`, `PUT`, and `PATCH`), related objects may be specified by either numeric ID (primary key), or by a set of attributes sufficiently unique to return the desired object.
+
+For example, when creating a new device, its rack can be specified by NetBox ID (PK):
 
 ```
 {
-    "id": 1201,
+    "name": "MyNewDevice",
-    "site": 7,
+    "rack": 123,
-    "group": 4,
+    ...
-    "vid": 102,
-    "name": "Users-Floor2",
-    "tenant": null,
-    "status": 1,
-    "role": 9,
-    "description": ""
 }
 ```
 
+Or by a set of nested attributes used to identify the rack:
+
+```
+{
+    "name": "MyNewDevice",
+    "rack": {
+        "site": {
+            "name": "Equinix DC6"
+        },
+        "name": "R204"
+    },
+    ...
+}
+```
+
+Note that if the provided parameters do not return exactly one object, a validation error is raised.
+
 ## Brief Format
 
 Most API endpoints support an optional "brief" format, which returns only a minimal representation of each object in the response. This is useful when you need only a list of the objects themselves without any related data, such as when populating a drop-down list in a form.

netbox/utilities/api.py

@@ -3,7 +3,7 @@ from collections import OrderedDict
 import pytz
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import FieldError, MultipleObjectsReturned, ObjectDoesNotExist
 from django.db.models import ManyToManyField
 from django.http import Http404
 from django.utils.decorators import method_decorator
@@ -15,7 +15,7 @@ from rest_framework.response import Response
 from rest_framework.serializers import Field, ModelSerializer, ValidationError
 from rest_framework.viewsets import ModelViewSet as _ModelViewSet, ViewSet
 
-from .utils import dynamic_import
+from .utils import dict_to_filter_params, dynamic_import
 
 
 class ServiceUnavailable(APIException):
@@ -202,15 +202,48 @@ class WritableNestedSerializer(ModelSerializer):
     """
     Returns a nested representation of an object on read, but accepts only a primary key on write.
     """
+
     def to_internal_value(self, data):
+
         if data is None:
             return None
+
+        # Dictionary of related object attributes
+        if isinstance(data, dict):
+            params = dict_to_filter_params(data)
+            try:
+                return self.Meta.model.objects.get(**params)
+            except ObjectDoesNotExist:
+                raise ValidationError(
+                    "Related object not found using the provided attributes: {}".format(params)
+                )
+            except MultipleObjectsReturned:
+                raise ValidationError(
+                    "Multiple objects match the provided attributes: {}".format(params)
+                )
+            except FieldError as e:
+                raise ValidationError(e)
+
+        # Integer PK of related object
+        if isinstance(data, int):
+            pk = data
+        else:
+            try:
+                # PK might have been mistakenly passed as a string
+                pk = int(data)
+            except (TypeError, ValueError):
+                raise ValidationError(
+                    "Related objects must be referenced by numeric ID or by dictionary of attributes. Received an "
+                    "unrecognized value: {}".format(data)
+                )
+
+        # Look up object by PK
         try:
             return self.Meta.model.objects.get(pk=int(data))
-        except (TypeError, ValueError):
-            raise ValidationError("Primary key must be an integer")
         except ObjectDoesNotExist:
-            raise ValidationError("Invalid ID")
+            raise ValidationError(
+                "Related object not found using the provided numeric ID: {}".format(pk)
+            )
 
 
 #

netbox/utilities/tests/test_api.py

@@ -0,0 +1,119 @@
+from django.urls import reverse
+from rest_framework import status
+
+from dcim.models import Region, Site
+from ipam.models import VLAN
+from utilities.testing import APITestCase
+
+
+class WritableNestedSerializerTest(APITestCase):
+    """
+    Test the operation of WritableNestedSerializer using VLANSerializer as our test subject.
+    """
+
+    def setUp(self):
+
+        super().setUp()
+
+        self.region_a = Region.objects.create(name='Region A', slug='region-a')
+        self.site1 = Site.objects.create(region=self.region_a, name='Site 1', slug='site-1')
+        self.site2 = Site.objects.create(region=self.region_a, name='Site 2', slug='site-2')
+
+    def test_related_by_pk(self):
+
+        data = {
+            'vid': 100,
+            'name': 'Test VLAN 100',
+            'site': self.site1.pk,
+        }
+
+        url = reverse('ipam-api:vlan-list')
+        response = self.client.post(url, data, format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_201_CREATED)
+        self.assertEqual(response.data['site']['id'], self.site1.pk)
+        vlan = VLAN.objects.get(pk=response.data['id'])
+        self.assertEqual(vlan.site, self.site1)
+
+    def test_related_by_pk_no_match(self):
+
+        data = {
+            'vid': 100,
+            'name': 'Test VLAN 100',
+            'site': 999,
+        }
+
+        url = reverse('ipam-api:vlan-list')
+        response = self.client.post(url, data, format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(VLAN.objects.count(), 0)
+        self.assertTrue(response.data['site'][0].startswith("Related object not found"))
+
+    def test_related_by_attributes(self):
+
+        data = {
+            'vid': 100,
+            'name': 'Test VLAN 100',
+            'site': {
+                'name': 'Site 1'
+            },
+        }
+
+        url = reverse('ipam-api:vlan-list')
+        response = self.client.post(url, data, format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_201_CREATED)
+        self.assertEqual(response.data['site']['id'], self.site1.pk)
+        vlan = VLAN.objects.get(pk=response.data['id'])
+        self.assertEqual(vlan.site, self.site1)
+
+    def test_related_by_attributes_no_match(self):
+
+        data = {
+            'vid': 100,
+            'name': 'Test VLAN 100',
+            'site': {
+                'name': 'Site X'
+            },
+        }
+
+        url = reverse('ipam-api:vlan-list')
+        response = self.client.post(url, data, format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(VLAN.objects.count(), 0)
+        self.assertTrue(response.data['site'][0].startswith("Related object not found"))
+
+    def test_related_by_attributes_multiple_matches(self):
+
+        data = {
+            'vid': 100,
+            'name': 'Test VLAN 100',
+            'site': {
+                'region': {
+                    "name": "Region A",
+                },
+            },
+        }
+
+        url = reverse('ipam-api:vlan-list')
+        response = self.client.post(url, data, format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(VLAN.objects.count(), 0)
+        self.assertTrue(response.data['site'][0].startswith("Multiple objects match"))
+
+    def test_related_by_invalid(self):
+
+        data = {
+            'vid': 100,
+            'name': 'Test VLAN 100',
+            'site': 'XXX',
+        }
+
+        url = reverse('ipam-api:vlan-list')
+        response = self.client.post(url, data, format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(VLAN.objects.count(), 0)

netbox/utilities/tests/test_utils.py

@@ -1,13 +1,48 @@
 from django.test import TestCase
 
-from utilities.utils import deepmerge
+from utilities.utils import deepmerge, dict_to_filter_params
+
+
+class DictToFilterParamsTest(TestCase):
+    """
+    Validate the operation of dict_to_filter_params().
+    """
+    def setUp(self):
+        return
+
+    def test_dict_to_filter_params(self):
+
+        input = {
+            'a': True,
+            'foo': {
+                'bar': 123,
+                'baz': 456,
+            },
+            'x': {
+                'y': {
+                    'z': False
+                }
+            }
+        }
+
+        output = {
+            'a': True,
+            'foo__bar': 123,
+            'foo__baz': 456,
+            'x__y__z': False,
+        }
+
+        self.assertEqual(dict_to_filter_params(input), output)
+
+        input['x']['y']['z'] = True
+
+        self.assertNotEqual(dict_to_filter_params(input), output)
 
 
 class DeepMergeTest(TestCase):
     """
     Validate the behavior of the deepmerge() utility.
     """
-
     def setUp(self):
         return
 

netbox/utilities/utils.py

@@ -85,6 +85,38 @@ def serialize_object(obj, extra=None):
     return data
 
 
+def dict_to_filter_params(d, prefix=''):
+    """
+    Translate a dictionary of attributes to a nested set of parameters suitable for QuerySet filtering. For example:
+
+        {
+            "name": "Foo",
+            "rack": {
+                "facility_id": "R101"
+            }
+        }
+
+    Becomes:
+
+        {
+            "name": "Foo",
+            "rack__facility_id": "R101"
+        }
+
+    And can be employed as filter parameters:
+
+        Device.objects.filter(**dict_to_filter(attrs_dict))
+    """
+    params = {}
+    for key, val in d.items():
+        k = prefix + key
+        if isinstance(val, dict):
+            params.update(dict_to_filter_params(val, k + '__'))
+        else:
+            params[k] = val
+    return params
+
+
 def deepmerge(original, new):
     """
     Deep merge two dictionaries (new into original) and return a new dict