Closes #1638: Migrate all primary keys to 64-bit integers

docs/release-notes/index.md

@@ -1 +1 @@
-version-2.10.md
+version-2.11.md

docs/release-notes/version-2.11.md

@@ -0,0 +1,9 @@
+# NetBox v2.11
+
+## v2.11-beta1 (FUTURE)
+
+**WARNING:** This is a beta release and is not suitable for production use. It is intended for development and evaluation purposes only. No upgrade path to the final v2.11 release will be provided from this beta, and users should assume that all data entered into the application will be lost.
+
+### Other Changes
+
+* [#1638](https://github.com/netbox-community/netbox/issues/1638) - Migrate all primary keys to 64-bit integers

netbox/circuits/migrations/0025_standardize_models.py

@@ -1,4 +1,5 @@
-import django.core.serializers.json
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -9,11 +10,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='circuittype',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
         migrations.AlterField(
             model_name='circuit',
             name='id',

netbox/dcim/migrations/0123_standardize_models.py

@@ -1,4 +1,5 @@
-import django.core.serializers.json
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -9,291 +10,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='consoleport',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='consoleport',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='consoleport',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='consoleporttemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='consoleporttemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='consoleporttemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='consoleserverport',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='consoleserverport',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='consoleserverport',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='consoleserverporttemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='consoleserverporttemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='consoleserverporttemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='devicebay',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='devicebay',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='devicebay',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='devicebaytemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='devicebaytemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='devicebaytemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='devicerole',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='frontport',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='frontport',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='frontport',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='frontporttemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='frontporttemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='frontporttemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='interface',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='interface',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='interface',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='interfacetemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='interfacetemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='interfacetemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='inventoryitem',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='inventoryitem',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='inventoryitem',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='manufacturer',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='platform',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='poweroutlet',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='poweroutlet',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='poweroutlet',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='poweroutlettemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='poweroutlettemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='poweroutlettemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='powerport',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='powerport',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='powerport',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='powerporttemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='powerporttemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='powerporttemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='rackgroup',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='rackrole',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='rearport',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='rearport',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='rearport',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='rearporttemplate',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='rearporttemplate',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='rearporttemplate',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='region',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
         migrations.AlterField(
             model_name='cable',
             name='id',

netbox/dcim/models/device_component_templates.py

@@ -6,7 +6,7 @@ from dcim.choices import *
 from dcim.constants import *
 from extras.models import ObjectChange
 from extras.utils import extras_features
-from netbox.models import PrimaryModel
+from netbox.models import BigIDModel
 from utilities.fields import NaturalOrderingField
 from utilities.querysets import RestrictedQuerySet
 from utilities.ordering import naturalize_interface
@@ -28,7 +28,7 @@ __all__ = (
 )
 
 
-class ComponentTemplateModel(PrimaryModel):
+class ComponentTemplateModel(BigIDModel):
     device_type = models.ForeignKey(
         to='dcim.DeviceType',
         on_delete=models.CASCADE,

netbox/dcim/models/device_components.py

@@ -13,7 +13,7 @@ from dcim.constants import *
 from dcim.fields import MACAddressField
 from extras.models import ObjectChange, TaggedItem
 from extras.utils import extras_features
-from netbox.models import PrimaryModel
+from netbox.models import BigIDModel
 from utilities.fields import NaturalOrderingField
 from utilities.mptt import TreeManager
 from utilities.ordering import naturalize_interface
@@ -38,7 +38,7 @@ __all__ = (
 )
 
 
-class ComponentModel(PrimaryModel):
+class ComponentModel(BigIDModel):
     """
     An abstract model inherited by any model which has a parent Device.
     """

netbox/extras/migrations/0054_standardize_models.py

@@ -1,4 +1,5 @@
-import django.core.serializers.json
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -9,11 +10,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='configcontext',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
         migrations.AlterField(
             model_name='configcontext',
             name='id',

netbox/extras/models/models.py

@@ -16,7 +16,7 @@ from extras.choices import *
 from extras.constants import *
 from extras.querysets import ConfigContextQuerySet
 from extras.utils import extras_features, FeatureQuery, image_upload
-from netbox.models import BigIDModel, PrimaryModel
+from netbox.models import BigIDModel, ChangeLoggingMixin
 from utilities.querysets import RestrictedQuerySet
 from utilities.utils import deepmerge, render_jinja2
 
@@ -361,7 +361,7 @@ class ImageAttachment(BigIDModel):
 # Config contexts
 #
 
-class ConfigContext(PrimaryModel):
+class ConfigContext(ChangeLoggingMixin, BigIDModel):
     """
     A ConfigContext represents a set of arbitrary data available to any Device or VirtualMachine matching its assigned
     qualifiers (region, site, etc.). For example, the data stored in a ConfigContext assigned to site A and tenant B

netbox/extras/models/tags.py

@@ -2,7 +2,7 @@ from django.db import models
 from django.utils.text import slugify
 from taggit.models import TagBase, GenericTaggedItemBase
 
-from netbox.models import BigIDModel, CoreModel
+from netbox.models import BigIDModel, ChangeLoggingMixin
 from utilities.choices import ColorChoices
 from utilities.fields import ColorField
 from utilities.querysets import RestrictedQuerySet
@@ -12,7 +12,7 @@ from utilities.querysets import RestrictedQuerySet
 # Tags
 #
 
-class Tag(TagBase, CoreModel):
+class Tag(ChangeLoggingMixin, BigIDModel, TagBase):
     color = ColorField(
         default=ColorChoices.COLOR_GREY
     )

netbox/ipam/migrations/0044_standardize_models.py

@@ -1,4 +1,5 @@
-import django.core.serializers.json
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -9,21 +10,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='rir',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='role',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='vlangroup',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
         migrations.AlterField(
             model_name='aggregate',
             name='id',

netbox/netbox/models.py

@@ -16,23 +16,13 @@ __all__ = (
 )
 
 
-class BigIDModel(models.Model):
+#
-    """
+# Mixins
-    Abstract base model for all Schematic data objects. Ensures the use of a 64-bit PK.
+#
-    """
-    id = models.BigAutoField(
-        primary_key=True
-    )
 
-    class Meta:
+class ChangeLoggingMixin(models.Model):
-        abstract = True
-
-
-class CoreModel(BigIDModel):
     """
-    Base class for all core objects. Provides the following:
+    Provides change logging support.
-    - Change logging
-    - Custom field support
     """
     created = models.DateField(
         auto_now_add=True,
@@ -62,9 +52,9 @@ class CoreModel(BigIDModel):
         )
 
 
-class PrimaryModel(CoreModel):
+class CustomFieldsMixin(models.Model):
     """
-    Primary models represent real objects within the infrastructure being modeled.
+    Provides support for custom fields.
     """
     custom_field_data = models.JSONField(
         encoder=DjangoJSONEncoder,
@@ -114,7 +104,33 @@ class PrimaryModel(CoreModel):
                 raise ValidationError(f"Missing required custom field '{cf.name}'.")
 
 
-class NestedGroupModel(PrimaryModel, MPTTModel):
+#
+# Base model classes
+
+class BigIDModel(models.Model):
+    """
+    Abstract base model for all data objects. Ensures the use of a 64-bit PK.
+    """
+    id = models.BigAutoField(
+        primary_key=True
+    )
+
+    class Meta:
+        abstract = True
+
+
+class PrimaryModel(ChangeLoggingMixin, CustomFieldsMixin, BigIDModel):
+    """
+    Primary models represent real objects within the infrastructure being modeled.
+    """
+    # TODO
+    # tags = TaggableManager(through=TaggedItem)
+
+    class Meta:
+        abstract = True
+
+
+class NestedGroupModel(ChangeLoggingMixin, BigIDModel, MPTTModel):
     """
     Base model for objects which are used to form a hierarchy (regions, locations, etc.). These models nest
     recursively using MPTT. Within each parent, each child instance must have a unique name.
@@ -157,7 +173,7 @@ class NestedGroupModel(PrimaryModel, MPTTModel):
         )
 
 
-class OrganizationalModel(PrimaryModel):
+class OrganizationalModel(ChangeLoggingMixin, BigIDModel):
     """
     Organizational models are those which are used solely to categorize and qualify other objects, and do not convey
     any real information about the infrastructure being modeled (for example, functional device roles). Organizational

netbox/secrets/migrations/0013_standardize_models.py

@@ -1,4 +1,5 @@
-import django.core.serializers.json
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -9,11 +10,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='secretrole',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
         migrations.AlterField(
             model_name='secret',
             name='id',

netbox/tenancy/migrations/0012_standardize_models.py

@@ -1,4 +1,5 @@
-import django.core.serializers.json
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -9,11 +10,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='tenantgroup',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
         migrations.AlterField(
             model_name='tenant',
             name='id',

netbox/users/migrations/0011_standardize_models.py

@@ -1,3 +1,5 @@
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -18,9 +20,4 @@ class Migration(migrations.Migration):
             name='id',
             field=models.BigAutoField(primary_key=True, serialize=False),
         ),
-        migrations.AlterField(
-            model_name='userconfig',
-            name='id',
-            field=models.BigAutoField(primary_key=True, serialize=False),
-        ),
     ]

netbox/users/models.py

@@ -51,7 +51,7 @@ class AdminUser(User):
 # User preferences
 #
 
-class UserConfig(BigIDModel):
+class UserConfig(models.Model):
     """
     This model stores arbitrary user-specific preferences in a JSON data structure.
     """

netbox/virtualization/migrations/0020_standardize_models.py

@@ -1,4 +1,5 @@
-import django.core.serializers.json
+# Generated by Django 3.2b1 on 2021-02-26 21:11
+
 from django.db import migrations, models
 
 
@@ -9,31 +10,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='clustergroup',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='clustertype',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='vminterface',
-            name='created',
-            field=models.DateField(auto_now_add=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='vminterface',
-            name='custom_field_data',
-            field=models.JSONField(blank=True, default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder),
-        ),
-        migrations.AddField(
-            model_name='vminterface',
-            name='last_updated',
-            field=models.DateTimeField(auto_now=True, null=True),
-        ),
         migrations.AlterField(
             model_name='cluster',
             name='id',

netbox/virtualization/models.py

@@ -9,7 +9,7 @@ from dcim.models import BaseInterface, Device
 from extras.models import ConfigContextModel, ObjectChange, TaggedItem
 from extras.querysets import ConfigContextModelQuerySet
 from extras.utils import extras_features
-from netbox.models import OrganizationalModel, PrimaryModel
+from netbox.models import BigIDModel, OrganizationalModel, PrimaryModel
 from utilities.fields import NaturalOrderingField
 from utilities.ordering import naturalize_interface
 from utilities.query_functions import CollateAsChar
@@ -374,7 +374,7 @@ class VirtualMachine(PrimaryModel, ConfigContextModel):
 #
 
 @extras_features('export_templates', 'webhooks')
-class VMInterface(PrimaryModel, BaseInterface):
+class VMInterface(BigIDModel, BaseInterface):
     virtual_machine = models.ForeignKey(
         to='virtualization.VirtualMachine',
         on_delete=models.CASCADE,