1
0
mirror of https://github.com/netbox-community/netbox.git synced 2024-05-10 07:54:54 +00:00

Closes #13381: Enable plugins to register custom data backends (#14095)

* Initial work on #13381

* Fix backend type display in table column

* Fix data source type choices during bulk edit

* Misc cleanup

* Move backend utils from core app to netbox

* Move backend type validation from serializer to model
This commit is contained in:
Jeremy Stretch
2023-10-24 11:35:53 -04:00
committed by GitHub
parent 7274e75b26
commit 30ce9edf1c
23 changed files with 250 additions and 113 deletions

View File

@ -0,0 +1,23 @@
# Data Backends
[Data sources](../../models/core/datasource.md) can be defined to reference data which exists on systems of record outside NetBox, such as a git repository or Amazon S3 bucket. Plugins can register their own backend classes to introduce support for additional resource types. This is done by subclassing NetBox's `DataBackend` class.
```python title="data_backends.py"
from netbox.data_backends import DataBackend
class MyDataBackend(DataBackend):
name = 'mybackend'
label = 'My Backend'
...
```
To register one or more data backends with NetBox, define a list named `backends` at the end of this file:
```python title="data_backends.py"
backends = [MyDataBackend]
```
!!! tip
The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance.
::: core.data_backends.DataBackend

View File

@ -109,6 +109,7 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
| `middleware` | A list of middleware classes to append after NetBox's build-in middleware | | `middleware` | A list of middleware classes to append after NetBox's build-in middleware |
| `queues` | A list of custom background task queues to create | | `queues` | A list of custom background task queues to create |
| `search_extensions` | The dotted path to the list of search index classes (default: `search.indexes`) | | `search_extensions` | The dotted path to the list of search index classes (default: `search.indexes`) |
| `data_backends` | The dotted path to the list of data source backend classes (default: `data_backends.backends`) |
| `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`) | | `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`) |
| `menu_items` | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`) | | `menu_items` | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`) |
| `graphql_schema` | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`) | | `graphql_schema` | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`) |

View File

@ -136,6 +136,7 @@ nav:
- Forms: 'plugins/development/forms.md' - Forms: 'plugins/development/forms.md'
- Filters & Filter Sets: 'plugins/development/filtersets.md' - Filters & Filter Sets: 'plugins/development/filtersets.md'
- Search: 'plugins/development/search.md' - Search: 'plugins/development/search.md'
- Data Backends: 'plugins/development/data-backends.md'
- REST API: 'plugins/development/rest-api.md' - REST API: 'plugins/development/rest-api.md'
- GraphQL API: 'plugins/development/graphql-api.md' - GraphQL API: 'plugins/development/graphql-api.md'
- Background Tasks: 'plugins/development/background-tasks.md' - Background Tasks: 'plugins/development/background-tasks.md'

View File

@ -4,6 +4,7 @@ from core.choices import *
from core.models import * from core.models import *
from netbox.api.fields import ChoiceField, ContentTypeField from netbox.api.fields import ChoiceField, ContentTypeField
from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
from netbox.utils import get_data_backend_choices
from users.api.nested_serializers import NestedUserSerializer from users.api.nested_serializers import NestedUserSerializer
from .nested_serializers import * from .nested_serializers import *
@ -19,7 +20,7 @@ class DataSourceSerializer(NetBoxModelSerializer):
view_name='core-api:datasource-detail' view_name='core-api:datasource-detail'
) )
type = ChoiceField( type = ChoiceField(
choices=DataSourceTypeChoices choices=get_data_backend_choices()
) )
status = ChoiceField( status = ChoiceField(
choices=DataSourceStatusChoices, choices=DataSourceStatusChoices,

View File

@ -7,18 +7,6 @@ from utilities.choices import ChoiceSet
# Data sources # Data sources
# #
class DataSourceTypeChoices(ChoiceSet):
LOCAL = 'local'
GIT = 'git'
AMAZON_S3 = 'amazon-s3'
CHOICES = (
(LOCAL, _('Local'), 'gray'),
(GIT, 'Git', 'blue'),
(AMAZON_S3, 'Amazon S3', 'blue'),
)
class DataSourceStatusChoices(ChoiceSet): class DataSourceStatusChoices(ChoiceSet):
NEW = 'new' NEW = 'new'
QUEUED = 'queued' QUEUED = 'queued'

View File

@ -10,61 +10,24 @@ from django import forms
from django.conf import settings from django.conf import settings
from django.utils.translation import gettext as _ from django.utils.translation import gettext as _
from netbox.registry import registry from netbox.data_backends import DataBackend
from .choices import DataSourceTypeChoices from netbox.utils import register_data_backend
from .exceptions import SyncError from .exceptions import SyncError
__all__ = ( __all__ = (
'LocalBackend',
'GitBackend', 'GitBackend',
'LocalBackend',
'S3Backend', 'S3Backend',
) )
logger = logging.getLogger('netbox.data_backends') logger = logging.getLogger('netbox.data_backends')
def register_backend(name): @register_data_backend()
"""
Decorator for registering a DataBackend class.
"""
def _wrapper(cls):
registry['data_backends'][name] = cls
return cls
return _wrapper
class DataBackend:
parameters = {}
sensitive_parameters = []
# Prevent Django's template engine from calling the backend
# class when referenced via DataSource.backend_class
do_not_call_in_templates = True
def __init__(self, url, **kwargs):
self.url = url
self.params = kwargs
self.config = self.init_config()
def init_config(self):
"""
Hook to initialize the instance's configuration.
"""
return
@property
def url_scheme(self):
return urlparse(self.url).scheme.lower()
@contextmanager
def fetch(self):
raise NotImplemented()
@register_backend(DataSourceTypeChoices.LOCAL)
class LocalBackend(DataBackend): class LocalBackend(DataBackend):
name = 'local'
label = _('Local')
is_local = True
@contextmanager @contextmanager
def fetch(self): def fetch(self):
@ -74,8 +37,10 @@ class LocalBackend(DataBackend):
yield local_path yield local_path
@register_backend(DataSourceTypeChoices.GIT) @register_data_backend()
class GitBackend(DataBackend): class GitBackend(DataBackend):
name = 'git'
label = 'Git'
parameters = { parameters = {
'username': forms.CharField( 'username': forms.CharField(
required=False, required=False,
@ -144,8 +109,10 @@ class GitBackend(DataBackend):
local_path.cleanup() local_path.cleanup()
@register_backend(DataSourceTypeChoices.AMAZON_S3) @register_data_backend()
class S3Backend(DataBackend): class S3Backend(DataBackend):
name = 'amazon-s3'
label = 'Amazon S3'
parameters = { parameters = {
'aws_access_key_id': forms.CharField( 'aws_access_key_id': forms.CharField(
label=_('AWS access key ID'), label=_('AWS access key ID'),

View File

@ -4,6 +4,7 @@ from django.utils.translation import gettext as _
import django_filters import django_filters
from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
from netbox.utils import get_data_backend_choices
from .choices import * from .choices import *
from .models import * from .models import *
@ -16,7 +17,7 @@ __all__ = (
class DataSourceFilterSet(NetBoxModelFilterSet): class DataSourceFilterSet(NetBoxModelFilterSet):
type = django_filters.MultipleChoiceFilter( type = django_filters.MultipleChoiceFilter(
choices=DataSourceTypeChoices, choices=get_data_backend_choices,
null_value=None null_value=None
) )
status = django_filters.MultipleChoiceFilter( status = django_filters.MultipleChoiceFilter(

View File

@ -1,10 +1,9 @@
from django import forms from django import forms
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from core.choices import DataSourceTypeChoices
from core.models import * from core.models import *
from netbox.forms import NetBoxModelBulkEditForm from netbox.forms import NetBoxModelBulkEditForm
from utilities.forms import add_blank_choice from netbox.utils import get_data_backend_choices
from utilities.forms.fields import CommentField from utilities.forms.fields import CommentField
from utilities.forms.widgets import BulkEditNullBooleanSelect from utilities.forms.widgets import BulkEditNullBooleanSelect
@ -16,9 +15,8 @@ __all__ = (
class DataSourceBulkEditForm(NetBoxModelBulkEditForm): class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
type = forms.ChoiceField( type = forms.ChoiceField(
label=_('Type'), label=_('Type'),
choices=add_blank_choice(DataSourceTypeChoices), choices=get_data_backend_choices,
required=False, required=False
initial=''
) )
enabled = forms.NullBooleanField( enabled = forms.NullBooleanField(
required=False, required=False,

View File

@ -8,6 +8,7 @@ from core.models import *
from extras.forms.mixins import SavedFiltersMixin from extras.forms.mixins import SavedFiltersMixin
from extras.utils import FeatureQuery from extras.utils import FeatureQuery
from netbox.forms import NetBoxModelFilterSetForm from netbox.forms import NetBoxModelFilterSetForm
from netbox.utils import get_data_backend_choices
from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField
from utilities.forms.widgets import APISelectMultiple, DateTimePicker from utilities.forms.widgets import APISelectMultiple, DateTimePicker
@ -27,7 +28,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
) )
type = forms.MultipleChoiceField( type = forms.MultipleChoiceField(
label=_('Type'), label=_('Type'),
choices=DataSourceTypeChoices, choices=get_data_backend_choices,
required=False required=False
) )
status = forms.MultipleChoiceField( status = forms.MultipleChoiceField(

View File

@ -7,6 +7,7 @@ from core.forms.mixins import SyncedDataMixin
from core.models import * from core.models import *
from netbox.forms import NetBoxModelForm from netbox.forms import NetBoxModelForm
from netbox.registry import registry from netbox.registry import registry
from netbox.utils import get_data_backend_choices
from utilities.forms import get_field_value from utilities.forms import get_field_value
from utilities.forms.fields import CommentField from utilities.forms.fields import CommentField
from utilities.forms.widgets import HTMXSelect from utilities.forms.widgets import HTMXSelect
@ -18,6 +19,10 @@ __all__ = (
class DataSourceForm(NetBoxModelForm): class DataSourceForm(NetBoxModelForm):
type = forms.ChoiceField(
choices=get_data_backend_choices,
widget=HTMXSelect()
)
comments = CommentField() comments = CommentField()
class Meta: class Meta:
@ -26,7 +31,6 @@ class DataSourceForm(NetBoxModelForm):
'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags', 'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
] ]
widgets = { widgets = {
'type': HTMXSelect(),
'ignore_rules': forms.Textarea( 'ignore_rules': forms.Textarea(
attrs={ attrs={
'rows': 5, 'rows': 5,
@ -56,12 +60,13 @@ class DataSourceForm(NetBoxModelForm):
# Add backend-specific form fields # Add backend-specific form fields
self.backend_fields = [] self.backend_fields = []
for name, form_field in backend.parameters.items(): if backend:
field_name = f'backend_{name}' for name, form_field in backend.parameters.items():
self.backend_fields.append(field_name) field_name = f'backend_{name}'
self.fields[field_name] = copy.copy(form_field) self.backend_fields.append(field_name)
if self.instance and self.instance.parameters: self.fields[field_name] = copy.copy(form_field)
self.fields[field_name].initial = self.instance.parameters.get(name) if self.instance and self.instance.parameters:
self.fields[field_name].initial = self.instance.parameters.get(name)
def save(self, *args, **kwargs): def save(self, *args, **kwargs):

View File

@ -0,0 +1,18 @@
# Generated by Django 4.2.6 on 2023-10-20 17:47
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0005_job_created_auto_now'),
]
operations = [
migrations.AlterField(
model_name='datasource',
name='type',
field=models.CharField(max_length=50),
),
]

View File

@ -45,9 +45,7 @@ class DataSource(JobsMixin, PrimaryModel):
) )
type = models.CharField( type = models.CharField(
verbose_name=_('type'), verbose_name=_('type'),
max_length=50, max_length=50
choices=DataSourceTypeChoices,
default=DataSourceTypeChoices.LOCAL
) )
source_url = models.CharField( source_url = models.CharField(
max_length=200, max_length=200,
@ -96,8 +94,9 @@ class DataSource(JobsMixin, PrimaryModel):
def docs_url(self): def docs_url(self):
return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/' return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
def get_type_color(self): def get_type_display(self):
return DataSourceTypeChoices.colors.get(self.type) if backend := registry['data_backends'].get(self.type):
return backend.label
def get_status_color(self): def get_status_color(self):
return DataSourceStatusChoices.colors.get(self.status) return DataSourceStatusChoices.colors.get(self.status)
@ -110,10 +109,6 @@ class DataSource(JobsMixin, PrimaryModel):
def backend_class(self): def backend_class(self):
return registry['data_backends'].get(self.type) return registry['data_backends'].get(self.type)
@property
def is_local(self):
return self.type == DataSourceTypeChoices.LOCAL
@property @property
def ready_for_sync(self): def ready_for_sync(self):
return self.enabled and self.status not in ( return self.enabled and self.status not in (
@ -123,8 +118,14 @@ class DataSource(JobsMixin, PrimaryModel):
def clean(self): def clean(self):
# Validate data backend type
if self.type and self.type not in registry['data_backends']:
raise ValidationError({
'type': _("Unknown backend type: {type}".format(type=self.type))
})
# Ensure URL scheme matches selected type # Ensure URL scheme matches selected type
if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''): if self.backend_class.is_local and self.url_scheme not in ('file', ''):
raise ValidationError({ raise ValidationError({
'source_url': f"URLs for local sources must start with file:// (or specify no scheme)" 'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
}) })

View File

@ -0,0 +1,20 @@
import django_tables2 as tables
from netbox.registry import registry
__all__ = (
'BackendTypeColumn',
)
class BackendTypeColumn(tables.Column):
"""
Display a data backend type.
"""
def render(self, value):
if backend := registry['data_backends'].get(value):
return backend.label
return value
def value(self, value):
return value

View File

@ -3,6 +3,7 @@ import django_tables2 as tables
from core.models import * from core.models import *
from netbox.tables import NetBoxTable, columns from netbox.tables import NetBoxTable, columns
from .columns import BackendTypeColumn
__all__ = ( __all__ = (
'DataFileTable', 'DataFileTable',
@ -15,8 +16,8 @@ class DataSourceTable(NetBoxTable):
verbose_name=_('Name'), verbose_name=_('Name'),
linkify=True linkify=True
) )
type = columns.ChoiceFieldColumn( type = BackendTypeColumn(
verbose_name=_('Type'), verbose_name=_('Type')
) )
status = columns.ChoiceFieldColumn( status = columns.ChoiceFieldColumn(
verbose_name=_('Status'), verbose_name=_('Status'),
@ -34,8 +35,8 @@ class DataSourceTable(NetBoxTable):
class Meta(NetBoxTable.Meta): class Meta(NetBoxTable.Meta):
model = DataSource model = DataSource
fields = ( fields = (
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created', 'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
'last_updated', 'file_count', 'created', 'last_updated', 'file_count',
) )
default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count') default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')

View File

@ -2,7 +2,6 @@ from django.urls import reverse
from django.utils import timezone from django.utils import timezone
from utilities.testing import APITestCase, APIViewTestCases from utilities.testing import APITestCase, APIViewTestCases
from ..choices import *
from ..models import * from ..models import *
@ -26,26 +25,26 @@ class DataSourceTest(APIViewTestCases.APIViewTestCase):
@classmethod @classmethod
def setUpTestData(cls): def setUpTestData(cls):
data_sources = ( data_sources = (
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'), DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'), DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'), DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
) )
DataSource.objects.bulk_create(data_sources) DataSource.objects.bulk_create(data_sources)
cls.create_data = [ cls.create_data = [
{ {
'name': 'Data Source 4', 'name': 'Data Source 4',
'type': DataSourceTypeChoices.GIT, 'type': 'git',
'source_url': 'https://example.com/git/source4' 'source_url': 'https://example.com/git/source4'
}, },
{ {
'name': 'Data Source 5', 'name': 'Data Source 5',
'type': DataSourceTypeChoices.GIT, 'type': 'git',
'source_url': 'https://example.com/git/source5' 'source_url': 'https://example.com/git/source5'
}, },
{ {
'name': 'Data Source 6', 'name': 'Data Source 6',
'type': DataSourceTypeChoices.GIT, 'type': 'git',
'source_url': 'https://example.com/git/source6' 'source_url': 'https://example.com/git/source6'
}, },
] ]
@ -63,7 +62,7 @@ class DataFileTest(
def setUpTestData(cls): def setUpTestData(cls):
datasource = DataSource.objects.create( datasource = DataSource.objects.create(
name='Data Source 1', name='Data Source 1',
type=DataSourceTypeChoices.LOCAL, type='local',
source_url='file:///var/tmp/source1/' source_url='file:///var/tmp/source1/'
) )

View File

@ -18,21 +18,21 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
data_sources = ( data_sources = (
DataSource( DataSource(
name='Data Source 1', name='Data Source 1',
type=DataSourceTypeChoices.LOCAL, type='local',
source_url='file:///var/tmp/source1/', source_url='file:///var/tmp/source1/',
status=DataSourceStatusChoices.NEW, status=DataSourceStatusChoices.NEW,
enabled=True enabled=True
), ),
DataSource( DataSource(
name='Data Source 2', name='Data Source 2',
type=DataSourceTypeChoices.LOCAL, type='local',
source_url='file:///var/tmp/source2/', source_url='file:///var/tmp/source2/',
status=DataSourceStatusChoices.SYNCING, status=DataSourceStatusChoices.SYNCING,
enabled=True enabled=True
), ),
DataSource( DataSource(
name='Data Source 3', name='Data Source 3',
type=DataSourceTypeChoices.GIT, type='git',
source_url='https://example.com/git/source3', source_url='https://example.com/git/source3',
status=DataSourceStatusChoices.COMPLETED, status=DataSourceStatusChoices.COMPLETED,
enabled=False enabled=False
@ -45,7 +45,7 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
def test_type(self): def test_type(self):
params = {'type': [DataSourceTypeChoices.LOCAL]} params = {'type': ['local']}
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
def test_enabled(self): def test_enabled(self):
@ -66,9 +66,9 @@ class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests):
@classmethod @classmethod
def setUpTestData(cls): def setUpTestData(cls):
data_sources = ( data_sources = (
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'), DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'), DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'), DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
) )
DataSource.objects.bulk_create(data_sources) DataSource.objects.bulk_create(data_sources)

View File

@ -1,7 +1,6 @@
from django.utils import timezone from django.utils import timezone
from utilities.testing import ViewTestCases, create_tags from utilities.testing import ViewTestCases, create_tags
from ..choices import *
from ..models import * from ..models import *
@ -11,9 +10,9 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
@classmethod @classmethod
def setUpTestData(cls): def setUpTestData(cls):
data_sources = ( data_sources = (
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'), DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'), DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'), DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
) )
DataSource.objects.bulk_create(data_sources) DataSource.objects.bulk_create(data_sources)
@ -21,7 +20,7 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
cls.form_data = { cls.form_data = {
'name': 'Data Source X', 'name': 'Data Source X',
'type': DataSourceTypeChoices.GIT, 'type': 'git',
'source_url': 'http:///exmaple/com/foo/bar/', 'source_url': 'http:///exmaple/com/foo/bar/',
'description': 'Something', 'description': 'Something',
'comments': 'Foo bar baz', 'comments': 'Foo bar baz',
@ -29,10 +28,10 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
} }
cls.csv_data = ( cls.csv_data = (
f"name,type,source_url,enabled", "name,type,source_url,enabled",
f"Data Source 4,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true", "Data Source 4,local,file:///var/tmp/source4/,true",
f"Data Source 5,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true", "Data Source 5,local,file:///var/tmp/source4/,true",
f"Data Source 6,{DataSourceTypeChoices.GIT},http:///exmaple/com/foo/bar/,false", "Data Source 6,git,http:///exmaple/com/foo/bar/,false",
) )
cls.csv_update_data = ( cls.csv_update_data = (
@ -60,7 +59,7 @@ class DataFileTestCase(
def setUpTestData(cls): def setUpTestData(cls):
datasource = DataSource.objects.create( datasource = DataSource.objects.create(
name='Data Source 1', name='Data Source 1',
type=DataSourceTypeChoices.LOCAL, type='local',
source_url='file:///var/tmp/source1/' source_url='file:///var/tmp/source1/'
) )

View File

@ -0,0 +1,53 @@
from contextlib import contextmanager
from urllib.parse import urlparse
__all__ = (
'DataBackend',
)
class DataBackend:
"""
A data backend represents a specific system of record for data, such as a git repository or Amazon S3 bucket.
Attributes:
name: The identifier under which this backend will be registered in NetBox
label: The human-friendly name for this backend
is_local: A boolean indicating whether this backend accesses local data
parameters: A dictionary mapping configuration form field names to their classes
sensitive_parameters: An iterable of field names for which the values should not be displayed to the user
"""
is_local = False
parameters = {}
sensitive_parameters = []
# Prevent Django's template engine from calling the backend
# class when referenced via DataSource.backend_class
do_not_call_in_templates = True
def __init__(self, url, **kwargs):
self.url = url
self.params = kwargs
self.config = self.init_config()
def init_config(self):
"""
A hook to initialize the instance's configuration. The data returned by this method is assigned to the
instance's `config` attribute upon initialization, which can be referenced by the `fetch()` method.
"""
return
@property
def url_scheme(self):
return urlparse(self.url).scheme.lower()
@contextmanager
def fetch(self):
"""
A context manager which performs the following:
1. Handles all setup and synchronization
2. Yields the local path at which data has been replicated
3. Performs any necessary cleanup
"""
raise NotImplemented()

View File

@ -8,6 +8,7 @@ from packaging import version
from netbox.registry import registry from netbox.registry import registry
from netbox.search import register_search from netbox.search import register_search
from netbox.utils import register_data_backend
from .navigation import * from .navigation import *
from .registration import * from .registration import *
from .templates import * from .templates import *
@ -24,6 +25,7 @@ registry['plugins'].update({
DEFAULT_RESOURCE_PATHS = { DEFAULT_RESOURCE_PATHS = {
'search_indexes': 'search.indexes', 'search_indexes': 'search.indexes',
'data_backends': 'data_backends.backends',
'graphql_schema': 'graphql.schema', 'graphql_schema': 'graphql.schema',
'menu': 'navigation.menu', 'menu': 'navigation.menu',
'menu_items': 'navigation.menu_items', 'menu_items': 'navigation.menu_items',
@ -70,6 +72,7 @@ class PluginConfig(AppConfig):
# Optional plugin resources # Optional plugin resources
search_indexes = None search_indexes = None
data_backends = None
graphql_schema = None graphql_schema = None
menu = None menu = None
menu_items = None menu_items = None
@ -98,6 +101,11 @@ class PluginConfig(AppConfig):
for idx in search_indexes: for idx in search_indexes:
register_search(idx) register_search(idx)
# Register data backends (if defined)
data_backends = self._load_resource('data_backends') or []
for backend in data_backends:
register_data_backend()(backend)
# Register template content (if defined) # Register template content (if defined)
if template_extensions := self._load_resource('template_extensions'): if template_extensions := self._load_resource('template_extensions'):
register_template_extensions(template_extensions) register_template_extensions(template_extensions)

View File

@ -0,0 +1,18 @@
from contextlib import contextmanager
from netbox.data_backends import DataBackend
class DummyBackend(DataBackend):
name = 'dummy'
label = 'Dummy'
is_local = True
@contextmanager
def fetch(self):
yield '/tmp'
backends = (
DummyBackend,
)

View File

@ -6,6 +6,7 @@ from django.test import Client, TestCase, override_settings
from django.urls import reverse from django.urls import reverse
from netbox.tests.dummy_plugin import config as dummy_config from netbox.tests.dummy_plugin import config as dummy_config
from netbox.tests.dummy_plugin.data_backends import DummyBackend
from netbox.plugins.navigation import PluginMenu from netbox.plugins.navigation import PluginMenu
from netbox.plugins.utils import get_plugin_config from netbox.plugins.utils import get_plugin_config
from netbox.graphql.schema import Query from netbox.graphql.schema import Query
@ -111,6 +112,13 @@ class PluginTest(TestCase):
""" """
self.assertIn('netbox.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE) self.assertIn('netbox.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
def test_data_backends(self):
"""
Check registered data backends.
"""
self.assertIn('dummy', registry['data_backends'])
self.assertIs(registry['data_backends']['dummy'], DummyBackend)
def test_queues(self): def test_queues(self):
""" """
Check that plugin queues are registered with the accurate name. Check that plugin queues are registered with the accurate name.

26
netbox/netbox/utils.py Normal file
View File

@ -0,0 +1,26 @@
from netbox.registry import registry
__all__ = (
'get_data_backend_choices',
'register_data_backend',
)
def get_data_backend_choices():
return [
(None, '---------'),
*[
(name, cls.label) for name, cls in registry['data_backends'].items()
]
]
def register_data_backend():
"""
Decorator for registering a DataBackend class.
"""
def _wrapper(cls):
registry['data_backends'][cls.name] = cls
return cls
return _wrapper

View File

@ -58,7 +58,7 @@
<tr> <tr>
<th scope="row">{% trans "URL" %}</th> <th scope="row">{% trans "URL" %}</th>
<td> <td>
{% if not object.is_local %} {% if not object.type.is_local %}
<a href="{{ object.source_url }}">{{ object.source_url }}</a> <a href="{{ object.source_url }}">{{ object.source_url }}</a>
{% else %} {% else %}
{{ object.source_url }} {{ object.source_url }}