mirror of
https://github.com/netbox-community/netbox.git
synced 2024-05-10 07:54:54 +00:00
* Initial work on #13381 * Fix backend type display in table column * Fix data source type choices during bulk edit * Misc cleanup * Move backend utils from core app to netbox * Move backend type validation from serializer to model
This commit is contained in:
23
docs/plugins/development/data-backends.md
Normal file
23
docs/plugins/development/data-backends.md
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
# Data Backends
|
||||||
|
|
||||||
|
[Data sources](../../models/core/datasource.md) can be defined to reference data which exists on systems of record outside NetBox, such as a git repository or Amazon S3 bucket. Plugins can register their own backend classes to introduce support for additional resource types. This is done by subclassing NetBox's `DataBackend` class.
|
||||||
|
|
||||||
|
```python title="data_backends.py"
|
||||||
|
from netbox.data_backends import DataBackend
|
||||||
|
|
||||||
|
class MyDataBackend(DataBackend):
|
||||||
|
name = 'mybackend'
|
||||||
|
label = 'My Backend'
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
To register one or more data backends with NetBox, define a list named `backends` at the end of this file:
|
||||||
|
|
||||||
|
```python title="data_backends.py"
|
||||||
|
backends = [MyDataBackend]
|
||||||
|
```
|
||||||
|
|
||||||
|
!!! tip
|
||||||
|
The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance.
|
||||||
|
|
||||||
|
::: core.data_backends.DataBackend
|
@ -109,6 +109,7 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
|
|||||||
| `middleware` | A list of middleware classes to append after NetBox's build-in middleware |
|
| `middleware` | A list of middleware classes to append after NetBox's build-in middleware |
|
||||||
| `queues` | A list of custom background task queues to create |
|
| `queues` | A list of custom background task queues to create |
|
||||||
| `search_extensions` | The dotted path to the list of search index classes (default: `search.indexes`) |
|
| `search_extensions` | The dotted path to the list of search index classes (default: `search.indexes`) |
|
||||||
|
| `data_backends` | The dotted path to the list of data source backend classes (default: `data_backends.backends`) |
|
||||||
| `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`) |
|
| `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`) |
|
||||||
| `menu_items` | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`) |
|
| `menu_items` | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`) |
|
||||||
| `graphql_schema` | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`) |
|
| `graphql_schema` | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`) |
|
||||||
|
@ -136,6 +136,7 @@ nav:
|
|||||||
- Forms: 'plugins/development/forms.md'
|
- Forms: 'plugins/development/forms.md'
|
||||||
- Filters & Filter Sets: 'plugins/development/filtersets.md'
|
- Filters & Filter Sets: 'plugins/development/filtersets.md'
|
||||||
- Search: 'plugins/development/search.md'
|
- Search: 'plugins/development/search.md'
|
||||||
|
- Data Backends: 'plugins/development/data-backends.md'
|
||||||
- REST API: 'plugins/development/rest-api.md'
|
- REST API: 'plugins/development/rest-api.md'
|
||||||
- GraphQL API: 'plugins/development/graphql-api.md'
|
- GraphQL API: 'plugins/development/graphql-api.md'
|
||||||
- Background Tasks: 'plugins/development/background-tasks.md'
|
- Background Tasks: 'plugins/development/background-tasks.md'
|
||||||
|
@ -4,6 +4,7 @@ from core.choices import *
|
|||||||
from core.models import *
|
from core.models import *
|
||||||
from netbox.api.fields import ChoiceField, ContentTypeField
|
from netbox.api.fields import ChoiceField, ContentTypeField
|
||||||
from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
|
from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
|
||||||
|
from netbox.utils import get_data_backend_choices
|
||||||
from users.api.nested_serializers import NestedUserSerializer
|
from users.api.nested_serializers import NestedUserSerializer
|
||||||
from .nested_serializers import *
|
from .nested_serializers import *
|
||||||
|
|
||||||
@ -19,7 +20,7 @@ class DataSourceSerializer(NetBoxModelSerializer):
|
|||||||
view_name='core-api:datasource-detail'
|
view_name='core-api:datasource-detail'
|
||||||
)
|
)
|
||||||
type = ChoiceField(
|
type = ChoiceField(
|
||||||
choices=DataSourceTypeChoices
|
choices=get_data_backend_choices()
|
||||||
)
|
)
|
||||||
status = ChoiceField(
|
status = ChoiceField(
|
||||||
choices=DataSourceStatusChoices,
|
choices=DataSourceStatusChoices,
|
||||||
|
@ -7,18 +7,6 @@ from utilities.choices import ChoiceSet
|
|||||||
# Data sources
|
# Data sources
|
||||||
#
|
#
|
||||||
|
|
||||||
class DataSourceTypeChoices(ChoiceSet):
|
|
||||||
LOCAL = 'local'
|
|
||||||
GIT = 'git'
|
|
||||||
AMAZON_S3 = 'amazon-s3'
|
|
||||||
|
|
||||||
CHOICES = (
|
|
||||||
(LOCAL, _('Local'), 'gray'),
|
|
||||||
(GIT, 'Git', 'blue'),
|
|
||||||
(AMAZON_S3, 'Amazon S3', 'blue'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DataSourceStatusChoices(ChoiceSet):
|
class DataSourceStatusChoices(ChoiceSet):
|
||||||
NEW = 'new'
|
NEW = 'new'
|
||||||
QUEUED = 'queued'
|
QUEUED = 'queued'
|
||||||
|
@ -10,61 +10,24 @@ from django import forms
|
|||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.utils.translation import gettext as _
|
from django.utils.translation import gettext as _
|
||||||
|
|
||||||
from netbox.registry import registry
|
from netbox.data_backends import DataBackend
|
||||||
from .choices import DataSourceTypeChoices
|
from netbox.utils import register_data_backend
|
||||||
from .exceptions import SyncError
|
from .exceptions import SyncError
|
||||||
|
|
||||||
__all__ = (
|
__all__ = (
|
||||||
'LocalBackend',
|
|
||||||
'GitBackend',
|
'GitBackend',
|
||||||
|
'LocalBackend',
|
||||||
'S3Backend',
|
'S3Backend',
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger('netbox.data_backends')
|
logger = logging.getLogger('netbox.data_backends')
|
||||||
|
|
||||||
|
|
||||||
def register_backend(name):
|
@register_data_backend()
|
||||||
"""
|
|
||||||
Decorator for registering a DataBackend class.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def _wrapper(cls):
|
|
||||||
registry['data_backends'][name] = cls
|
|
||||||
return cls
|
|
||||||
|
|
||||||
return _wrapper
|
|
||||||
|
|
||||||
|
|
||||||
class DataBackend:
|
|
||||||
parameters = {}
|
|
||||||
sensitive_parameters = []
|
|
||||||
|
|
||||||
# Prevent Django's template engine from calling the backend
|
|
||||||
# class when referenced via DataSource.backend_class
|
|
||||||
do_not_call_in_templates = True
|
|
||||||
|
|
||||||
def __init__(self, url, **kwargs):
|
|
||||||
self.url = url
|
|
||||||
self.params = kwargs
|
|
||||||
self.config = self.init_config()
|
|
||||||
|
|
||||||
def init_config(self):
|
|
||||||
"""
|
|
||||||
Hook to initialize the instance's configuration.
|
|
||||||
"""
|
|
||||||
return
|
|
||||||
|
|
||||||
@property
|
|
||||||
def url_scheme(self):
|
|
||||||
return urlparse(self.url).scheme.lower()
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def fetch(self):
|
|
||||||
raise NotImplemented()
|
|
||||||
|
|
||||||
|
|
||||||
@register_backend(DataSourceTypeChoices.LOCAL)
|
|
||||||
class LocalBackend(DataBackend):
|
class LocalBackend(DataBackend):
|
||||||
|
name = 'local'
|
||||||
|
label = _('Local')
|
||||||
|
is_local = True
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def fetch(self):
|
def fetch(self):
|
||||||
@ -74,8 +37,10 @@ class LocalBackend(DataBackend):
|
|||||||
yield local_path
|
yield local_path
|
||||||
|
|
||||||
|
|
||||||
@register_backend(DataSourceTypeChoices.GIT)
|
@register_data_backend()
|
||||||
class GitBackend(DataBackend):
|
class GitBackend(DataBackend):
|
||||||
|
name = 'git'
|
||||||
|
label = 'Git'
|
||||||
parameters = {
|
parameters = {
|
||||||
'username': forms.CharField(
|
'username': forms.CharField(
|
||||||
required=False,
|
required=False,
|
||||||
@ -144,8 +109,10 @@ class GitBackend(DataBackend):
|
|||||||
local_path.cleanup()
|
local_path.cleanup()
|
||||||
|
|
||||||
|
|
||||||
@register_backend(DataSourceTypeChoices.AMAZON_S3)
|
@register_data_backend()
|
||||||
class S3Backend(DataBackend):
|
class S3Backend(DataBackend):
|
||||||
|
name = 'amazon-s3'
|
||||||
|
label = 'Amazon S3'
|
||||||
parameters = {
|
parameters = {
|
||||||
'aws_access_key_id': forms.CharField(
|
'aws_access_key_id': forms.CharField(
|
||||||
label=_('AWS access key ID'),
|
label=_('AWS access key ID'),
|
||||||
|
@ -4,6 +4,7 @@ from django.utils.translation import gettext as _
|
|||||||
import django_filters
|
import django_filters
|
||||||
|
|
||||||
from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
|
from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
|
||||||
|
from netbox.utils import get_data_backend_choices
|
||||||
from .choices import *
|
from .choices import *
|
||||||
from .models import *
|
from .models import *
|
||||||
|
|
||||||
@ -16,7 +17,7 @@ __all__ = (
|
|||||||
|
|
||||||
class DataSourceFilterSet(NetBoxModelFilterSet):
|
class DataSourceFilterSet(NetBoxModelFilterSet):
|
||||||
type = django_filters.MultipleChoiceFilter(
|
type = django_filters.MultipleChoiceFilter(
|
||||||
choices=DataSourceTypeChoices,
|
choices=get_data_backend_choices,
|
||||||
null_value=None
|
null_value=None
|
||||||
)
|
)
|
||||||
status = django_filters.MultipleChoiceFilter(
|
status = django_filters.MultipleChoiceFilter(
|
||||||
|
@ -1,10 +1,9 @@
|
|||||||
from django import forms
|
from django import forms
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from core.choices import DataSourceTypeChoices
|
|
||||||
from core.models import *
|
from core.models import *
|
||||||
from netbox.forms import NetBoxModelBulkEditForm
|
from netbox.forms import NetBoxModelBulkEditForm
|
||||||
from utilities.forms import add_blank_choice
|
from netbox.utils import get_data_backend_choices
|
||||||
from utilities.forms.fields import CommentField
|
from utilities.forms.fields import CommentField
|
||||||
from utilities.forms.widgets import BulkEditNullBooleanSelect
|
from utilities.forms.widgets import BulkEditNullBooleanSelect
|
||||||
|
|
||||||
@ -16,9 +15,8 @@ __all__ = (
|
|||||||
class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
|
class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
|
||||||
type = forms.ChoiceField(
|
type = forms.ChoiceField(
|
||||||
label=_('Type'),
|
label=_('Type'),
|
||||||
choices=add_blank_choice(DataSourceTypeChoices),
|
choices=get_data_backend_choices,
|
||||||
required=False,
|
required=False
|
||||||
initial=''
|
|
||||||
)
|
)
|
||||||
enabled = forms.NullBooleanField(
|
enabled = forms.NullBooleanField(
|
||||||
required=False,
|
required=False,
|
||||||
|
@ -8,6 +8,7 @@ from core.models import *
|
|||||||
from extras.forms.mixins import SavedFiltersMixin
|
from extras.forms.mixins import SavedFiltersMixin
|
||||||
from extras.utils import FeatureQuery
|
from extras.utils import FeatureQuery
|
||||||
from netbox.forms import NetBoxModelFilterSetForm
|
from netbox.forms import NetBoxModelFilterSetForm
|
||||||
|
from netbox.utils import get_data_backend_choices
|
||||||
from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
|
from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
|
||||||
from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField
|
from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField
|
||||||
from utilities.forms.widgets import APISelectMultiple, DateTimePicker
|
from utilities.forms.widgets import APISelectMultiple, DateTimePicker
|
||||||
@ -27,7 +28,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
|
|||||||
)
|
)
|
||||||
type = forms.MultipleChoiceField(
|
type = forms.MultipleChoiceField(
|
||||||
label=_('Type'),
|
label=_('Type'),
|
||||||
choices=DataSourceTypeChoices,
|
choices=get_data_backend_choices,
|
||||||
required=False
|
required=False
|
||||||
)
|
)
|
||||||
status = forms.MultipleChoiceField(
|
status = forms.MultipleChoiceField(
|
||||||
|
@ -7,6 +7,7 @@ from core.forms.mixins import SyncedDataMixin
|
|||||||
from core.models import *
|
from core.models import *
|
||||||
from netbox.forms import NetBoxModelForm
|
from netbox.forms import NetBoxModelForm
|
||||||
from netbox.registry import registry
|
from netbox.registry import registry
|
||||||
|
from netbox.utils import get_data_backend_choices
|
||||||
from utilities.forms import get_field_value
|
from utilities.forms import get_field_value
|
||||||
from utilities.forms.fields import CommentField
|
from utilities.forms.fields import CommentField
|
||||||
from utilities.forms.widgets import HTMXSelect
|
from utilities.forms.widgets import HTMXSelect
|
||||||
@ -18,6 +19,10 @@ __all__ = (
|
|||||||
|
|
||||||
|
|
||||||
class DataSourceForm(NetBoxModelForm):
|
class DataSourceForm(NetBoxModelForm):
|
||||||
|
type = forms.ChoiceField(
|
||||||
|
choices=get_data_backend_choices,
|
||||||
|
widget=HTMXSelect()
|
||||||
|
)
|
||||||
comments = CommentField()
|
comments = CommentField()
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
@ -26,7 +31,6 @@ class DataSourceForm(NetBoxModelForm):
|
|||||||
'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
|
'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
|
||||||
]
|
]
|
||||||
widgets = {
|
widgets = {
|
||||||
'type': HTMXSelect(),
|
|
||||||
'ignore_rules': forms.Textarea(
|
'ignore_rules': forms.Textarea(
|
||||||
attrs={
|
attrs={
|
||||||
'rows': 5,
|
'rows': 5,
|
||||||
@ -56,12 +60,13 @@ class DataSourceForm(NetBoxModelForm):
|
|||||||
|
|
||||||
# Add backend-specific form fields
|
# Add backend-specific form fields
|
||||||
self.backend_fields = []
|
self.backend_fields = []
|
||||||
for name, form_field in backend.parameters.items():
|
if backend:
|
||||||
field_name = f'backend_{name}'
|
for name, form_field in backend.parameters.items():
|
||||||
self.backend_fields.append(field_name)
|
field_name = f'backend_{name}'
|
||||||
self.fields[field_name] = copy.copy(form_field)
|
self.backend_fields.append(field_name)
|
||||||
if self.instance and self.instance.parameters:
|
self.fields[field_name] = copy.copy(form_field)
|
||||||
self.fields[field_name].initial = self.instance.parameters.get(name)
|
if self.instance and self.instance.parameters:
|
||||||
|
self.fields[field_name].initial = self.instance.parameters.get(name)
|
||||||
|
|
||||||
def save(self, *args, **kwargs):
|
def save(self, *args, **kwargs):
|
||||||
|
|
||||||
|
@ -0,0 +1,18 @@
|
|||||||
|
# Generated by Django 4.2.6 on 2023-10-20 17:47
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('core', '0005_job_created_auto_now'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='datasource',
|
||||||
|
name='type',
|
||||||
|
field=models.CharField(max_length=50),
|
||||||
|
),
|
||||||
|
]
|
@ -45,9 +45,7 @@ class DataSource(JobsMixin, PrimaryModel):
|
|||||||
)
|
)
|
||||||
type = models.CharField(
|
type = models.CharField(
|
||||||
verbose_name=_('type'),
|
verbose_name=_('type'),
|
||||||
max_length=50,
|
max_length=50
|
||||||
choices=DataSourceTypeChoices,
|
|
||||||
default=DataSourceTypeChoices.LOCAL
|
|
||||||
)
|
)
|
||||||
source_url = models.CharField(
|
source_url = models.CharField(
|
||||||
max_length=200,
|
max_length=200,
|
||||||
@ -96,8 +94,9 @@ class DataSource(JobsMixin, PrimaryModel):
|
|||||||
def docs_url(self):
|
def docs_url(self):
|
||||||
return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
|
return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
|
||||||
|
|
||||||
def get_type_color(self):
|
def get_type_display(self):
|
||||||
return DataSourceTypeChoices.colors.get(self.type)
|
if backend := registry['data_backends'].get(self.type):
|
||||||
|
return backend.label
|
||||||
|
|
||||||
def get_status_color(self):
|
def get_status_color(self):
|
||||||
return DataSourceStatusChoices.colors.get(self.status)
|
return DataSourceStatusChoices.colors.get(self.status)
|
||||||
@ -110,10 +109,6 @@ class DataSource(JobsMixin, PrimaryModel):
|
|||||||
def backend_class(self):
|
def backend_class(self):
|
||||||
return registry['data_backends'].get(self.type)
|
return registry['data_backends'].get(self.type)
|
||||||
|
|
||||||
@property
|
|
||||||
def is_local(self):
|
|
||||||
return self.type == DataSourceTypeChoices.LOCAL
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def ready_for_sync(self):
|
def ready_for_sync(self):
|
||||||
return self.enabled and self.status not in (
|
return self.enabled and self.status not in (
|
||||||
@ -123,8 +118,14 @@ class DataSource(JobsMixin, PrimaryModel):
|
|||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
|
|
||||||
|
# Validate data backend type
|
||||||
|
if self.type and self.type not in registry['data_backends']:
|
||||||
|
raise ValidationError({
|
||||||
|
'type': _("Unknown backend type: {type}".format(type=self.type))
|
||||||
|
})
|
||||||
|
|
||||||
# Ensure URL scheme matches selected type
|
# Ensure URL scheme matches selected type
|
||||||
if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''):
|
if self.backend_class.is_local and self.url_scheme not in ('file', ''):
|
||||||
raise ValidationError({
|
raise ValidationError({
|
||||||
'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
|
'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
|
||||||
})
|
})
|
||||||
|
20
netbox/core/tables/columns.py
Normal file
20
netbox/core/tables/columns.py
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
import django_tables2 as tables
|
||||||
|
|
||||||
|
from netbox.registry import registry
|
||||||
|
|
||||||
|
__all__ = (
|
||||||
|
'BackendTypeColumn',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class BackendTypeColumn(tables.Column):
|
||||||
|
"""
|
||||||
|
Display a data backend type.
|
||||||
|
"""
|
||||||
|
def render(self, value):
|
||||||
|
if backend := registry['data_backends'].get(value):
|
||||||
|
return backend.label
|
||||||
|
return value
|
||||||
|
|
||||||
|
def value(self, value):
|
||||||
|
return value
|
@ -3,6 +3,7 @@ import django_tables2 as tables
|
|||||||
|
|
||||||
from core.models import *
|
from core.models import *
|
||||||
from netbox.tables import NetBoxTable, columns
|
from netbox.tables import NetBoxTable, columns
|
||||||
|
from .columns import BackendTypeColumn
|
||||||
|
|
||||||
__all__ = (
|
__all__ = (
|
||||||
'DataFileTable',
|
'DataFileTable',
|
||||||
@ -15,8 +16,8 @@ class DataSourceTable(NetBoxTable):
|
|||||||
verbose_name=_('Name'),
|
verbose_name=_('Name'),
|
||||||
linkify=True
|
linkify=True
|
||||||
)
|
)
|
||||||
type = columns.ChoiceFieldColumn(
|
type = BackendTypeColumn(
|
||||||
verbose_name=_('Type'),
|
verbose_name=_('Type')
|
||||||
)
|
)
|
||||||
status = columns.ChoiceFieldColumn(
|
status = columns.ChoiceFieldColumn(
|
||||||
verbose_name=_('Status'),
|
verbose_name=_('Status'),
|
||||||
@ -34,8 +35,8 @@ class DataSourceTable(NetBoxTable):
|
|||||||
class Meta(NetBoxTable.Meta):
|
class Meta(NetBoxTable.Meta):
|
||||||
model = DataSource
|
model = DataSource
|
||||||
fields = (
|
fields = (
|
||||||
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created',
|
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
|
||||||
'last_updated', 'file_count',
|
'created', 'last_updated', 'file_count',
|
||||||
)
|
)
|
||||||
default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')
|
default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')
|
||||||
|
|
||||||
|
@ -2,7 +2,6 @@ from django.urls import reverse
|
|||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from utilities.testing import APITestCase, APIViewTestCases
|
from utilities.testing import APITestCase, APIViewTestCases
|
||||||
from ..choices import *
|
|
||||||
from ..models import *
|
from ..models import *
|
||||||
|
|
||||||
|
|
||||||
@ -26,26 +25,26 @@ class DataSourceTest(APIViewTestCases.APIViewTestCase):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def setUpTestData(cls):
|
def setUpTestData(cls):
|
||||||
data_sources = (
|
data_sources = (
|
||||||
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
|
DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
|
||||||
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
|
DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
|
||||||
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
|
DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
|
||||||
)
|
)
|
||||||
DataSource.objects.bulk_create(data_sources)
|
DataSource.objects.bulk_create(data_sources)
|
||||||
|
|
||||||
cls.create_data = [
|
cls.create_data = [
|
||||||
{
|
{
|
||||||
'name': 'Data Source 4',
|
'name': 'Data Source 4',
|
||||||
'type': DataSourceTypeChoices.GIT,
|
'type': 'git',
|
||||||
'source_url': 'https://example.com/git/source4'
|
'source_url': 'https://example.com/git/source4'
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'name': 'Data Source 5',
|
'name': 'Data Source 5',
|
||||||
'type': DataSourceTypeChoices.GIT,
|
'type': 'git',
|
||||||
'source_url': 'https://example.com/git/source5'
|
'source_url': 'https://example.com/git/source5'
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'name': 'Data Source 6',
|
'name': 'Data Source 6',
|
||||||
'type': DataSourceTypeChoices.GIT,
|
'type': 'git',
|
||||||
'source_url': 'https://example.com/git/source6'
|
'source_url': 'https://example.com/git/source6'
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
@ -63,7 +62,7 @@ class DataFileTest(
|
|||||||
def setUpTestData(cls):
|
def setUpTestData(cls):
|
||||||
datasource = DataSource.objects.create(
|
datasource = DataSource.objects.create(
|
||||||
name='Data Source 1',
|
name='Data Source 1',
|
||||||
type=DataSourceTypeChoices.LOCAL,
|
type='local',
|
||||||
source_url='file:///var/tmp/source1/'
|
source_url='file:///var/tmp/source1/'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -18,21 +18,21 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
|
|||||||
data_sources = (
|
data_sources = (
|
||||||
DataSource(
|
DataSource(
|
||||||
name='Data Source 1',
|
name='Data Source 1',
|
||||||
type=DataSourceTypeChoices.LOCAL,
|
type='local',
|
||||||
source_url='file:///var/tmp/source1/',
|
source_url='file:///var/tmp/source1/',
|
||||||
status=DataSourceStatusChoices.NEW,
|
status=DataSourceStatusChoices.NEW,
|
||||||
enabled=True
|
enabled=True
|
||||||
),
|
),
|
||||||
DataSource(
|
DataSource(
|
||||||
name='Data Source 2',
|
name='Data Source 2',
|
||||||
type=DataSourceTypeChoices.LOCAL,
|
type='local',
|
||||||
source_url='file:///var/tmp/source2/',
|
source_url='file:///var/tmp/source2/',
|
||||||
status=DataSourceStatusChoices.SYNCING,
|
status=DataSourceStatusChoices.SYNCING,
|
||||||
enabled=True
|
enabled=True
|
||||||
),
|
),
|
||||||
DataSource(
|
DataSource(
|
||||||
name='Data Source 3',
|
name='Data Source 3',
|
||||||
type=DataSourceTypeChoices.GIT,
|
type='git',
|
||||||
source_url='https://example.com/git/source3',
|
source_url='https://example.com/git/source3',
|
||||||
status=DataSourceStatusChoices.COMPLETED,
|
status=DataSourceStatusChoices.COMPLETED,
|
||||||
enabled=False
|
enabled=False
|
||||||
@ -45,7 +45,7 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
|
|||||||
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
|
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
|
||||||
|
|
||||||
def test_type(self):
|
def test_type(self):
|
||||||
params = {'type': [DataSourceTypeChoices.LOCAL]}
|
params = {'type': ['local']}
|
||||||
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
|
self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
|
||||||
|
|
||||||
def test_enabled(self):
|
def test_enabled(self):
|
||||||
@ -66,9 +66,9 @@ class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def setUpTestData(cls):
|
def setUpTestData(cls):
|
||||||
data_sources = (
|
data_sources = (
|
||||||
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
|
DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
|
||||||
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
|
DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
|
||||||
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
|
DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
|
||||||
)
|
)
|
||||||
DataSource.objects.bulk_create(data_sources)
|
DataSource.objects.bulk_create(data_sources)
|
||||||
|
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from utilities.testing import ViewTestCases, create_tags
|
from utilities.testing import ViewTestCases, create_tags
|
||||||
from ..choices import *
|
|
||||||
from ..models import *
|
from ..models import *
|
||||||
|
|
||||||
|
|
||||||
@ -11,9 +10,9 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def setUpTestData(cls):
|
def setUpTestData(cls):
|
||||||
data_sources = (
|
data_sources = (
|
||||||
DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
|
DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
|
||||||
DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
|
DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
|
||||||
DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
|
DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
|
||||||
)
|
)
|
||||||
DataSource.objects.bulk_create(data_sources)
|
DataSource.objects.bulk_create(data_sources)
|
||||||
|
|
||||||
@ -21,7 +20,7 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
|
|||||||
|
|
||||||
cls.form_data = {
|
cls.form_data = {
|
||||||
'name': 'Data Source X',
|
'name': 'Data Source X',
|
||||||
'type': DataSourceTypeChoices.GIT,
|
'type': 'git',
|
||||||
'source_url': 'http:///exmaple/com/foo/bar/',
|
'source_url': 'http:///exmaple/com/foo/bar/',
|
||||||
'description': 'Something',
|
'description': 'Something',
|
||||||
'comments': 'Foo bar baz',
|
'comments': 'Foo bar baz',
|
||||||
@ -29,10 +28,10 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
|
|||||||
}
|
}
|
||||||
|
|
||||||
cls.csv_data = (
|
cls.csv_data = (
|
||||||
f"name,type,source_url,enabled",
|
"name,type,source_url,enabled",
|
||||||
f"Data Source 4,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
|
"Data Source 4,local,file:///var/tmp/source4/,true",
|
||||||
f"Data Source 5,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
|
"Data Source 5,local,file:///var/tmp/source4/,true",
|
||||||
f"Data Source 6,{DataSourceTypeChoices.GIT},http:///exmaple/com/foo/bar/,false",
|
"Data Source 6,git,http:///exmaple/com/foo/bar/,false",
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.csv_update_data = (
|
cls.csv_update_data = (
|
||||||
@ -60,7 +59,7 @@ class DataFileTestCase(
|
|||||||
def setUpTestData(cls):
|
def setUpTestData(cls):
|
||||||
datasource = DataSource.objects.create(
|
datasource = DataSource.objects.create(
|
||||||
name='Data Source 1',
|
name='Data Source 1',
|
||||||
type=DataSourceTypeChoices.LOCAL,
|
type='local',
|
||||||
source_url='file:///var/tmp/source1/'
|
source_url='file:///var/tmp/source1/'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
53
netbox/netbox/data_backends.py
Normal file
53
netbox/netbox/data_backends.py
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
from contextlib import contextmanager
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
__all__ = (
|
||||||
|
'DataBackend',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DataBackend:
|
||||||
|
"""
|
||||||
|
A data backend represents a specific system of record for data, such as a git repository or Amazon S3 bucket.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
name: The identifier under which this backend will be registered in NetBox
|
||||||
|
label: The human-friendly name for this backend
|
||||||
|
is_local: A boolean indicating whether this backend accesses local data
|
||||||
|
parameters: A dictionary mapping configuration form field names to their classes
|
||||||
|
sensitive_parameters: An iterable of field names for which the values should not be displayed to the user
|
||||||
|
"""
|
||||||
|
is_local = False
|
||||||
|
parameters = {}
|
||||||
|
sensitive_parameters = []
|
||||||
|
|
||||||
|
# Prevent Django's template engine from calling the backend
|
||||||
|
# class when referenced via DataSource.backend_class
|
||||||
|
do_not_call_in_templates = True
|
||||||
|
|
||||||
|
def __init__(self, url, **kwargs):
|
||||||
|
self.url = url
|
||||||
|
self.params = kwargs
|
||||||
|
self.config = self.init_config()
|
||||||
|
|
||||||
|
def init_config(self):
|
||||||
|
"""
|
||||||
|
A hook to initialize the instance's configuration. The data returned by this method is assigned to the
|
||||||
|
instance's `config` attribute upon initialization, which can be referenced by the `fetch()` method.
|
||||||
|
"""
|
||||||
|
return
|
||||||
|
|
||||||
|
@property
|
||||||
|
def url_scheme(self):
|
||||||
|
return urlparse(self.url).scheme.lower()
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def fetch(self):
|
||||||
|
"""
|
||||||
|
A context manager which performs the following:
|
||||||
|
|
||||||
|
1. Handles all setup and synchronization
|
||||||
|
2. Yields the local path at which data has been replicated
|
||||||
|
3. Performs any necessary cleanup
|
||||||
|
"""
|
||||||
|
raise NotImplemented()
|
@ -8,6 +8,7 @@ from packaging import version
|
|||||||
|
|
||||||
from netbox.registry import registry
|
from netbox.registry import registry
|
||||||
from netbox.search import register_search
|
from netbox.search import register_search
|
||||||
|
from netbox.utils import register_data_backend
|
||||||
from .navigation import *
|
from .navigation import *
|
||||||
from .registration import *
|
from .registration import *
|
||||||
from .templates import *
|
from .templates import *
|
||||||
@ -24,6 +25,7 @@ registry['plugins'].update({
|
|||||||
|
|
||||||
DEFAULT_RESOURCE_PATHS = {
|
DEFAULT_RESOURCE_PATHS = {
|
||||||
'search_indexes': 'search.indexes',
|
'search_indexes': 'search.indexes',
|
||||||
|
'data_backends': 'data_backends.backends',
|
||||||
'graphql_schema': 'graphql.schema',
|
'graphql_schema': 'graphql.schema',
|
||||||
'menu': 'navigation.menu',
|
'menu': 'navigation.menu',
|
||||||
'menu_items': 'navigation.menu_items',
|
'menu_items': 'navigation.menu_items',
|
||||||
@ -70,6 +72,7 @@ class PluginConfig(AppConfig):
|
|||||||
|
|
||||||
# Optional plugin resources
|
# Optional plugin resources
|
||||||
search_indexes = None
|
search_indexes = None
|
||||||
|
data_backends = None
|
||||||
graphql_schema = None
|
graphql_schema = None
|
||||||
menu = None
|
menu = None
|
||||||
menu_items = None
|
menu_items = None
|
||||||
@ -98,6 +101,11 @@ class PluginConfig(AppConfig):
|
|||||||
for idx in search_indexes:
|
for idx in search_indexes:
|
||||||
register_search(idx)
|
register_search(idx)
|
||||||
|
|
||||||
|
# Register data backends (if defined)
|
||||||
|
data_backends = self._load_resource('data_backends') or []
|
||||||
|
for backend in data_backends:
|
||||||
|
register_data_backend()(backend)
|
||||||
|
|
||||||
# Register template content (if defined)
|
# Register template content (if defined)
|
||||||
if template_extensions := self._load_resource('template_extensions'):
|
if template_extensions := self._load_resource('template_extensions'):
|
||||||
register_template_extensions(template_extensions)
|
register_template_extensions(template_extensions)
|
||||||
|
18
netbox/netbox/tests/dummy_plugin/data_backends.py
Normal file
18
netbox/netbox/tests/dummy_plugin/data_backends.py
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
|
from netbox.data_backends import DataBackend
|
||||||
|
|
||||||
|
|
||||||
|
class DummyBackend(DataBackend):
|
||||||
|
name = 'dummy'
|
||||||
|
label = 'Dummy'
|
||||||
|
is_local = True
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def fetch(self):
|
||||||
|
yield '/tmp'
|
||||||
|
|
||||||
|
|
||||||
|
backends = (
|
||||||
|
DummyBackend,
|
||||||
|
)
|
@ -6,6 +6,7 @@ from django.test import Client, TestCase, override_settings
|
|||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
|
|
||||||
from netbox.tests.dummy_plugin import config as dummy_config
|
from netbox.tests.dummy_plugin import config as dummy_config
|
||||||
|
from netbox.tests.dummy_plugin.data_backends import DummyBackend
|
||||||
from netbox.plugins.navigation import PluginMenu
|
from netbox.plugins.navigation import PluginMenu
|
||||||
from netbox.plugins.utils import get_plugin_config
|
from netbox.plugins.utils import get_plugin_config
|
||||||
from netbox.graphql.schema import Query
|
from netbox.graphql.schema import Query
|
||||||
@ -111,6 +112,13 @@ class PluginTest(TestCase):
|
|||||||
"""
|
"""
|
||||||
self.assertIn('netbox.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
|
self.assertIn('netbox.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
|
||||||
|
|
||||||
|
def test_data_backends(self):
|
||||||
|
"""
|
||||||
|
Check registered data backends.
|
||||||
|
"""
|
||||||
|
self.assertIn('dummy', registry['data_backends'])
|
||||||
|
self.assertIs(registry['data_backends']['dummy'], DummyBackend)
|
||||||
|
|
||||||
def test_queues(self):
|
def test_queues(self):
|
||||||
"""
|
"""
|
||||||
Check that plugin queues are registered with the accurate name.
|
Check that plugin queues are registered with the accurate name.
|
||||||
|
26
netbox/netbox/utils.py
Normal file
26
netbox/netbox/utils.py
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
from netbox.registry import registry
|
||||||
|
|
||||||
|
__all__ = (
|
||||||
|
'get_data_backend_choices',
|
||||||
|
'register_data_backend',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_data_backend_choices():
|
||||||
|
return [
|
||||||
|
(None, '---------'),
|
||||||
|
*[
|
||||||
|
(name, cls.label) for name, cls in registry['data_backends'].items()
|
||||||
|
]
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def register_data_backend():
|
||||||
|
"""
|
||||||
|
Decorator for registering a DataBackend class.
|
||||||
|
"""
|
||||||
|
def _wrapper(cls):
|
||||||
|
registry['data_backends'][cls.name] = cls
|
||||||
|
return cls
|
||||||
|
|
||||||
|
return _wrapper
|
@ -58,7 +58,7 @@
|
|||||||
<tr>
|
<tr>
|
||||||
<th scope="row">{% trans "URL" %}</th>
|
<th scope="row">{% trans "URL" %}</th>
|
||||||
<td>
|
<td>
|
||||||
{% if not object.is_local %}
|
{% if not object.type.is_local %}
|
||||||
<a href="{{ object.source_url }}">{{ object.source_url }}</a>
|
<a href="{{ object.source_url }}">{{ object.source_url }}</a>
|
||||||
{% else %}
|
{% else %}
|
||||||
{{ object.source_url }}
|
{{ object.source_url }}
|
||||||
|
Reference in New Issue
Block a user