From 64d8512fc3dd16f4acdf6bf195645389ab9e9449 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 26 Oct 2021 11:39:39 -0400 Subject: [PATCH] Add PAGINATE_COUNT, MAX_PAGE_SIZE --- netbox/extras/admin.py | 6 ++--- netbox/ipam/api/mixins.py | 12 ++++++--- netbox/netbox/api/pagination.py | 12 +++++---- netbox/netbox/config/parameters.py | 14 ++++++++++ netbox/netbox/configuration.example.py | 8 ------ netbox/netbox/settings.py | 37 +++++++------------------- netbox/templates/inc/paginator.html | 2 +- netbox/utilities/paginator.py | 25 ++++++++++++----- netbox/utilities/tests/test_api.py | 4 +-- 9 files changed, 62 insertions(+), 58 deletions(-) diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 6df9c55cf..aaacbb0af 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -25,9 +25,9 @@ class ConfigRevisionAdmin(admin.ModelAdmin): # ('Logging', { # 'fields': ('CHANGELOG_RETENTION',), # }), - # ('Pagination', { - # 'fields': ('MAX_PAGE_SIZE', 'PAGINATE_COUNT'), - # }), + ('Pagination', { + 'fields': ('PAGINATE_COUNT', 'MAX_PAGE_SIZE'), + }), ('Miscellaneous', { 'fields': ('MAINTENANCE_MODE', 'MAPS_URL'), }), diff --git a/netbox/ipam/api/mixins.py b/netbox/ipam/api/mixins.py index c09494d48..43d743a63 100644 --- a/netbox/ipam/api/mixins.py +++ b/netbox/ipam/api/mixins.py @@ -9,6 +9,7 @@ from rest_framework.decorators import action from rest_framework.response import Response from ipam.models import * +from netbox.config import Config from utilities.constants import ADVISORY_LOCK_KEYS from . import serializers @@ -160,12 +161,15 @@ class AvailableIPsMixin: # Determine the maximum number of IPs to return else: + config = Config() + PAGINATE_COUNT = config.PAGINATE_COUNT + MAX_PAGE_SIZE = config.MAX_PAGE_SIZE try: - limit = int(request.query_params.get('limit', settings.PAGINATE_COUNT)) + limit = int(request.query_params.get('limit', PAGINATE_COUNT)) except ValueError: - limit = settings.PAGINATE_COUNT - if settings.MAX_PAGE_SIZE: - limit = min(limit, settings.MAX_PAGE_SIZE) + limit = PAGINATE_COUNT + if MAX_PAGE_SIZE: + limit = min(limit, MAX_PAGE_SIZE) # Calculate available IPs within the parent ip_list = [] diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index e34cb27d0..a57b0bd33 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -2,6 +2,8 @@ from django.conf import settings from django.db.models import QuerySet from rest_framework.pagination import LimitOffsetPagination +from netbox.config import Config + class OptionalLimitOffsetPagination(LimitOffsetPagination): """ @@ -9,6 +11,8 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination): matching a query, but retains the same format as a paginated request. The limit can only be disabled if MAX_PAGE_SIZE has been set to 0 or None. """ + def __init__(self): + self.default_limit = Config().PAGINATE_COUNT def paginate_queryset(self, queryset, request, view=None): @@ -40,11 +44,9 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination): if limit < 0: raise ValueError() # Enforce maximum page size, if defined - if settings.MAX_PAGE_SIZE: - if limit == 0: - return settings.MAX_PAGE_SIZE - else: - return min(limit, settings.MAX_PAGE_SIZE) + MAX_PAGE_SIZE = Config().MAX_PAGE_SIZE + if MAX_PAGE_SIZE: + return MAX_PAGE_SIZE if limit == 0 else min(limit, MAX_PAGE_SIZE) return limit except (KeyError, ValueError): pass diff --git a/netbox/netbox/config/parameters.py b/netbox/netbox/config/parameters.py index 4e77cec0e..be981e3ea 100644 --- a/netbox/netbox/config/parameters.py +++ b/netbox/netbox/config/parameters.py @@ -82,6 +82,20 @@ PARAMS = ( field_kwargs={'base_field': forms.CharField()} ), + # Pagination + ConfigParam( + name='PAGINATE_COUNT', + label='Default page size', + default=50, + field=forms.IntegerField + ), + ConfigParam( + name='MAX_PAGE_SIZE', + label='Maximum page size', + default=1000, + field=forms.IntegerField + ), + # Miscellaneous ConfigParam( name='MAINTENANCE_MODE', diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index c40ea4eff..ffcac8fba 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -158,11 +158,6 @@ LOGIN_REQUIRED = False # re-authenticate. (Default: 1209600 [14 days]) LOGIN_TIMEOUT = None -# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. -# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request -# all objects by specifying "?limit=0". -MAX_PAGE_SIZE = 1000 - # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' @@ -191,9 +186,6 @@ NAPALM_TIMEOUT = 30 # be provided as a dictionary. NAPALM_ARGS = {} -# Determine how many objects to display per page within a list. (Default: 50) -PAGINATE_COUNT = 50 - # Enable installed plugins. Add the name of each plugin to the list. PLUGINS = [] diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index beae4d568..f3526a70b 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -75,6 +75,7 @@ ADMINS = getattr(configuration, 'ADMINS', []) BASE_PATH = getattr(configuration, 'BASE_PATH', '') if BASE_PATH: BASE_PATH = BASE_PATH.strip('/') + '/' # Enforce trailing slash only +CHANGELOG_RETENTION = getattr(configuration, 'CHANGELOG_RETENTION', 90) CORS_ORIGIN_ALLOW_ALL = getattr(configuration, 'CORS_ORIGIN_ALLOW_ALL', False) CORS_ORIGIN_REGEX_WHITELIST = getattr(configuration, 'CORS_ORIGIN_REGEX_WHITELIST', []) CORS_ORIGIN_WHITELIST = getattr(configuration, 'CORS_ORIGIN_WHITELIST', []) @@ -85,12 +86,19 @@ DEBUG = getattr(configuration, 'DEBUG', False) DEVELOPER = getattr(configuration, 'DEVELOPER', False) DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BASE_DIR), 'docs')) EMAIL = getattr(configuration, 'EMAIL', {}) +EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) +GRAPHQL_ENABLED = getattr(configuration, 'GRAPHQL_ENABLED', True) HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None) INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1')) LOGGING = getattr(configuration, 'LOGGING', {}) +LOGIN_PERSISTENCE = getattr(configuration, 'LOGIN_PERSISTENCE', False) +LOGIN_REQUIRED = getattr(configuration, 'LOGIN_REQUIRED', False) +LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') +METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) PLUGINS = getattr(configuration, 'PLUGINS', []) PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {}) +RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None) REMOTE_AUTH_AUTO_CREATE_USER = getattr(configuration, 'REMOTE_AUTH_AUTO_CREATE_USER', False) REMOTE_AUTH_BACKEND = getattr(configuration, 'REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') REMOTE_AUTH_DEFAULT_GROUPS = getattr(configuration, 'REMOTE_AUTH_DEFAULT_GROUPS', []) @@ -122,20 +130,10 @@ for param in PARAMS: if hasattr(configuration, param.name): globals()[param.name] = getattr(configuration, param.name) -CHANGELOG_RETENTION = getattr(configuration, 'CHANGELOG_RETENTION', 90) -EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) -GRAPHQL_ENABLED = getattr(configuration, 'GRAPHQL_ENABLED', True) -LOGIN_PERSISTENCE = getattr(configuration, 'LOGIN_PERSISTENCE', False) -LOGIN_REQUIRED = getattr(configuration, 'LOGIN_REQUIRED', False) -LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) -MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) -METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) NAPALM_PASSWORD = getattr(configuration, 'NAPALM_PASSWORD', '') NAPALM_TIMEOUT = getattr(configuration, 'NAPALM_TIMEOUT', 30) NAPALM_USERNAME = getattr(configuration, 'NAPALM_USERNAME', '') -PAGINATE_COUNT = getattr(configuration, 'PAGINATE_COUNT', 50) -RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None) # Validate update repo URL and timeout if RELEASE_CHECK_URL: @@ -462,7 +460,7 @@ REST_FRAMEWORK = { ), 'DEFAULT_VERSION': REST_FRAMEWORK_VERSION, 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.AcceptHeaderVersioning', - 'PAGE_SIZE': PAGINATE_COUNT, + # 'PAGE_SIZE': PAGINATE_COUNT, 'SCHEMA_COERCE_METHOD_NAMES': { # Default mappings 'retrieve': 'read', @@ -561,23 +559,6 @@ RQ_QUEUES = { } -# -# NetBox internal settings -# - -# Pagination -if MAX_PAGE_SIZE and PAGINATE_COUNT > MAX_PAGE_SIZE: - raise ImproperlyConfigured( - f"PAGINATE_COUNT ({PAGINATE_COUNT}) must be less than or equal to MAX_PAGE_SIZE ({MAX_PAGE_SIZE}), if set." - ) -PER_PAGE_DEFAULTS = [ - 25, 50, 100, 250, 500, 1000 -] -if PAGINATE_COUNT not in PER_PAGE_DEFAULTS: - PER_PAGE_DEFAULTS.append(PAGINATE_COUNT) - PER_PAGE_DEFAULTS = sorted(PER_PAGE_DEFAULTS) - - # # Plugins # diff --git a/netbox/templates/inc/paginator.html b/netbox/templates/inc/paginator.html index c55203be3..8242ffcde 100644 --- a/netbox/templates/inc/paginator.html +++ b/netbox/templates/inc/paginator.html @@ -36,7 +36,7 @@ {% endfor %}
diff --git a/netbox/utilities/paginator.py b/netbox/utilities/paginator.py index e46af4b3e..20f3f6de2 100644 --- a/netbox/utilities/paginator.py +++ b/netbox/utilities/paginator.py @@ -1,8 +1,12 @@ -from django.conf import settings from django.core.paginator import Paginator, Page +from netbox.config import Config + class EnhancedPaginator(Paginator): + default_page_lengths = ( + 25, 50, 100, 250, 500, 1000 + ) def __init__(self, object_list, per_page, orphans=None, **kwargs): @@ -10,9 +14,9 @@ class EnhancedPaginator(Paginator): try: per_page = int(per_page) if per_page < 1: - per_page = settings.PAGINATE_COUNT + per_page = Config().PAGINATE_COUNT except ValueError: - per_page = settings.PAGINATE_COUNT + per_page = Config().PAGINATE_COUNT # Set orphans count based on page size if orphans is None and per_page <= 50: @@ -25,6 +29,11 @@ class EnhancedPaginator(Paginator): def _get_page(self, *args, **kwargs): return EnhancedPage(*args, **kwargs) + def get_page_lengths(self): + if self.per_page not in self.default_page_lengths: + return sorted([*self.default_page_lengths, self.per_page]) + return self.default_page_lengths + class EnhancedPage(Page): @@ -57,17 +66,19 @@ def get_paginate_count(request): Return the lesser of the calculated value and MAX_PAGE_SIZE. """ + config = Config() + if 'per_page' in request.GET: try: per_page = int(request.GET.get('per_page')) if request.user.is_authenticated: request.user.config.set('pagination.per_page', per_page, commit=True) - return min(per_page, settings.MAX_PAGE_SIZE) + return min(per_page, config.MAX_PAGE_SIZE) except ValueError: pass if request.user.is_authenticated: - per_page = request.user.config.get('pagination.per_page', settings.PAGINATE_COUNT) - return min(per_page, settings.MAX_PAGE_SIZE) + per_page = request.user.config.get('pagination.per_page', config.PAGINATE_COUNT) + return min(per_page, config.MAX_PAGE_SIZE) - return min(settings.PAGINATE_COUNT, settings.MAX_PAGE_SIZE) + return min(config.PAGINATE_COUNT, config.MAX_PAGE_SIZE) diff --git a/netbox/utilities/tests/test_api.py b/netbox/utilities/tests/test_api.py index 5b711056a..cc3bb1ddc 100644 --- a/netbox/utilities/tests/test_api.py +++ b/netbox/utilities/tests/test_api.py @@ -1,6 +1,5 @@ import urllib.parse -from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.test import Client, TestCase, override_settings from django.urls import reverse @@ -10,6 +9,7 @@ from dcim.models import Region, Site from extras.choices import CustomFieldTypeChoices from extras.models import CustomField from ipam.models import VLAN +from netbox.config import Config from utilities.testing import APITestCase, disable_warnings @@ -137,7 +137,7 @@ class APIPaginationTestCase(APITestCase): def test_default_page_size(self): response = self.client.get(self.url, format='json', **self.header) - page_size = settings.PAGINATE_COUNT + page_size = Config().PAGINATE_COUNT self.assertLess(page_size, 100, "Default page size not sufficient for data set") self.assertHttpStatus(response, status.HTTP_200_OK)