From f1e75b0fbb8accdc249bfa1a40edfb3797753771 Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Wed, 11 Dec 2019 16:09:32 +0100 Subject: [PATCH] Implement storage configuration as suggested by @jeremystretch --- netbox/extras/models.py | 19 +++++----- netbox/netbox/configuration.example.py | 28 +++++---------- netbox/netbox/settings.py | 50 +++++++------------------- 3 files changed, 29 insertions(+), 68 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index b82fbb812..09fd63772 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -611,21 +611,20 @@ class ImageAttachment(models.Model): @property def size(self): """ - Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. When S3 storage is used - ClientError is suppressed instead. + Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. Also opportunistically + catch other exceptions that we know other storage back-ends to throw. """ - from django.conf import settings - if settings.MEDIA_STORAGE and settings.MEDIA_STORAGE['BACKEND'] == 'S3': - # For S3 we need to handle a different exception + expected_exceptions = [OSError] + + try: from botocore.exceptions import ClientError - try: - return self.image.size - except ClientError: - return None + expected_exceptions.append(ClientError) + except ImportError: + pass try: return self.image.size - except OSError: + except tuple(expected_exceptions): return None diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 26b847b82..3bd271581 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -141,26 +141,14 @@ MAX_PAGE_SIZE = 1000 # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' -# By default uploaded media is stored on the local filesystem. Use the following configuration to store media on -# AWS S3 or compatible service. -# MEDIA_STORAGE = { -# # Required configuration -# 'BACKEND': 'S3', -# 'ACCESS_KEY_ID': 'Key ID', -# 'SECRET_ACCESS_KEY': 'Secret', -# 'BUCKET_NAME': 'netbox', -# -# # Optional configuration, defaults are shown -# 'REGION_NAME': '', -# 'ENDPOINT_URL': None, -# 'AUTO_CREATE_BUCKET': False, -# 'BUCKET_ACL': 'public-read', -# 'DEFAULT_ACL': 'public-read', -# 'OBJECT_PARAMETERS': { -# 'CacheControl': 'max-age=86400', -# }, -# 'QUERYSTRING_AUTH': True, -# 'QUERYSTRING_EXPIRE': 3600, +# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', # } # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 208123348..3a5ea4069 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -83,7 +83,8 @@ LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) MAINTENANCE_MODE = getattr(configuration, 'MAINTENANCE_MODE', False) MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') -MEDIA_STORAGE = getattr(configuration, 'MEDIA_STORAGE', None) +STORAGE_BACKEND = getattr(configuration, 'STORAGE_BACKEND', None) +STORAGE_CONFIG = getattr(configuration, 'STORAGE_CONFIG', {}) METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) NAPALM_PASSWORD = getattr(configuration, 'NAPALM_PASSWORD', '') @@ -123,46 +124,19 @@ DATABASES = { # Media storage # -if MEDIA_STORAGE: - if 'BACKEND' not in MEDIA_STORAGE: - raise ImproperlyConfigured( - "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." - ) +if STORAGE_BACKEND is not None: + DEFAULT_FILE_STORAGE = STORAGE_BACKEND - if MEDIA_STORAGE['BACKEND'] == 'S3': - # Enforce required configuration parameters - for parameter in ['ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'BUCKET_NAME']: - if parameter not in MEDIA_STORAGE: - raise ImproperlyConfigured( - "Required parameter {} is missing from MEDIA_STORAGE in configuration.py.".format(parameter) - ) + if STORAGE_BACKEND.startswith('storages.'): + # Monkey-patch Django-storages to also fetch settings from STORAGE_CONFIG + import storages.utils - # Check that django-storages is installed - try: - import storages - except ImportError: - raise ImproperlyConfigured( - "S3 storage has been configured, but django-storages is not installed." - ) + def _setting(name, default=None): + if name in STORAGE_CONFIG: + return STORAGE_CONFIG[name] + return globals().get(name, default) - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - AWS_ACCESS_KEY_ID = MEDIA_STORAGE['ACCESS_KEY_ID'] - AWS_SECRET_ACCESS_KEY = MEDIA_STORAGE['SECRET_ACCESS_KEY'] - AWS_STORAGE_BUCKET_NAME = MEDIA_STORAGE['BUCKET_NAME'] - AWS_S3_REGION_NAME = MEDIA_STORAGE.get('REGION_NAME', None) - AWS_S3_ENDPOINT_URL = MEDIA_STORAGE.get('ENDPOINT_URL', None) - AWS_AUTO_CREATE_BUCKET = MEDIA_STORAGE.get('AUTO_CREATE_BUCKET', False) - AWS_BUCKET_ACL = MEDIA_STORAGE.get('BUCKET_ACL', 'public-read') - AWS_DEFAULT_ACL = MEDIA_STORAGE.get('DEFAULT_ACL', 'public-read') - AWS_S3_OBJECT_PARAMETERS = MEDIA_STORAGE.get('OBJECT_PARAMETERS', { - 'CacheControl': 'max-age=86400', - }) - AWS_QUERYSTRING_AUTH = MEDIA_STORAGE.get('QUERYSTRING_AUTH', True) - AWS_QUERYSTRING_EXPIRE = MEDIA_STORAGE.get('QUERYSTRING_EXPIRE', 3600) - else: - raise ImproperlyConfigured( - "Unknown storage back-end '{}'".format(MEDIA_STORAGE['BACKEND']) - ) + storages.utils.setting = _setting # # Redis