From 7306d56902f1db5ed82839dc2b21c0b848b5f01a Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Sun, 3 Nov 2019 14:16:12 +0300 Subject: [PATCH 1/3] Add support for S3 storage for media --- netbox/extras/models.py | 11 +++++-- netbox/netbox/configuration.example.py | 22 +++++++++++++ netbox/netbox/settings.py | 45 ++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 170035eb7..d74e14790 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -750,11 +750,18 @@ class ImageAttachment(models.Model): @property def size(self): """ - Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. + Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. When S3 storage is used + ClientError is suppressed instead. """ + from django.conf import settings + if settings.MEDIA_STORAGE and settings.MEDIA_STORAGE['BACKEND'] == 'S3': + from botocore.exceptions import ClientError as AccessError + else: + AccessError = OSError + try: return self.image.size - except OSError: + except AccessError: return None diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index eb092fc29..8826942da 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -130,6 +130,28 @@ MAX_PAGE_SIZE = 1000 # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' +# By default uploaded media is stored on the local filesystem. Use the following configuration to store media on +# AWS S3 or compatible service. +# MEDIA_STORAGE = { +# # Required configuration +# 'BACKEND': 'S3', +# 'ACCESS_KEY_ID': 'Key ID', +# 'SECRET_ACCESS_KEY': 'Secret', +# 'BUCKET_NAME': 'netbox', +# +# # Optional configuration, defaults are shown +# 'REGION_NAME': '', +# 'ENDPOINT_URL': None, +# 'AUTO_CREATE_BUCKET': False, +# 'BUCKET_ACL': 'public-read', +# 'DEFAULT_ACL': 'public-read', +# 'OBJECT_PARAMETERS': { +# 'CacheControl': 'max-age=86400', +# }, +# 'QUERYSTRING_AUTH': True, +# 'QUERYSTRING_EXPIRE': 3600, +# } + # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = False diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 721ec052a..5699962b5 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -77,6 +77,7 @@ LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) MAINTENANCE_MODE = getattr(configuration, 'MAINTENANCE_MODE', False) MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') +MEDIA_STORAGE = getattr(configuration, 'MEDIA_STORAGE', None) METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) NAPALM_PASSWORD = getattr(configuration, 'NAPALM_PASSWORD', '') @@ -113,6 +114,50 @@ DATABASES = { 'default': DATABASE, } +# +# Media storage +# + +if MEDIA_STORAGE: + if not 'BACKEND' in MEDIA_STORAGE: + raise ImproperlyConfigured( + "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." + ) + + if MEDIA_STORAGE['BACKEND'] == 'S3': + # Enforce required configuration parameters + for parameter in ['ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'BUCKET_NAME']: + if parameter not in MEDIA_STORAGE: + raise ImproperlyConfigured( + "Required parameter {} is missing from MEDIA_STORAGE in configuration.py.".format(parameter) + ) + + # Check that django-storages is installed + try: + import storages + except ImportError: + raise ImproperlyConfigured( + "S3 storage has been configured, but django-storages is not installed." + ) + + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + AWS_ACCESS_KEY_ID = MEDIA_STORAGE['ACCESS_KEY_ID'] + AWS_SECRET_ACCESS_KEY = MEDIA_STORAGE['SECRET_ACCESS_KEY'] + AWS_STORAGE_BUCKET_NAME = MEDIA_STORAGE['BUCKET_NAME'] + AWS_S3_REGION_NAME = MEDIA_STORAGE.get('REGION_NAME', None) + AWS_S3_ENDPOINT_URL = MEDIA_STORAGE.get('ENDPOINT_URL', None) + AWS_AUTO_CREATE_BUCKET = MEDIA_STORAGE.get('AUTO_CREATE_BUCKET', False) + AWS_BUCKET_ACL = MEDIA_STORAGE.get('BUCKET_ACL', 'public-read') + AWS_DEFAULT_ACL = MEDIA_STORAGE.get('DEFAULT_ACL', 'public-read') + AWS_S3_OBJECT_PARAMETERS = MEDIA_STORAGE.get('OBJECT_PARAMETERS', { + 'CacheControl': 'max-age=86400', + }) + AWS_QUERYSTRING_AUTH = MEDIA_STORAGE.get('QUERYSTRING_AUTH', True) + AWS_QUERYSTRING_EXPIRE = MEDIA_STORAGE.get('QUERYSTRING_EXPIRE', 3600) + else: + raise ImproperlyConfigured( + "Unknown storage back-end '{}'".format(MEDIA_STORAGE['BACKEND']) + ) # # Redis From a6f2d5b414be5067570b77820aa712d9556b59bf Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Sun, 3 Nov 2019 16:12:39 +0300 Subject: [PATCH 2/3] Fix code for PEP8 --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 5699962b5..c88e9d05e 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -119,7 +119,7 @@ DATABASES = { # if MEDIA_STORAGE: - if not 'BACKEND' in MEDIA_STORAGE: + if 'BACKEND' not in MEDIA_STORAGE: raise ImproperlyConfigured( "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." ) From 02a009b7a72acfffe72b0076dbce74cbcd366e34 Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Fri, 6 Dec 2019 16:32:18 +0100 Subject: [PATCH 3/3] Don't redefine exception but split the code --- netbox/extras/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index d74e14790..4056b0b1a 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -755,13 +755,16 @@ class ImageAttachment(models.Model): """ from django.conf import settings if settings.MEDIA_STORAGE and settings.MEDIA_STORAGE['BACKEND'] == 'S3': - from botocore.exceptions import ClientError as AccessError - else: - AccessError = OSError + # For S3 we need to handle a different exception + from botocore.exceptions import ClientError + try: + return self.image.size + except ClientError: + return None try: return self.image.size - except AccessError: + except OSError: return None