From 7306d56902f1db5ed82839dc2b21c0b848b5f01a Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Sun, 3 Nov 2019 14:16:12 +0300 Subject: [PATCH 1/6] Add support for S3 storage for media --- netbox/extras/models.py | 11 +++++-- netbox/netbox/configuration.example.py | 22 +++++++++++++ netbox/netbox/settings.py | 45 ++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 170035eb7..d74e14790 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -750,11 +750,18 @@ class ImageAttachment(models.Model): @property def size(self): """ - Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. + Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. When S3 storage is used + ClientError is suppressed instead. """ + from django.conf import settings + if settings.MEDIA_STORAGE and settings.MEDIA_STORAGE['BACKEND'] == 'S3': + from botocore.exceptions import ClientError as AccessError + else: + AccessError = OSError + try: return self.image.size - except OSError: + except AccessError: return None diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index eb092fc29..8826942da 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -130,6 +130,28 @@ MAX_PAGE_SIZE = 1000 # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' +# By default uploaded media is stored on the local filesystem. Use the following configuration to store media on +# AWS S3 or compatible service. +# MEDIA_STORAGE = { +# # Required configuration +# 'BACKEND': 'S3', +# 'ACCESS_KEY_ID': 'Key ID', +# 'SECRET_ACCESS_KEY': 'Secret', +# 'BUCKET_NAME': 'netbox', +# +# # Optional configuration, defaults are shown +# 'REGION_NAME': '', +# 'ENDPOINT_URL': None, +# 'AUTO_CREATE_BUCKET': False, +# 'BUCKET_ACL': 'public-read', +# 'DEFAULT_ACL': 'public-read', +# 'OBJECT_PARAMETERS': { +# 'CacheControl': 'max-age=86400', +# }, +# 'QUERYSTRING_AUTH': True, +# 'QUERYSTRING_EXPIRE': 3600, +# } + # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = False diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 721ec052a..5699962b5 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -77,6 +77,7 @@ LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) MAINTENANCE_MODE = getattr(configuration, 'MAINTENANCE_MODE', False) MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') +MEDIA_STORAGE = getattr(configuration, 'MEDIA_STORAGE', None) METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) NAPALM_PASSWORD = getattr(configuration, 'NAPALM_PASSWORD', '') @@ -113,6 +114,50 @@ DATABASES = { 'default': DATABASE, } +# +# Media storage +# + +if MEDIA_STORAGE: + if not 'BACKEND' in MEDIA_STORAGE: + raise ImproperlyConfigured( + "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." + ) + + if MEDIA_STORAGE['BACKEND'] == 'S3': + # Enforce required configuration parameters + for parameter in ['ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'BUCKET_NAME']: + if parameter not in MEDIA_STORAGE: + raise ImproperlyConfigured( + "Required parameter {} is missing from MEDIA_STORAGE in configuration.py.".format(parameter) + ) + + # Check that django-storages is installed + try: + import storages + except ImportError: + raise ImproperlyConfigured( + "S3 storage has been configured, but django-storages is not installed." + ) + + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + AWS_ACCESS_KEY_ID = MEDIA_STORAGE['ACCESS_KEY_ID'] + AWS_SECRET_ACCESS_KEY = MEDIA_STORAGE['SECRET_ACCESS_KEY'] + AWS_STORAGE_BUCKET_NAME = MEDIA_STORAGE['BUCKET_NAME'] + AWS_S3_REGION_NAME = MEDIA_STORAGE.get('REGION_NAME', None) + AWS_S3_ENDPOINT_URL = MEDIA_STORAGE.get('ENDPOINT_URL', None) + AWS_AUTO_CREATE_BUCKET = MEDIA_STORAGE.get('AUTO_CREATE_BUCKET', False) + AWS_BUCKET_ACL = MEDIA_STORAGE.get('BUCKET_ACL', 'public-read') + AWS_DEFAULT_ACL = MEDIA_STORAGE.get('DEFAULT_ACL', 'public-read') + AWS_S3_OBJECT_PARAMETERS = MEDIA_STORAGE.get('OBJECT_PARAMETERS', { + 'CacheControl': 'max-age=86400', + }) + AWS_QUERYSTRING_AUTH = MEDIA_STORAGE.get('QUERYSTRING_AUTH', True) + AWS_QUERYSTRING_EXPIRE = MEDIA_STORAGE.get('QUERYSTRING_EXPIRE', 3600) + else: + raise ImproperlyConfigured( + "Unknown storage back-end '{}'".format(MEDIA_STORAGE['BACKEND']) + ) # # Redis From a6f2d5b414be5067570b77820aa712d9556b59bf Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Sun, 3 Nov 2019 16:12:39 +0300 Subject: [PATCH 2/6] Fix code for PEP8 --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 5699962b5..c88e9d05e 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -119,7 +119,7 @@ DATABASES = { # if MEDIA_STORAGE: - if not 'BACKEND' in MEDIA_STORAGE: + if 'BACKEND' not in MEDIA_STORAGE: raise ImproperlyConfigured( "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." ) From 02a009b7a72acfffe72b0076dbce74cbcd366e34 Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Fri, 6 Dec 2019 16:32:18 +0100 Subject: [PATCH 3/6] Don't redefine exception but split the code --- netbox/extras/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index d74e14790..4056b0b1a 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -755,13 +755,16 @@ class ImageAttachment(models.Model): """ from django.conf import settings if settings.MEDIA_STORAGE and settings.MEDIA_STORAGE['BACKEND'] == 'S3': - from botocore.exceptions import ClientError as AccessError - else: - AccessError = OSError + # For S3 we need to handle a different exception + from botocore.exceptions import ClientError + try: + return self.image.size + except ClientError: + return None try: return self.image.size - except AccessError: + except OSError: return None From dafa2513e3061c4c8ebb7b49d9c3fa038bf62bd7 Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Sun, 3 Nov 2019 14:16:12 +0300 Subject: [PATCH 4/6] Add support for S3 storage for media --- netbox/extras/models.py | 11 +++++-- netbox/netbox/configuration.example.py | 22 +++++++++++++ netbox/netbox/settings.py | 45 ++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 52dea8674..bf9a445cd 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -611,11 +611,18 @@ class ImageAttachment(models.Model): @property def size(self): """ - Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. + Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. When S3 storage is used + ClientError is suppressed instead. """ + from django.conf import settings + if settings.MEDIA_STORAGE and settings.MEDIA_STORAGE['BACKEND'] == 'S3': + from botocore.exceptions import ClientError as AccessError + else: + AccessError = OSError + try: return self.image.size - except OSError: + except AccessError: return None diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 6b0680da0..26b847b82 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -141,6 +141,28 @@ MAX_PAGE_SIZE = 1000 # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' +# By default uploaded media is stored on the local filesystem. Use the following configuration to store media on +# AWS S3 or compatible service. +# MEDIA_STORAGE = { +# # Required configuration +# 'BACKEND': 'S3', +# 'ACCESS_KEY_ID': 'Key ID', +# 'SECRET_ACCESS_KEY': 'Secret', +# 'BUCKET_NAME': 'netbox', +# +# # Optional configuration, defaults are shown +# 'REGION_NAME': '', +# 'ENDPOINT_URL': None, +# 'AUTO_CREATE_BUCKET': False, +# 'BUCKET_ACL': 'public-read', +# 'DEFAULT_ACL': 'public-read', +# 'OBJECT_PARAMETERS': { +# 'CacheControl': 'max-age=86400', +# }, +# 'QUERYSTRING_AUTH': True, +# 'QUERYSTRING_EXPIRE': 3600, +# } + # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = False diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index eb1b9a523..5941c3130 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -83,6 +83,7 @@ LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) MAINTENANCE_MODE = getattr(configuration, 'MAINTENANCE_MODE', False) MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') +MEDIA_STORAGE = getattr(configuration, 'MEDIA_STORAGE', None) METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) NAPALM_PASSWORD = getattr(configuration, 'NAPALM_PASSWORD', '') @@ -118,6 +119,50 @@ DATABASES = { 'default': DATABASE, } +# +# Media storage +# + +if MEDIA_STORAGE: + if not 'BACKEND' in MEDIA_STORAGE: + raise ImproperlyConfigured( + "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." + ) + + if MEDIA_STORAGE['BACKEND'] == 'S3': + # Enforce required configuration parameters + for parameter in ['ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'BUCKET_NAME']: + if parameter not in MEDIA_STORAGE: + raise ImproperlyConfigured( + "Required parameter {} is missing from MEDIA_STORAGE in configuration.py.".format(parameter) + ) + + # Check that django-storages is installed + try: + import storages + except ImportError: + raise ImproperlyConfigured( + "S3 storage has been configured, but django-storages is not installed." + ) + + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + AWS_ACCESS_KEY_ID = MEDIA_STORAGE['ACCESS_KEY_ID'] + AWS_SECRET_ACCESS_KEY = MEDIA_STORAGE['SECRET_ACCESS_KEY'] + AWS_STORAGE_BUCKET_NAME = MEDIA_STORAGE['BUCKET_NAME'] + AWS_S3_REGION_NAME = MEDIA_STORAGE.get('REGION_NAME', None) + AWS_S3_ENDPOINT_URL = MEDIA_STORAGE.get('ENDPOINT_URL', None) + AWS_AUTO_CREATE_BUCKET = MEDIA_STORAGE.get('AUTO_CREATE_BUCKET', False) + AWS_BUCKET_ACL = MEDIA_STORAGE.get('BUCKET_ACL', 'public-read') + AWS_DEFAULT_ACL = MEDIA_STORAGE.get('DEFAULT_ACL', 'public-read') + AWS_S3_OBJECT_PARAMETERS = MEDIA_STORAGE.get('OBJECT_PARAMETERS', { + 'CacheControl': 'max-age=86400', + }) + AWS_QUERYSTRING_AUTH = MEDIA_STORAGE.get('QUERYSTRING_AUTH', True) + AWS_QUERYSTRING_EXPIRE = MEDIA_STORAGE.get('QUERYSTRING_EXPIRE', 3600) + else: + raise ImproperlyConfigured( + "Unknown storage back-end '{}'".format(MEDIA_STORAGE['BACKEND']) + ) # # Redis From 378883df2be2d783ea12a63a8da65fcb1c198e50 Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Sun, 3 Nov 2019 16:12:39 +0300 Subject: [PATCH 5/6] Fix code for PEP8 --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 5941c3130..208123348 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -124,7 +124,7 @@ DATABASES = { # if MEDIA_STORAGE: - if not 'BACKEND' in MEDIA_STORAGE: + if 'BACKEND' not in MEDIA_STORAGE: raise ImproperlyConfigured( "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." ) From f1e75b0fbb8accdc249bfa1a40edfb3797753771 Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Wed, 11 Dec 2019 16:09:32 +0100 Subject: [PATCH 6/6] Implement storage configuration as suggested by @jeremystretch --- netbox/extras/models.py | 19 +++++----- netbox/netbox/configuration.example.py | 28 +++++---------- netbox/netbox/settings.py | 50 +++++++------------------- 3 files changed, 29 insertions(+), 68 deletions(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index b82fbb812..09fd63772 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -611,21 +611,20 @@ class ImageAttachment(models.Model): @property def size(self): """ - Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. When S3 storage is used - ClientError is suppressed instead. + Wrapper around `image.size` to suppress an OSError in case the file is inaccessible. Also opportunistically + catch other exceptions that we know other storage back-ends to throw. """ - from django.conf import settings - if settings.MEDIA_STORAGE and settings.MEDIA_STORAGE['BACKEND'] == 'S3': - # For S3 we need to handle a different exception + expected_exceptions = [OSError] + + try: from botocore.exceptions import ClientError - try: - return self.image.size - except ClientError: - return None + expected_exceptions.append(ClientError) + except ImportError: + pass try: return self.image.size - except OSError: + except tuple(expected_exceptions): return None diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 26b847b82..3bd271581 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -141,26 +141,14 @@ MAX_PAGE_SIZE = 1000 # the default value of this setting is derived from the installed location. # MEDIA_ROOT = '/opt/netbox/netbox/media' -# By default uploaded media is stored on the local filesystem. Use the following configuration to store media on -# AWS S3 or compatible service. -# MEDIA_STORAGE = { -# # Required configuration -# 'BACKEND': 'S3', -# 'ACCESS_KEY_ID': 'Key ID', -# 'SECRET_ACCESS_KEY': 'Secret', -# 'BUCKET_NAME': 'netbox', -# -# # Optional configuration, defaults are shown -# 'REGION_NAME': '', -# 'ENDPOINT_URL': None, -# 'AUTO_CREATE_BUCKET': False, -# 'BUCKET_ACL': 'public-read', -# 'DEFAULT_ACL': 'public-read', -# 'OBJECT_PARAMETERS': { -# 'CacheControl': 'max-age=86400', -# }, -# 'QUERYSTRING_AUTH': True, -# 'QUERYSTRING_EXPIRE': 3600, +# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', # } # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 208123348..3a5ea4069 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -83,7 +83,8 @@ LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) MAINTENANCE_MODE = getattr(configuration, 'MAINTENANCE_MODE', False) MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') -MEDIA_STORAGE = getattr(configuration, 'MEDIA_STORAGE', None) +STORAGE_BACKEND = getattr(configuration, 'STORAGE_BACKEND', None) +STORAGE_CONFIG = getattr(configuration, 'STORAGE_CONFIG', {}) METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) NAPALM_PASSWORD = getattr(configuration, 'NAPALM_PASSWORD', '') @@ -123,46 +124,19 @@ DATABASES = { # Media storage # -if MEDIA_STORAGE: - if 'BACKEND' not in MEDIA_STORAGE: - raise ImproperlyConfigured( - "Required parameter BACKEND is missing from MEDIA_STORAGE in configuration.py." - ) +if STORAGE_BACKEND is not None: + DEFAULT_FILE_STORAGE = STORAGE_BACKEND - if MEDIA_STORAGE['BACKEND'] == 'S3': - # Enforce required configuration parameters - for parameter in ['ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'BUCKET_NAME']: - if parameter not in MEDIA_STORAGE: - raise ImproperlyConfigured( - "Required parameter {} is missing from MEDIA_STORAGE in configuration.py.".format(parameter) - ) + if STORAGE_BACKEND.startswith('storages.'): + # Monkey-patch Django-storages to also fetch settings from STORAGE_CONFIG + import storages.utils - # Check that django-storages is installed - try: - import storages - except ImportError: - raise ImproperlyConfigured( - "S3 storage has been configured, but django-storages is not installed." - ) + def _setting(name, default=None): + if name in STORAGE_CONFIG: + return STORAGE_CONFIG[name] + return globals().get(name, default) - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - AWS_ACCESS_KEY_ID = MEDIA_STORAGE['ACCESS_KEY_ID'] - AWS_SECRET_ACCESS_KEY = MEDIA_STORAGE['SECRET_ACCESS_KEY'] - AWS_STORAGE_BUCKET_NAME = MEDIA_STORAGE['BUCKET_NAME'] - AWS_S3_REGION_NAME = MEDIA_STORAGE.get('REGION_NAME', None) - AWS_S3_ENDPOINT_URL = MEDIA_STORAGE.get('ENDPOINT_URL', None) - AWS_AUTO_CREATE_BUCKET = MEDIA_STORAGE.get('AUTO_CREATE_BUCKET', False) - AWS_BUCKET_ACL = MEDIA_STORAGE.get('BUCKET_ACL', 'public-read') - AWS_DEFAULT_ACL = MEDIA_STORAGE.get('DEFAULT_ACL', 'public-read') - AWS_S3_OBJECT_PARAMETERS = MEDIA_STORAGE.get('OBJECT_PARAMETERS', { - 'CacheControl': 'max-age=86400', - }) - AWS_QUERYSTRING_AUTH = MEDIA_STORAGE.get('QUERYSTRING_AUTH', True) - AWS_QUERYSTRING_EXPIRE = MEDIA_STORAGE.get('QUERYSTRING_EXPIRE', 3600) - else: - raise ImproperlyConfigured( - "Unknown storage back-end '{}'".format(MEDIA_STORAGE['BACKEND']) - ) + storages.utils.setting = _setting # # Redis