From d87ec82fe34d92a84ee6f2a97ba0a87a53eed015 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 11 Jun 2021 10:44:17 -0400 Subject: [PATCH] Closes #6590: Introduce a nightly housekeeping command to clear expired sessions and change records --- contrib/netbox-housekeeping.sh | 9 ++++ docs/administration/housekeeping.md | 10 ++++ docs/installation/3-netbox.md | 12 +++++ docs/installation/upgrading.md | 11 +++- docs/release-notes/version-3.0.md | 1 + mkdocs.yml | 1 + .../management/commands/housekeeping.py | 51 +++++++++++++++++++ netbox/extras/signals.py | 10 ---- 8 files changed, 93 insertions(+), 12 deletions(-) create mode 100644 contrib/netbox-housekeeping.sh create mode 100644 docs/administration/housekeeping.md create mode 100644 netbox/extras/management/commands/housekeeping.py diff --git a/contrib/netbox-housekeeping.sh b/contrib/netbox-housekeeping.sh new file mode 100644 index 000000000..5b1c46c5e --- /dev/null +++ b/contrib/netbox-housekeeping.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# This shell script invokes NetBox's housekeeping management command, which +# intended to be run nightly. This script can be copied into your system's +# daily cron directory (e.g. /etc/cron.daily), or referenced directly from +# within the cron configuration file. +# +# If NetBox has been installed into a nonstandard location, update the paths +# below. +/opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py housekeeping diff --git a/docs/administration/housekeeping.md b/docs/administration/housekeeping.md new file mode 100644 index 000000000..c562613eb --- /dev/null +++ b/docs/administration/housekeeping.md @@ -0,0 +1,10 @@ +# Housekeeping + +NetBox includes a `housekeeping` management command that should be run nightly. This command handles: + +* Clearing expired authentication sessions from the database +* Deleting changelog records older than the configured [retention time](../configuration/optional-settings.md#changelog_retention) + +This command can be invoked directly, or by using the shell script provided at `/opt/netbox/contrib/netbox-housekeeping.sh`. This script can be copied into your cron scheduler's daily jobs directory (e.g. `/etc/cron.daily`) or referenced directly within the cron configuration file. + +The `housekeeping` command can also be run manually at any time: Running the command outside of scheduled execution times will not interfere with its operation. diff --git a/docs/installation/3-netbox.md b/docs/installation/3-netbox.md index 7a8e0bc80..9ffa19e3a 100644 --- a/docs/installation/3-netbox.md +++ b/docs/installation/3-netbox.md @@ -247,6 +247,18 @@ Password (again): Superuser created successfully. ``` +## Schedule the Housekeeping Task + +NetBox includes a `housekeeping` management command that handles some recurring cleanup tasks, such as clearing out old sessions and expired change records. Although this command may be run manually, it is recommended to configure a scheduled job using the system's `cron` daemon or a similar utility. + +A shell script which invokes this command is included at `contrib/netbox-housekeeping.sh`. It can be copied to your system's daily cron task directory, or included within the crontab directly. (If installing NetBox into a nonstandard path, be sure to update the system paths within this script first.) + +```shell +cp /opt/netbox/contrib/netbox-housekeeping.sh /etc/cron.daily/ +``` + +See the [housekeeping documentation](../administration/housekeeping.md) for further details. + ## Test the Application At this point, we should be able to run NetBox's development server for testing. We can check by starting a development instance: diff --git a/docs/installation/upgrading.md b/docs/installation/upgrading.md index e824ad7ab..0bf356ae4 100644 --- a/docs/installation/upgrading.md +++ b/docs/installation/upgrading.md @@ -102,5 +102,12 @@ Finally, restart the gunicorn and RQ services: sudo systemctl restart netbox netbox-rq ``` -!!! note - If upgrading from an installation that uses supervisord, please see the instructions for [migrating to systemd](migrating-to-systemd.md). The use of supervisord is no longer supported. +## Verify Housekeeping Scheduling + +If upgrading from a release prior to NetBox v3.0, check that a cron task (or similar scheduled process) has been configured to run NetBox's nightly housekeeping command. A shell script which invokes this command is included at `contrib/netbox-housekeeping.sh`. It can be copied to your system's daily cron task directory, or included within the crontab directly. (If NetBox has been installed in a nonstandard path, be sure to update the system paths within this script first.) + +```shell +cp /opt/netbox/contrib/netbox-housekeeping.sh /etc/cron.daily/ +``` + +See the [housekeeping documentation](../administration/housekeeping.md) for further details. diff --git a/docs/release-notes/version-3.0.md b/docs/release-notes/version-3.0.md index 2bec542ed..2faf14c80 100644 --- a/docs/release-notes/version-3.0.md +++ b/docs/release-notes/version-3.0.md @@ -39,6 +39,7 @@ CustomValidator can also be subclassed to enforce more complex logic by overridi * [#4609](https://github.com/netbox-community/netbox/issues/4609) - Allow marking prefixes as fully utilized * [#5806](https://github.com/netbox-community/netbox/issues/5806) - Add kilometer and mile as choices for cable length unit * [#6154](https://github.com/netbox-community/netbox/issues/6154) - Allow decimal values for cable lengths +* [#6590](https://github.com/netbox-community/netbox/issues/6590) - Introduce a nightly housekeeping command to clear expired sessions and change records ### Other Changes diff --git a/mkdocs.yml b/mkdocs.yml index 9bb96065a..875d04d3e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,7 @@ nav: - Developing Plugins: 'plugins/development.md' - Administration: - Permissions: 'administration/permissions.md' + - Housekeeping: 'administration/housekeeping.md' - Replicating NetBox: 'administration/replicating-netbox.md' - NetBox Shell: 'administration/netbox-shell.md' - REST API: diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py new file mode 100644 index 000000000..53dd74b96 --- /dev/null +++ b/netbox/extras/management/commands/housekeeping.py @@ -0,0 +1,51 @@ +from datetime import timedelta +from importlib import import_module + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.db import DEFAULT_DB_ALIAS +from django.utils import timezone + +from extras.models import ObjectChange + + +class Command(BaseCommand): + help = "Perform nightly housekeeping tasks. (This command can be run at any time.)" + + def handle(self, *args, **options): + + # Clear expired authentication sessions (essentially replicating the `clearsessions` command) + self.stdout.write("[*] Clearing expired authentication sessions") + if options['verbosity'] >= 2: + self.stdout.write(f"\tConfigured session engine: {settings.SESSION_ENGINE}") + engine = import_module(settings.SESSION_ENGINE) + try: + engine.SessionStore.clear_expired() + self.stdout.write("\tSessions cleared.", self.style.SUCCESS) + except NotImplementedError: + self.stdout.write( + f"\tThe configured session engine ({settings.SESSION_ENGINE}) does not support " + f"clearing sessions; skipping." + ) + + # Delete expired ObjectRecords + self.stdout.write("[*] Checking for expired changelog records") + if settings.CHANGELOG_RETENTION: + cutoff = timezone.now() - timedelta(days=settings.CHANGELOG_RETENTION) + if options['verbosity'] >= 2: + self.stdout.write(f"Retention period: {settings.CHANGELOG_RETENTION} days") + self.stdout.write(f"\tCut-off time: {cutoff}") + expired_records = ObjectChange.objects.filter(time__lt=cutoff).count() + if expired_records: + self.stdout.write(f"\tDeleting {expired_records} expired records... ", self.style.WARNING, ending="") + self.stdout.flush() + ObjectChange.objects.filter(time__lt=cutoff)._raw_delete(using=DEFAULT_DB_ALIAS) + self.stdout.write("Done.", self.style.WARNING) + else: + self.stdout.write("\tNo expired records found.") + else: + self.stdout.write( + f"\tSkipping: No retention period specified (CHANGELOG_RETENTION = {settings.CHANGELOG_RETENTION})" + ) + + self.stdout.write("Finished.", self.style.SUCCESS) diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index 68c999946..5fd68cfff 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -1,13 +1,8 @@ -import random -from datetime import timedelta - from cacheops.signals import cache_invalidated, cache_read from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django.db import DEFAULT_DB_ALIAS from django.db.models.signals import m2m_changed, post_save, pre_delete from django.dispatch import receiver -from django.utils import timezone from django_prometheus.models import model_deletes, model_inserts, model_updates from prometheus_client import Counter @@ -79,11 +74,6 @@ def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs): elif action == ObjectChangeActionChoices.ACTION_UPDATE: model_updates.labels(instance._meta.model_name).inc() - # Housekeeping: 0.1% chance of clearing out expired ObjectChanges - if settings.CHANGELOG_RETENTION and random.randint(1, 1000) == 1: - cutoff = timezone.now() - timedelta(days=settings.CHANGELOG_RETENTION) - ObjectChange.objects.filter(time__lt=cutoff)._raw_delete(using=DEFAULT_DB_ALIAS) - def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs): """