Merge pull request #4001 from steffann/738-automatically-check-for-new-releases

Fixes: #738: Automatically check for new versions

docs/configuration/optional-settings.md

@@ -157,6 +157,24 @@ Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce uni
 
 ---
 
+## UPDATE_REPO_URL
+
+Default: None
+
+The releases of this repository are checked to detect new releases, which are shown on the home page of the web interface. You can change this to your own fork of the NetBox repository, or set it to `None` to disable the check. The URL provided **must** be compatible with the GitHub API.
+
+Use `'https://api.github.com/repos/netbox-community/netbox'` to check for release in the official NetBox repository.
+
+---
+
+## UPDATE_CACHE_TIMEOUT
+
+Default: 86,400 (24 hours)
+
+The number of seconds to retain the latest version that is fetched from the GitHub API before automatically invalidating it and fetching it from the API again. This must be set to at least one hour (3600 seconds).
+
+---
+
 ## LOGGING
 
 By default, all messages of INFO severity or higher will be logged to the console. Additionally, if `DEBUG` is False and email access has been configured, ERROR and CRITICAL messages will be emailed to the users defined in `ADMINS`.

netbox/netbox/configuration.example.py

@@ -124,6 +124,14 @@ EXEMPT_VIEW_PERMISSIONS = [
     # 'ipam.prefix',
 ]
 
+# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
+# version check or use the URL below to check for release in the official NetBox repository.
+UPDATE_REPO_URL = None
+# UPDATE_REPO_URL = 'https://api.github.com/repos/netbox-community/netbox'
+
+# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
+UPDATE_CACHE_TIMEOUT = 24 * 3600
+
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
 #   https://docs.djangoproject.com/en/stable/topics/logging/
 LOGGING = {}

netbox/netbox/settings.py

@@ -1,8 +1,10 @@
 import logging
 import os
 import platform
+import re
 import socket
 import warnings
+from urllib.parse import urlsplit
 
 from django.contrib.messages import constants as messages
 from django.core.exceptions import ImproperlyConfigured
@@ -78,6 +80,8 @@ DEVELOPER = getattr(configuration, 'DEVELOPER', False)
 EMAIL = getattr(configuration, 'EMAIL', {})
 ENFORCE_GLOBAL_UNIQUE = getattr(configuration, 'ENFORCE_GLOBAL_UNIQUE', False)
 EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', [])
+UPDATE_REPO_URL = getattr(configuration, 'UPDATE_REPO_URL', None)
+UPDATE_CACHE_TIMEOUT = getattr(configuration, 'UPDATE_CACHE_TIMEOUT', 24 * 3600)
 LOGGING = getattr(configuration, 'LOGGING', {})
 LOGIN_REQUIRED = getattr(configuration, 'LOGIN_REQUIRED', False)
 LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None)
@@ -302,6 +306,31 @@ AUTHENTICATION_BACKENDS = [
     'utilities.auth_backends.ViewExemptModelBackend',
 ]
 
+# GitHub repository for version check
+if UPDATE_REPO_URL:
+    UPDATE_REPO_URL = UPDATE_REPO_URL.rstrip('/')
+    try:
+        scheme, netloc, path, query, fragment = urlsplit(UPDATE_REPO_URL)
+    except ValueError:
+        raise ImproperlyConfigured("UPDATE_REPO_URL must be a valid URL")
+
+    if scheme not in ('http', 'https'):
+        raise ImproperlyConfigured("UPDATE_REPO_URL must be a valid http:// or https:// URL")
+
+    if not re.fullmatch(r'/repos/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', path):
+        raise ImproperlyConfigured(
+            "GITHUB_REPOSITORY must contain the base URL of the GitHub API in a form like "
+            "'https://api.github.com/repos/<owner>/<repository>'"
+        )
+
+    # Don't allow ? (query) and # (fragment) in the URL
+    if query or fragment:
+        raise ImproperlyConfigured("UPDATE_REPO_URL may not contain a ? (query) or # (fragment)")
+
+# Enforce a cache timeout of at least an hour to protect GitHub
+if UPDATE_CACHE_TIMEOUT < 3600:
+    raise ImproperlyConfigured("UPDATE_CACHE_TIMEOUT has to be at least 3600 seconds (1 hour)")
+
 # Internationalization
 LANGUAGE_CODE = 'en-us'
 USE_I18N = True

netbox/netbox/tests/test_get_releases.py

@@ -0,0 +1,160 @@
+from io import BytesIO
+from logging import ERROR
+from unittest.mock import Mock, patch
+
+import requests
+from cacheops import CacheMiss, RedisCache
+from django.test import SimpleTestCase, override_settings
+from packaging.version import Version
+from requests import Response
+
+from utilities.background_tasks import get_releases
+
+
+def successful_github_response(url, *_args, **_kwargs):
+    r = Response()
+    r.url = url
+    r.status_code = 200
+    r.reason = 'OK'
+    r.headers = {
+        'Content-Type': 'application/json; charset=utf-8',
+    }
+    r.raw = BytesIO(b'''[
+        {
+            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.7.8",
+            "tag_name": "v2.7.8",
+            "prerelease": false
+        },
+        {
+            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1",
+            "tag_name": "v2.6-beta1",
+            "prerelease": true
+        },
+        {
+            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.5.9",
+            "tag_name": "v2.5.9",
+            "prerelease": false
+        }
+    ]
+    ''')
+    return r
+
+
+def unsuccessful_github_response(url, *_args, **_kwargs):
+    r = Response()
+    r.url = url
+    r.status_code = 404
+    r.reason = 'Not Found'
+    r.headers = {
+        'Content-Type': 'application/json; charset=utf-8',
+    }
+    r.raw = BytesIO(b'''{
+        "message": "Not Found",
+        "documentation_url": "https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository"
+    }
+    ''')
+    return r
+
+
+@override_settings(UPDATE_REPO_URL='https://localhost/unittest', UPDATE_CACHE_TIMEOUT=160876)
+class GetReleasesTestCase(SimpleTestCase):
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_pre_releases(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.side_effect = CacheMiss()
+        dummy_request_get.side_effect = successful_github_response
+
+        releases = get_releases(pre_releases=True)
+
+        # Check result
+        self.assertListEqual(releases, [
+            (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
+            (Version('2.6b1'), 'https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1'),
+            (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
+        ])
+
+        # Check if correct request is made
+        dummy_request_get.assert_called_once()
+        dummy_request_get.assert_called_with('https://localhost/unittest/releases',
+                                             headers={
+                                                 'Accept': 'application/vnd.github.v3+json'
+                                             })
+
+        # Check if result is put in cache
+        dummy_cache_set.assert_called_once()
+        dummy_cache_set.assert_called_with('netbox_releases', releases, 160876)
+
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_no_pre_releases(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.side_effect = CacheMiss()
+        dummy_request_get.side_effect = successful_github_response
+
+        releases = get_releases(pre_releases=False)
+
+        # Check result
+        self.assertListEqual(releases, [
+            (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
+            (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
+        ])
+
+        # Check if correct request is made
+        dummy_request_get.assert_called_once()
+        dummy_request_get.assert_called_with('https://localhost/unittest/releases',
+                                             headers={
+                                                 'Accept': 'application/vnd.github.v3+json'
+                                             })
+
+        # Check if result is put in cache
+        dummy_cache_set.assert_called_once()
+        dummy_cache_set.assert_called_with('netbox_releases', releases, 160876)
+
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_failed_request(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.side_effect = CacheMiss()
+        dummy_request_get.side_effect = unsuccessful_github_response
+
+        with self.assertLogs(level=ERROR) as cm:
+            releases = get_releases()
+
+        # Check log entry
+        self.assertEqual(len(cm.output), 1)
+        log_output = cm.output[0]
+        last_log_line = log_output.split('\n')[-1]
+        self.assertRegex(last_log_line, '404 .* Not Found')
+
+        # Check result
+        self.assertListEqual(releases, [])
+
+        # Check if correct request is made
+        dummy_request_get.assert_called_once()
+        dummy_request_get.assert_called_with('https://localhost/unittest/releases',
+                                             headers={
+                                                 'Accept': 'application/vnd.github.v3+json'
+                                             })
+
+        # Check if failure is put in cache
+        dummy_cache_set.assert_called_once()
+        dummy_cache_set.assert_called_with('netbox_releases_no_retry', 'https://localhost/unittest/releases', 900)
+
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_blocked_retry(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.return_value = 'https://localhost/unittest/releases'
+        dummy_request_get.side_effect = successful_github_response
+
+        releases = get_releases()
+
+        # Check result
+        self.assertListEqual(releases, [])
+
+        # Check if request is NOT made
+        dummy_request_get.assert_not_called()
+
+        # Check if cache is not updated
+        dummy_cache_set.assert_not_called()

netbox/netbox/views.py

@@ -1,8 +1,10 @@
 from collections import OrderedDict
 
-from django.db.models import Count, F
+from django.conf import settings
+from django.db.models import Count, F, OuterRef, Subquery
 from django.shortcuts import render
 from django.views.generic import View
+from packaging import version
 from rest_framework.response import Response
 from rest_framework.reverse import reverse
 from rest_framework.views import APIView
@@ -31,6 +33,7 @@ from secrets.tables import SecretTable
 from tenancy.filters import TenantFilterSet
 from tenancy.models import Tenant
 from tenancy.tables import TenantTable
+from utilities.releases import get_latest_release
 from virtualization.filters import ClusterFilterSet, VirtualMachineFilterSet
 from virtualization.models import Cluster, VirtualMachine
 from virtualization.tables import ClusterTable, VirtualMachineDetailTable
@@ -240,11 +243,25 @@ class HomeView(View):
 
         }
 
+        new_release = None
+        new_release_url = None
+
+        if request.user.is_staff or request.user.is_superuser:
+            # Only check for new releases if the current user might be able to do anything about it
+            latest_release, github_url = get_latest_release()
+            if isinstance(latest_release, version.Version):
+                current_version = version.parse(settings.VERSION)
+                if latest_release > current_version:
+                    new_release = str(latest_release)
+                    new_release_url = github_url
+
         return render(request, self.template_name, {
             'search_form': SearchForm(),
             'stats': stats,
             'report_results': ReportResult.objects.order_by('-created')[:10],
-            'changelog': ObjectChange.objects.prefetch_related('user', 'changed_object_type')[:15]
+            'changelog': ObjectChange.objects.prefetch_related('user', 'changed_object_type')[:15],
+            'new_release': new_release,
+            'new_release_url': new_release_url,
         })
 
 

netbox/templates/home.html

@@ -1,6 +1,19 @@
 {% extends '_base.html' %}
 {% load helpers %}
 
+{% block header %}
+    {{ block.super }}
+    {% if new_release %}
+        <div class="alert alert-info" role="alert">
+            A new release is available:
+            {% if new_release_url %}<a target="_blank" href="{{ new_release_url }}">{% endif %}
+                NetBox v{{ new_release }}
+            {% if latest_version_url %}</a>{% endif %}
+        </div>
+    {% endif %}
+{% endblock %}
+
+
 {% block content %}
 {% include 'search_form.html' %}
 <div class="row">

netbox/utilities/background_tasks.py

@@ -0,0 +1,51 @@
+import logging
+
+import requests
+from cacheops.simple import cache, CacheMiss
+from django.conf import settings
+from django_rq import job
+from packaging import version
+
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
+
+@job
+def get_releases(pre_releases=False):
+    url = '{}/releases'.format(settings.UPDATE_REPO_URL)
+    headers = {
+        'Accept': 'application/vnd.github.v3+json',
+    }
+
+    # Check whether this URL has failed and shouldn't be retried yet
+    try:
+        failed_url = cache.get('netbox_releases_no_retry')
+        if url == failed_url:
+            return []
+    except CacheMiss:
+        pass
+
+    releases = []
+
+    # noinspection PyBroadException
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+
+        for release in response.json():
+            if 'tag_name' not in release:
+                continue
+
+            if not pre_releases and (release.get('devrelease') or release.get('prerelease')):
+                continue
+
+            releases.append((version.parse(release['tag_name']), release.get('html_url')))
+    except Exception:
+        # Don't retry this URL for 15 minutes
+        cache.set('netbox_releases_no_retry', url, 900)
+
+        logger.exception("Error while fetching {}".format(url))
+        return []
+
+    cache.set('netbox_releases', releases, settings.UPDATE_CACHE_TIMEOUT)
+    return releases

netbox/utilities/releases.py

@@ -0,0 +1,24 @@
+import logging
+
+from cacheops import CacheMiss, cache
+from django.conf import settings
+
+from utilities.background_tasks import get_releases
+
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
+
+def get_latest_release(pre_releases=False):
+    if settings.UPDATE_REPO_URL:
+        try:
+            releases = cache.get('netbox_releases')
+            if releases:
+                return max(releases)
+        except CacheMiss:
+            logger.debug("Starting background task to get releases")
+
+            # Get the releases in the background worker, it will fill the cache
+            get_releases.delay(pre_releases=pre_releases)
+
+    return 'unknown', None