Merge pull request #4001 from steffann/738-automatically-check-for-new-releases

Fixes: #738: Automatically check for new versions

netbox/netbox/configuration.example.py

@@ -124,6 +124,14 @@ EXEMPT_VIEW_PERMISSIONS = [
     # 'ipam.prefix',
 ]
 
+# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
+# version check or use the URL below to check for release in the official NetBox repository.
+UPDATE_REPO_URL = None
+# UPDATE_REPO_URL = 'https://api.github.com/repos/netbox-community/netbox'
+
+# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
+UPDATE_CACHE_TIMEOUT = 24 * 3600
+
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
 #   https://docs.djangoproject.com/en/stable/topics/logging/
 LOGGING = {}

netbox/netbox/settings.py

@@ -1,8 +1,10 @@
 import logging
 import os
 import platform
+import re
 import socket
 import warnings
+from urllib.parse import urlsplit
 
 from django.contrib.messages import constants as messages
 from django.core.exceptions import ImproperlyConfigured
@@ -78,6 +80,8 @@ DEVELOPER = getattr(configuration, 'DEVELOPER', False)
 EMAIL = getattr(configuration, 'EMAIL', {})
 ENFORCE_GLOBAL_UNIQUE = getattr(configuration, 'ENFORCE_GLOBAL_UNIQUE', False)
 EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', [])
+UPDATE_REPO_URL = getattr(configuration, 'UPDATE_REPO_URL', None)
+UPDATE_CACHE_TIMEOUT = getattr(configuration, 'UPDATE_CACHE_TIMEOUT', 24 * 3600)
 LOGGING = getattr(configuration, 'LOGGING', {})
 LOGIN_REQUIRED = getattr(configuration, 'LOGIN_REQUIRED', False)
 LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None)
@@ -302,6 +306,31 @@ AUTHENTICATION_BACKENDS = [
     'utilities.auth_backends.ViewExemptModelBackend',
 ]
 
+# GitHub repository for version check
+if UPDATE_REPO_URL:
+    UPDATE_REPO_URL = UPDATE_REPO_URL.rstrip('/')
+    try:
+        scheme, netloc, path, query, fragment = urlsplit(UPDATE_REPO_URL)
+    except ValueError:
+        raise ImproperlyConfigured("UPDATE_REPO_URL must be a valid URL")
+
+    if scheme not in ('http', 'https'):
+        raise ImproperlyConfigured("UPDATE_REPO_URL must be a valid http:// or https:// URL")
+
+    if not re.fullmatch(r'/repos/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', path):
+        raise ImproperlyConfigured(
+            "GITHUB_REPOSITORY must contain the base URL of the GitHub API in a form like "
+            "'https://api.github.com/repos/<owner>/<repository>'"
+        )
+
+    # Don't allow ? (query) and # (fragment) in the URL
+    if query or fragment:
+        raise ImproperlyConfigured("UPDATE_REPO_URL may not contain a ? (query) or # (fragment)")
+
+# Enforce a cache timeout of at least an hour to protect GitHub
+if UPDATE_CACHE_TIMEOUT < 3600:
+    raise ImproperlyConfigured("UPDATE_CACHE_TIMEOUT has to be at least 3600 seconds (1 hour)")
+
 # Internationalization
 LANGUAGE_CODE = 'en-us'
 USE_I18N = True

netbox/netbox/tests/test_get_releases.py

@@ -0,0 +1,160 @@
+from io import BytesIO
+from logging import ERROR
+from unittest.mock import Mock, patch
+
+import requests
+from cacheops import CacheMiss, RedisCache
+from django.test import SimpleTestCase, override_settings
+from packaging.version import Version
+from requests import Response
+
+from utilities.background_tasks import get_releases
+
+
+def successful_github_response(url, *_args, **_kwargs):
+    r = Response()
+    r.url = url
+    r.status_code = 200
+    r.reason = 'OK'
+    r.headers = {
+        'Content-Type': 'application/json; charset=utf-8',
+    }
+    r.raw = BytesIO(b'''[
+        {
+            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.7.8",
+            "tag_name": "v2.7.8",
+            "prerelease": false
+        },
+        {
+            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1",
+            "tag_name": "v2.6-beta1",
+            "prerelease": true
+        },
+        {
+            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.5.9",
+            "tag_name": "v2.5.9",
+            "prerelease": false
+        }
+    ]
+    ''')
+    return r
+
+
+def unsuccessful_github_response(url, *_args, **_kwargs):
+    r = Response()
+    r.url = url
+    r.status_code = 404
+    r.reason = 'Not Found'
+    r.headers = {
+        'Content-Type': 'application/json; charset=utf-8',
+    }
+    r.raw = BytesIO(b'''{
+        "message": "Not Found",
+        "documentation_url": "https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository"
+    }
+    ''')
+    return r
+
+
+@override_settings(UPDATE_REPO_URL='https://localhost/unittest', UPDATE_CACHE_TIMEOUT=160876)
+class GetReleasesTestCase(SimpleTestCase):
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_pre_releases(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.side_effect = CacheMiss()
+        dummy_request_get.side_effect = successful_github_response
+
+        releases = get_releases(pre_releases=True)
+
+        # Check result
+        self.assertListEqual(releases, [
+            (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
+            (Version('2.6b1'), 'https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1'),
+            (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
+        ])
+
+        # Check if correct request is made
+        dummy_request_get.assert_called_once()
+        dummy_request_get.assert_called_with('https://localhost/unittest/releases',
+                                             headers={
+                                                 'Accept': 'application/vnd.github.v3+json'
+                                             })
+
+        # Check if result is put in cache
+        dummy_cache_set.assert_called_once()
+        dummy_cache_set.assert_called_with('netbox_releases', releases, 160876)
+
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_no_pre_releases(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.side_effect = CacheMiss()
+        dummy_request_get.side_effect = successful_github_response
+
+        releases = get_releases(pre_releases=False)
+
+        # Check result
+        self.assertListEqual(releases, [
+            (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
+            (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
+        ])
+
+        # Check if correct request is made
+        dummy_request_get.assert_called_once()
+        dummy_request_get.assert_called_with('https://localhost/unittest/releases',
+                                             headers={
+                                                 'Accept': 'application/vnd.github.v3+json'
+                                             })
+
+        # Check if result is put in cache
+        dummy_cache_set.assert_called_once()
+        dummy_cache_set.assert_called_with('netbox_releases', releases, 160876)
+
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_failed_request(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.side_effect = CacheMiss()
+        dummy_request_get.side_effect = unsuccessful_github_response
+
+        with self.assertLogs(level=ERROR) as cm:
+            releases = get_releases()
+
+        # Check log entry
+        self.assertEqual(len(cm.output), 1)
+        log_output = cm.output[0]
+        last_log_line = log_output.split('\n')[-1]
+        self.assertRegex(last_log_line, '404 .* Not Found')
+
+        # Check result
+        self.assertListEqual(releases, [])
+
+        # Check if correct request is made
+        dummy_request_get.assert_called_once()
+        dummy_request_get.assert_called_with('https://localhost/unittest/releases',
+                                             headers={
+                                                 'Accept': 'application/vnd.github.v3+json'
+                                             })
+
+        # Check if failure is put in cache
+        dummy_cache_set.assert_called_once()
+        dummy_cache_set.assert_called_with('netbox_releases_no_retry', 'https://localhost/unittest/releases', 900)
+
+    @patch.object(requests, 'get')
+    @patch.object(RedisCache, 'set')
+    @patch.object(RedisCache, 'get')
+    def test_blocked_retry(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock):
+        dummy_cache_get.return_value = 'https://localhost/unittest/releases'
+        dummy_request_get.side_effect = successful_github_response
+
+        releases = get_releases()
+
+        # Check result
+        self.assertListEqual(releases, [])
+
+        # Check if request is NOT made
+        dummy_request_get.assert_not_called()
+
+        # Check if cache is not updated
+        dummy_cache_set.assert_not_called()

netbox/netbox/views.py

@@ -1,8 +1,10 @@
 from collections import OrderedDict
 
-from django.db.models import Count, F
+from django.conf import settings
+from django.db.models import Count, F, OuterRef, Subquery
 from django.shortcuts import render
 from django.views.generic import View
+from packaging import version
 from rest_framework.response import Response
 from rest_framework.reverse import reverse
 from rest_framework.views import APIView
@@ -31,6 +33,7 @@ from secrets.tables import SecretTable
 from tenancy.filters import TenantFilterSet
 from tenancy.models import Tenant
 from tenancy.tables import TenantTable
+from utilities.releases import get_latest_release
 from virtualization.filters import ClusterFilterSet, VirtualMachineFilterSet
 from virtualization.models import Cluster, VirtualMachine
 from virtualization.tables import ClusterTable, VirtualMachineDetailTable
@@ -240,11 +243,25 @@ class HomeView(View):
 
         }
 
+        new_release = None
+        new_release_url = None
+
+        if request.user.is_staff or request.user.is_superuser:
+            # Only check for new releases if the current user might be able to do anything about it
+            latest_release, github_url = get_latest_release()
+            if isinstance(latest_release, version.Version):
+                current_version = version.parse(settings.VERSION)
+                if latest_release > current_version:
+                    new_release = str(latest_release)
+                    new_release_url = github_url
+
         return render(request, self.template_name, {
             'search_form': SearchForm(),
             'stats': stats,
             'report_results': ReportResult.objects.order_by('-created')[:10],
-            'changelog': ObjectChange.objects.prefetch_related('user', 'changed_object_type')[:15]
+            'changelog': ObjectChange.objects.prefetch_related('user', 'changed_object_type')[:15],
+            'new_release': new_release,
+            'new_release_url': new_release_url,
         })
 
 

netbox/templates/home.html

@@ -1,6 +1,19 @@
 {% extends '_base.html' %}
 {% load helpers %}
 
+{% block header %}
+    {{ block.super }}
+    {% if new_release %}
+        <div class="alert alert-info" role="alert">
+            A new release is available:
+            {% if new_release_url %}<a target="_blank" href="{{ new_release_url }}">{% endif %}
+                NetBox v{{ new_release }}
+            {% if latest_version_url %}</a>{% endif %}
+        </div>
+    {% endif %}
+{% endblock %}
+
+
 {% block content %}
 {% include 'search_form.html' %}
 <div class="row">

netbox/utilities/background_tasks.py

@@ -0,0 +1,51 @@
+import logging
+
+import requests
+from cacheops.simple import cache, CacheMiss
+from django.conf import settings
+from django_rq import job
+from packaging import version
+
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
+
+@job
+def get_releases(pre_releases=False):
+    url = '{}/releases'.format(settings.UPDATE_REPO_URL)
+    headers = {
+        'Accept': 'application/vnd.github.v3+json',
+    }
+
+    # Check whether this URL has failed and shouldn't be retried yet
+    try:
+        failed_url = cache.get('netbox_releases_no_retry')
+        if url == failed_url:
+            return []
+    except CacheMiss:
+        pass
+
+    releases = []
+
+    # noinspection PyBroadException
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+
+        for release in response.json():
+            if 'tag_name' not in release:
+                continue
+
+            if not pre_releases and (release.get('devrelease') or release.get('prerelease')):
+                continue
+
+            releases.append((version.parse(release['tag_name']), release.get('html_url')))
+    except Exception:
+        # Don't retry this URL for 15 minutes
+        cache.set('netbox_releases_no_retry', url, 900)
+
+        logger.exception("Error while fetching {}".format(url))
+        return []
+
+    cache.set('netbox_releases', releases, settings.UPDATE_CACHE_TIMEOUT)
+    return releases

netbox/utilities/releases.py

@@ -0,0 +1,24 @@
+import logging
+
+from cacheops import CacheMiss, cache
+from django.conf import settings
+
+from utilities.background_tasks import get_releases
+
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
+
+def get_latest_release(pre_releases=False):
+    if settings.UPDATE_REPO_URL:
+        try:
+            releases = cache.get('netbox_releases')
+            if releases:
+                return max(releases)
+        except CacheMiss:
+            logger.debug("Starting background task to get releases")
+
+            # Get the releases in the background worker, it will fill the cache
+            get_releases.delay(pre_releases=pre_releases)
+
+    return 'unknown', None