feature: Notify about failed updates, block detectable bad updates (#7188)

* Feature: Notify about failed updates, block detectable bad updates
Ability to post notifications when the update fails.
Detect and roll back updates that will cause broken installs. (Needs testing)
Add severity to notifications, critical (2) notifications will display a toast.

This will be used for removing in-tree dependencies and raising the minimum php version.

* Improve naming a bit add phpdoc to new_notification
In case multiple notifications are created, remove them all.

* Remove notifications when update is disabled.

* update travis to use db testing

* added missing index

.travis.yml

@@ -6,6 +6,7 @@ matrix:
   fast_finish: true
   include:
   - php: 7.1
+    env: EXECUTE_BUILD_SCHEMA=true
   - php: 7.0
     env: SKIP_STYLE_CHECK=1
   - php: 5.3
@@ -17,7 +18,7 @@ matrix:
   - php: 5.5
     env: SKIP_STYLE_CHECK=1
   - php: 5.6
-    env: SKIP_STYLE_CHECK=1 EXECUTE_BUILD_DOCS=true EXECUTE_BUILD_SCHEMA=false
+    env: SKIP_STYLE_CHECK=1 EXECUTE_BUILD_DOCS=true
 #  - php: hhvm
 #    env: SKIP_STYLE_CHECK=1
 

daily.php

@@ -8,8 +8,9 @@
 
 $init_modules = array('alerts');
 require __DIR__ . '/includes/init.php';
+include_once __DIR__ . '/includes/notifications.php';
 
-$options = getopt('f:d:o:');
+$options = getopt('f:d:o:t:r:');
 
 if (isset($options['d'])) {
     echo "DEBUG\n";
@@ -97,8 +98,26 @@ if ($options['f'] === 'device_perf') {
     }
 }
 
+if ($options['f'] === 'set_notification') {
+    if ($options['t'] === 'update') {
+        $title = 'Error: Daily update failed';
+
+        if ($options['r']) {
+            remove_notification($title);
+        } else {
+            new_notification(
+                $title,
+                'The daily update script (daily.sh) has failed. Please check output by hand. If you need assistance, '
+                . 'visit the <a href="https://www.librenms.org/#support">LibreNMS Website</a> to find out how.',
+                2,
+                'daily.sh'
+            );
+        }
+    }
+}
+
 if ($options['f'] === 'notifications') {
-    include_once 'includes/notifications.php';
+    post_notifications();
 }
 
 if ($options['f'] === 'bill_data') {

daily.sh

@@ -22,6 +22,7 @@
 # define DAILY_SCRIPT as the full path to this script and LIBRENMS_DIR as the directory this script is in
 DAILY_SCRIPT=$(readlink -f "$0")
 LIBRENMS_DIR=$(dirname "$DAILY_SCRIPT")
+COMPOSER="php ${LIBRENMS_DIR}/scripts/composer_wrapper.php"
 
 # set log_file, using librenms $config['log_dir'], if set
 # otherwise we default to <LibreNMS Install Directory>/logs
@@ -95,6 +96,25 @@ call_daily_php() {
     done
 }
 
+#######################################
+# Set critical notifications for the user
+# Globals:
+#   LIBRENMS_DIR
+# Arguments:
+#   args:
+#        Type: update
+#        Result: 1 for success, 0 for failure
+# Returns:
+#   Exit-Code of Command
+#######################################
+set_notification() {
+    local args="$@";
+    local arg_type=$1;
+    local arg_result=$2;
+
+    php "${LIBRENMS_DIR}/daily.php" -f set_notification -t ${arg_type} -r ${arg_result};
+}
+
 #######################################
 # Entry into program
 # Globals:
@@ -122,28 +142,51 @@ main () {
         fi
     fi
 
+    # make sure autoload.php exists before trying to run any php that may require it
+    if [ ! -f "${LIBRENMS_DIR}/vendor/autoload.php" ]; then
+        ${COMPOSER} install --no-dev
+    fi
+
     if [[ -z "$arg" ]]; then
         up=$(php daily.php -f update >&2; echo $?)
         if [[ "$up" == "0" ]]; then
-            $DAILY_SCRIPT no-code-update
+            ${DAILY_SCRIPT} no-code-update
+            set_notification update 0  # make sure there are no update notifications if update is disabled
             exit
-        elif [[ "$up" == "1" ]]; then
+        fi
-            # Update to Master-Branch
+
+        # make sure the vendor directory is clean
         git checkout vendor/ --quiet > /dev/null 2>&1
+
+        update_res=0
+        if [[ "$up" == "1" ]]; then
+            # Update to Master-Branch
             old_ver=$(git show --pretty="%H" -s HEAD)
             status_run 'Updating to latest codebase' 'git pull --quiet' 'update'
+            update_res=$?
             new_ver=$(git show --pretty="%H" -s HEAD)
-            if [ "$old_ver" != "$new_ver" ]; then
-                status_run "Updated from $old_ver to $new_ver" ''
-            fi
         elif [[ "$up" == "3" ]]; then
             # Update to last Tag
-            git checkout vendor/ --quiet > /dev/null 2>&1
             old_ver=$(git describe --exact-match --tags $(git log -n1 --pretty='%h'))
             status_run 'Updating to latest release' 'git fetch --tags && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))' 'update'
+            update_res=$?
             new_ver=$(git describe --exact-match --tags $(git log -n1 --pretty='%h'))
+        fi
+
+        if (( $update_res > 0 )); then
+            set_notification update 0
+        fi
+
         if [[ "$old_ver" != "$new_ver" ]]; then
+#            status_run 'Updating Composer packages' "${COMPOSER} install --no-dev" 'update'
+
+            # Run post update checks
+            if [ ! -f "${LIBRENMS_DIR}/vendor/autoload.php" ]; then
+                status_run "Reverting update, check the output of composer diagnose" "git checkout $old_ver" 'update'
+                set_notification update 0
+            else
                 status_run "Updated from $old_ver to $new_ver" ''
+                set_notification update 1  # only clear the error if update was a success
             fi
         fi
 
@@ -156,7 +199,7 @@ main () {
 
         if [[ -z "$cnf" ]] || [[ "$cnf" == "0" ]] || [[ "$cnf" == "false" ]]; then
             # Call ourself again in case above pull changed or added something to daily.sh
-            $DAILY_SCRIPT post-pull
+            ${DAILY_SCRIPT} post-pull
         fi
     else
         case $arg in

html/index.php

@@ -302,6 +302,14 @@ if (dbFetchCell("SELECT COUNT(*) FROM `devices` WHERE `last_polled` <= DATE_ADD(
     $msg_box[] = array('type' => 'warning', 'message' => "<a href=\"poll-log/filter=unpolled/\">It appears as though you have some devices that haven't completed polling within the last 15 minutes, you may want to check that out :)</a>",'title' => 'Devices unpolled');
 }
 
+foreach (dbFetchRows('SELECT * FROM `notifications` WHERE `severity` > 1') as $notification) {
+    $msg_box[] = array(
+        'type' => 'error',
+        'message' => "<a href='notifications/'>${notification['body']}</a>",
+        'title' => $notification['title']
+    );
+}
+
 if (is_array($msg_box)) {
     echo("<script>
         toastr.options.timeout = 10;

html/pages/notifications.inc.php

@@ -66,16 +66,28 @@ $notifications = new ObjectCache('notifications');
 foreach ($notifications['sticky'] as $notif) {
     if (is_numeric($notif['source'])) {
         $notif['source'] = dbFetchCell('select username from users where user_id =?', array($notif['source']));
-    } ?>
+    }
-  <div class="well">
+    echo '<div class="well"><div class="row"> <div class="col-md-12">';
-    <div class="row">
+
-      <div class="col-md-12">
+    $class = $notif['severity'] == 2 ? 'text-danger' : 'text-warning';
-        <h4 class="text-warning" id="<?php echo $notif['notifications_id']; ?>"><strong><i class="fa fa-bell-o"></i>&nbsp;&nbsp;&nbsp;<?php echo $notif['title']; ?></strong>&nbsp;<span class="pull-right"><?php echo ($notif['user_id'] != $_SESSION['user_id'] ? '<code>Sticky by '.dbFetchCell('select username from users where user_id = ?', array($notif['user_id'])).'</code>' : '<button class="btn btn-primary fa fa-bell-slash-o unstick-notif" data-toggle="tooltip" data-placement="bottom" title="Remove Sticky" style="margin-top:-10px;"></button>'); ?></span></h4>
+    echo "<h4 class='$class' id='${notif['notifications_id']}'>";
+    echo "<strong><i class='fa fa-bell-o'></i>&nbsp;${notif['title']}</strong>";
+    echo "<span class='pull-right'>";
+
+    if ($notif['user_id'] != $_SESSION['user_id']) {
+        $sticky_user = get_user($notif['user_id']);
+        echo "<code>Sticky by ${sticky_user['username']}</code>";
+    } else {
+        echo '<button class="btn btn-primary fa fa-bell-slash-o unstick-notif" data-toggle="tooltip" data-placement="bottom" title="Remove Sticky" style="margin-top:-10px;"></button>';
+    }
+
+    echo '</span></h4>';
+?>
       </div>
     </div>
     <div class="row">
       <div class="col-md-12">
-        <blockquote>
+        <blockquote<?php echo $notif['severity'] == 2 ? ' style="border-color: darkred;"' : '' ?>>
           <p><?php echo $notif['body']; ?></p>
           <footer><?php echo $notif['datetime']; ?> | Source: <code><?php echo $notif['source']; ?></code></footer>
         </blockquote>
@@ -89,14 +101,24 @@ foreach ($notifications['sticky'] as $notif) {
 <?php
 foreach ($notifications['unread'] as $notif) {
     if (is_numeric($notif['source'])) {
-        $notif['source'] = dbFetchCell('select username from users where user_id =?', array($notif['source']));
+        $source_user = get_user($notif['source']);
-    } ?>
+        $notif['source'] = $source_user['username'];
-  <div class="well">
+    }
-    <div class="row">
+    echo '<div class="well"><div class="row"> <div class="col-md-12">';
-      <div class="col-md-12">
+    d_echo($notif);
-        <h4 class="text-success" id="<?php echo $notif['notifications_id']; ?>"><strong><?php echo $notif['title']; ?></strong><span class="pull-right">
+    $class = 'text-success';
-<?php echo ($_SESSION['userlevel'] == 10 ? '<button class="btn btn-primary fa fa-bell-o stick-notif" data-toggle="tooltip" data-placement="bottom" title="Mark as Sticky" style="margin-top:-10px;"></button>' : ''); ?>
+    if ($notif['severity'] == 1) {
-&nbsp;
+        $class='text-warning';
+    } elseif ($notif['severity'] == 2) {
+        $class = 'text-danger';
+    }
+    echo "<h4 class='$class' id='${notif['notifications_id']}'>${notif['title']}<span class='pull-right'>";
+
+    if (is_admin()) {
+        echo '<button class="btn btn-primary fa fa-bell-o stick-notif" data-toggle="tooltip" data-placement="bottom" title="Mark as Sticky" style="margin-top:-10px;"></button>';
+    }
+?>
+
 <button class="btn btn-primary fa fa-eye read-notif" data-toggle="tooltip" data-placement="bottom" title="Mark as Read" style="margin-top:-10px;"></button>
 </span>
 </h4>
@@ -104,7 +126,7 @@ foreach ($notifications['unread'] as $notif) {
     </div>
     <div class="row">
       <div class="col-md-12">
-        <blockquote>
+          <blockquote<?php echo $notif['severity'] == 2 ? ' style="border-color: darkred;"' : '' ?>>
           <p><?php echo preg_replace('/\\\n/', '<br />', $notif['body']); ?></p>
           <footer><?php echo $notif['datetime']; ?> | Source: <code><?php echo $notif['source']; ?></code></footer>
         </blockquote>
@@ -125,17 +147,26 @@ foreach ($notifications['unread'] as $notif) {
       <h2>Archive</h2>
     </div>
   </div>
-<?php    foreach (array_reverse($notifications['read']) as $notif) { ?>
+<?php
-  <div class="well">
+foreach (array_reverse($notifications['read']) as $notif) {
-    <div class="row">
+    echo '<div class="well"><div class="row"> <div class="col-md-12"><h4';
-      <div class="col-md-12">
+    if ($notif['severity'] == 1) {
-        <h4 id="<?php echo $notif['notifications_id']; ?>"><?php echo $notif['title'];
+        echo ' class="text-warning"';
-        echo ($_SESSION['userlevel'] == 10 ? '<span class="pull-right"><button class="btn btn-primary fa fa-bell-o stick-notif" data-toggle="tooltip" data-placement="bottom" title="Mark as Sticky" style="margin-top:-10px;"></button></span>' : ''); ?>
+    } elseif ($notif['severity'] == 2) {
+        echo ' class="text-danger"';
+    }
+    echo  " id='${notif['notifications_id']}'>${notif['title']}";
+
+    if ($_SESSION['userlevel'] == 10) {
+        echo '<span class="pull-right"><button class="btn btn-primary fa fa-bell-o stick-notif" data-toggle="tooltip" data-placement="bottom" title="Mark as Sticky" style="margin-top:-10px;"></button></span>';
+    }
+?>
+        </h4>
       </div>
     </div>
     <div class="row">
       <div class="col-md-12">
-        <blockquote>
+          <blockquote<?php echo $notif['severity'] == 2 ? ' style="border-color: darkred;"' : '' ?>>
           <p><?php echo preg_replace('/\\\n/', '<br />', $notif['body']); ?></p>
           <footer><?php echo $notif['datetime']; ?> | Source: <code><?php echo $notif['source']; ?></code></footer>
         </blockquote>

includes/notifications.php

@@ -15,10 +15,13 @@
 
 /**
  * Notification Poller
- * @author Daniel Preussker
  * @copyright 2015 Daniel Preussker, QuxLabs UG
+ * @copyright 2017 Tony Murray
+ * @author    Daniel Preussker
+ * @author    Tony Murray <murraytony@gmail.com>
  * @license   GPL
  * @package   LibreNMS
+ * @link      http://librenms.org
  * @subpackage Notifications
  */
 
@@ -109,4 +112,45 @@ function parse_atom($feed)
     return $obj;
 }
 
-post_notifications();
+/**
+ * Create a new custom notification. Duplicate title+message notifications will not be created.
+ *
+ * @param string $title
+ * @param string $message
+ * @param int $severity 0=ok, 1=warning, 2=critical
+ * @param string $source A string describing what created this notification
+ * @param string $date
+ * @return bool
+ */
+function new_notification($title, $message, $severity = 0, $source = 'adhoc', $date = null)
+{
+    $notif = array(
+        'title' => $title,
+        'body' => $message,
+        'severity' => $severity,
+        'source' => $source,
+        'checksum' => hash('sha512', $title . $message),
+        'datetime' => strftime('%F', is_null($date) ? time() : strtotime($date))
+    );
+
+    if (dbFetchCell('SELECT 1 FROM `notifications` WHERE `checksum` = ?', array($notif['checksum'])) != 1) {
+        return dbInsert('notifications', $notif) > 0;
+    }
+
+    return false;
+}
+
+/**
+ * Removes all notifications with the given title.
+ * This should be used with care.
+ *
+ * @param string $title
+ */
+function remove_notification($title)
+{
+    $ids = dbFetchColumn('SELECT `notifications_id` FROM `notifications` WHERE `title`=?', array($title));
+    foreach ($ids as $id) {
+        dbDelete('notifications', '`notifications_id`=?', array($id));
+        dbDelete('notifications_attribs', '`notifications_id`=?', array($id));
+    }
+}

misc/db_schema.yaml

@@ -856,13 +856,15 @@ notifications:
   Columns:
     body: { Field: body, Type: text, 'Null': false, Default: 'NULL', Extra: '' }
     checksum: { Field: checksum, Type: varchar(128), 'Null': false, Default: 'NULL', Extra: '' }
-    datetime: { Field: datetime, Type: timestamp, 'Null': false, Default: CURRENT_TIMESTAMP, Extra: '' }
+    datetime: { Field: datetime, Type: timestamp, 'Null': false, Default: '1970-01-02 00:00:00', Extra: '' }
     notifications_id: { Field: notifications_id, Type: int(11), 'Null': false, Default: 'NULL', Extra: auto_increment }
+    severity: { Field: severity, Type: int(11), 'Null': true, Default: '0', Extra: '' }
     source: { Field: source, Type: varchar(255), 'Null': false, Default: '', Extra: '' }
     title: { Field: title, Type: varchar(255), 'Null': false, Default: '', Extra: '' }
   Indexes:
     PRIMARY: { Name: PRIMARY, Columns: [notifications_id], Unique: true, Type: BTREE }
     checksum: { Name: checksum, Columns: [checksum], Unique: true, Type: BTREE }
+    notifications_severity_index: { Name: notifications_severity_index, Columns: [severity], Unique: false, Type: BTREE }
 notifications_attribs:
   Columns:
     attrib_id: { Field: attrib_id, Type: int(11), 'Null': false, Default: 'NULL', Extra: auto_increment }

sql-schema/205.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `notifications` CHANGE `datetime` `datetime` timestamp NOT NULL DEFAULT '1970-01-02 00:00:00';
+ALTER TABLE `notifications` ADD `severity` INT DEFAULT 0 NULL COMMENT '0=ok,1=warning,2=critical' AFTER `body`;
+CREATE INDEX `notifications_severity_index` ON `notifications` (`severity`);