Merge branch 'master' into calix

doc/General/Changelog.md

@@ -1,3 +1,35 @@
+### February 2016
+
+#### Bug fixes
+  - Discovery / Polling:
+    - Quote snmp v2c community (PR2927)
+    - For entity-sensor, changed variable name again (PR2948)
+    - Fix some issues with/introduced by port association mode configuration (PR2923)
+  - WebUI:
+    - Fix ceph graps (PR2909, PR2942)
+    - BGP Overlib (PR2915)
+    - Added `application/json` headers where json is returned (PR2936)
+    - Stop realtime graph page from auto refreshing (PR2939)
+    - Updated parsing of alert rules to allow `|` (PR2917)
+    - Fix IP Display (PR2951)
+  - Misc:
+    - Updated `device_by_id_cache()` to convert IP column (PR2940)
+  - Documentation:
+    - Removed devloping doc as none of the info is current (PR2911)
+
+#### Improvements
+  - Discovery / Polling:
+    - Added ability to ignore device sensors from entity mib (PR2862)
+    - Added `ifOperStatus_prev` and `ifAdminStatus_prev` values to db (PR2912)
+  - Added detection for:
+    - Dell Networking N2048 (PR2949)
+  - Misc:
+    - Added check for rrd vadility (PR2908)
+    - Add systemd unit file for the python poller service (PR2913)
+  - Documentation:
+    -  Added description of AD configuration options (PR2910)
+    -  Add description to mibbases polling (PR2919)
+
 ### January 2016
 
 #### Bug fixes

doc/Support/Install Validation.md

@@ -17,7 +17,9 @@ So, to try and help with some of the general issues people come across we've put
 
 Optionally you can also pass -m and a module name for that to be tested. Current modules are:
 
- - mail. This will validate your mail transport configuration.
+ - mail - This will validate your mail transport configuration.
+ - dist-poller - This will test your distributed poller configuration.
+ - rrdcheck - This will test your rrd files to see if they are unreadable or corrupted (source of broken graphs).
 
 Output, this is color coded to try and make things a little easier:
 

html/ajax_dash.php

@@ -55,5 +55,5 @@ $response = array(
                   'html' => $output,
                   'title' => $title,
                  );
-
+header('Content-type: application/json');
 echo _json_encode($response);

html/ajax_form.php

@@ -29,6 +29,7 @@ if (!$_SESSION['authenticated']) {
 
 if (preg_match('/^[a-zA-Z0-9\-]+$/', $_POST['type']) == 1) {
     if (file_exists('includes/forms/'.$_POST['type'].'.inc.php')) {
+        header('Content-type: application/json');
         include_once 'includes/forms/'.$_POST['type'].'.inc.php';
     }
 }

html/ajax_rulesuggest.php

@@ -62,7 +62,7 @@ function levsort($base, $obj) {
 
 }
 
-
+header('Content-type: application/json');
 $obj     = array(array('name' => 'Error: No suggestions found.'));
 $term    = array();
 $current = false;

html/ajax_search.php

@@ -19,7 +19,7 @@ $bgp    = array();
 
 if (isset($_REQUEST['search'])) {
     $search = mres($_REQUEST['search']);
-
+    header('Content-type: application/json');
     if (strlen($search) > 0) {
         $found = 0;
 

html/ajax_table.php

@@ -36,6 +36,7 @@ $response     = array();
 
 if (isset($id)) {
     if (file_exists("includes/table/$id.inc.php")) {
+        header('Content-type: application/json');
         include_once "includes/table/$id.inc.php";
     }
 }

html/includes/dev-overview-data.inc.php

@@ -30,10 +30,10 @@ echo '<tr>
         <td>'.$device['sysName'].' </td>
       </tr>';
 
-if ($ip = inet6_ntop($device['ip'])) {
+if (!empty($device['ip'])) {
     echo '<tr>
              <td>Resolved IP</td>
-             <td>'.$ip.'</td>
+             <td>'.$device['ip'].'</td>
          </tr>';
 }
 

html/includes/forms/parse-alert-rule.inc.php

@@ -20,7 +20,7 @@ $alert_id = $_POST['alert_id'];
 
 if (is_numeric($alert_id) && $alert_id > 0) {
     $rule               = dbFetchRow('SELECT * FROM `alert_rules` WHERE `id` = ? LIMIT 1', array($alert_id));
-    $rule_split         = preg_split('/([a-zA-Z0-9_\-\.\=\%\<\>\ \"\'\!\~\(\)\*\/\@]+[&&\|\|]+)/', $rule['rule'], -1, (PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY));
+    $rule_split         = preg_split('/([a-zA-Z0-9_\-\.\=\%\<\>\ \"\'\!\~\(\)\*\/\@\|]+[&&|\|\|]{2})/', $rule['rule'], -1, (PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY));
     $count              = (count($rule_split) - 1);
     $rule_split[$count] = $rule_split[$count].'  &&';
     $output             = array(

html/pages/device/apps/ceph.inc.php

@@ -17,9 +17,9 @@ foreach ($graphs as $key => $text) {
 
     if ($key == "ceph_poolstats") {
         foreach (glob($rrddir."/app-ceph-".$app['app_id']."-pool-*") as $rrd_filename) {
-            $graph_array['to']     = $config['time']['now'];
-            $graph_array['id']     = $app['app_id'];
             if (preg_match("/.*-pool-(.+)\.rrd$/", $rrd_filename, $pools)) {
+                $graph_array['to']     = $config['time']['now'];
+                $graph_array['id']     = $app['app_id'];
                 $pool = $pools[1];
                 echo '<h3>'.$pool.' Reads/Writes</h3>';
                 $graph_array['type']   = 'application_ceph_pool_io';
@@ -29,6 +29,8 @@ foreach ($graphs as $key => $text) {
                 include 'includes/print-graphrow.inc.php';
                 echo '</td></tr>';
 
+                $graph_array['to']     = $config['time']['now'];
+                $graph_array['id']     = $app['app_id'];
                 echo '<h3>'.$pool.' IOPS</h3>';
                 $graph_array['type']   = 'application_ceph_pool_iops';
                 $graph_array['pool']   = $pool;

html/pages/device/port/realtime.inc.php

@@ -1,5 +1,7 @@
 <?php
 
+$no_refresh = true;
+
 // FIXME - do this in a function and/or do it in graph-realtime.php
 if (!isset($vars['interval'])) {
     if ($device['os'] == 'linux') {

includes/common.php

@@ -315,6 +315,7 @@ function device_by_id_cache($device_id, $refresh = '0') {
     }
     else {
         $device = dbFetchRow("SELECT * FROM `devices` WHERE `device_id` = ?", array($device_id));
+        $device['ip'] = inet6_ntop($device['ip']);
         $cache['devices']['id'][$device_id] = $device;
     }
     return $device;
@@ -1248,7 +1249,7 @@ function get_port_id ($ports_mapped, $port, $port_association_mode) {
     */
     $maps  = $ports_mapped['maps'];
 
-    if (in_array ($port_association_mode, array ('ifIndex', 'ifName', 'ifDescr'))) {
+    if (in_array ($port_association_mode, array ('ifIndex', 'ifName', 'ifDescr', 'ifAlias'))) {
         $port_id = $maps[$port_association_mode][$port[$port_association_mode]];
     }
 

includes/defaults.inc.php

@@ -858,3 +858,6 @@ $config['update_channel']                               = 'master';
 
 // Default port association mode
 $config['default_port_association_mode'] = 'ifIndex';
+// Ignore ports which can't be mapped using a devices port_association_mode
+// See include/polling/ports.inc.php for a lenghty explanation.
+$config['ignore_unmapable_port'] = False;

includes/discovery/os/dnos.inc.php

@@ -7,4 +7,7 @@ if (!$os) {
     if (strstr($sysObjectId, '.1.3.6.1.4.1.674.10895.3042')) {
         $os = 'dnos';
     }
+    if (strstr($sysObjectId, '.1.3.6.1.4.1.674.10895.3054')) {
+        $os = 'dnos';
+    }
 }

includes/discovery/sensors/entity-sensor.inc.php

@@ -50,7 +50,7 @@ if (is_array($oids)) {
                 $descr = rewrite_entity_descr($descr);
             }
 
-            $valid = check_entity_sensor($descr, $device);
+            $valid_sensor = check_entity_sensor($descr, $device);
 
             $type = $entitysensor[$entry['entPhySensorType']];
 
@@ -103,16 +103,16 @@ if (is_array($oids)) {
 
             if ($type == 'temperature') {
                 if ($current > '200') {
-                    $valid = false;
+                    $valid_sensor = false;
                 } $descr = preg_replace('/[T|t]emperature[|s]/', '', $descr);
             }
 
             // echo($descr . "|" . $index . "|" .$current . "|" . $multiplier . "|" . $divisor ."|" . $entry['entPhySensorScale'] . "|" . $entry['entPhySensorPrecision'] . "\n");
             if ($current == '-127') {
-                $valid = false;
+                $valid_sensor = false;
             }
 
-            if ($valid && dbFetchCell("SELECT COUNT(*) FROM `sensors` WHERE device_id = ? AND `sensor_class` = ? AND `sensor_type` = 'cisco-entity-sensor' AND `sensor_index` = ?", array($device['device_id'], $type, $index)) == '0') {
+            if ($valid_sensor && dbFetchCell("SELECT COUNT(*) FROM `sensors` WHERE device_id = ? AND `sensor_class` = ? AND `sensor_type` = 'cisco-entity-sensor' AND `sensor_index` = ?", array($device['device_id'], $type, $index)) == '0') {
                 // Check to make sure we've not already seen this sensor via cisco's entity sensor mib
                 discover_sensor($valid['sensor'], $type, $device, $oid, $index, 'entity-sensor', $descr, $divisor, $multiplier, null, null, null, null, $current);
             }

includes/functions.php

@@ -1366,3 +1366,70 @@ function dnslookup($device,$type=false,$return=false) {
     $record = dns_get_record($device['hostname'],$type);
     return $record[0][$return];
 }//end dnslookup
+
+
+/**
+ * Reursive Filter Iterator to iterate directories and locate .rrd files.
+ *
+ * @method boolean isDir()
+ *
+**/
+
+class RRDRecursiveFilterIterator extends \RecursiveFilterIterator {
+
+    public function accept() {
+        $filename = $this->current()->getFilename();
+        if ($filename[0] === '.') {
+            // Ignore hidden files and directories
+            return false;
+        }
+        if ($this->isDir()) {
+            // We want to search into directories
+            return true;
+        }
+        // Matches files with .rrd in the filename.
+        // We are only searching rrd folder, but there could be other files and we don't want to cause a stink.
+        return strpos($filename, '.rrd') !== false;
+    }
+}
+
+/**
+ * Run rrdtool info on a file path
+ *
+ * @param string $path Path to pass to rrdtool info
+ * @param string $stdOutput Variable to recieve the output of STDOUT
+ * @param string $stdError Variable to recieve the output of STDERR
+ *
+ * @return int exit code
+ *
+**/
+
+function rrdtest($path, &$stdOutput, &$stdError) {
+    global $config;
+    //rrdtool info <escaped rrd path>
+    $command = $config['rrdtool'].' info '.escapeshellarg($path);
+    $process = proc_open(
+        $command,
+        array (
+            0 => array('pipe', 'r'),
+            1 => array('pipe', 'w'),
+            2 => array('pipe', 'w'),
+        ),
+        $pipes
+    );
+
+    if (!is_resource($process)) {
+        throw new \RuntimeException('Could not create a valid process');
+    }
+
+    $status = proc_get_status($process);
+    while($status['running']) {
+        usleep(2000); // Sleep 2000 microseconds or 2 milliseconds
+        $status = proc_get_status($process);
+    }
+
+    $stdOutput = stream_get_contents($pipes[1]);
+    $stdError  = stream_get_contents($pipes[2]);
+    proc_close($process);
+    return $status['exitcode'];
+}

includes/polling/port-adsl.inc.php

@@ -39,10 +39,7 @@
 // adslAturPerfESs.1 = 0 seconds
 // adslAturPerfValidIntervals.1 = 0
 // adslAturPerfInvalidIntervals.1 = 0
-if (isset($port_stats[$port_id]['adslLineCoding'])) {
-    // Check to make sure Port data is cached.
-    $this_port = &$port_stats[$port_id];
-
+if (isset($this_port['adslLineCoding'])) {
     $rrdfile = get_port_rrdfile_path ($device['hostname'], $port_id, 'adsl');
 
     $rrd_create  = ' --step 300';

includes/polling/port-etherlike.inc.php

@@ -1,11 +1,6 @@
 <?php
 
-if ($port_stats[$port_id] &&
-    $port['ifType'] == 'ethernetCsmacd' &&
-    isset($port_stats[$port_id]['dot3StatsIndex'])) {
-    // Check to make sure Port data is cached.
-    $this_port = &$port_stats[$port_id];
-
+if ($this_port['dot3StatsIndex'] and $port['ifType'] == 'ethernetCsmacd') {
     // TODO: remove legacy check?
     $old_rrdfile = $config['rrd_dir'].'/'.$device['hostname'].'/'.safename('etherlike-'.$port['ifIndex'].'.rrd');
     $rrd_file    = get_port_rrdfile_path ($device['hostname'], $port_id, 'dot3');

includes/polling/port-poe.inc.php

@@ -35,12 +35,7 @@ $peth_oids = array(
     'pethMainPseConsumptionPower',
 );
 
-if ($port_stats[$port_id]
-    && $port['ifType'] == 'ethernetCsmacd'
-    && isset($port_stats[$port_id]['dot3StatsIndex'])) {
-    // Check to make sure Port data is cached.
-    $this_port = &$port_stats[$port_id];
-
+if ($this_port['dot3StatsIndex'] && $port['ifType'] == 'ethernetCsmacd') {
     $rrdfile = get_port_rrdfile_path ($device['hostname'], $port_id, 'poe');
     if (!file_exists($rrdfile)) {
         $rrd_create .= $config['rrd_rra'];

includes/polling/ports.inc.php

@@ -224,6 +224,7 @@ foreach ($ports_mapped['maps']['ifIndex'] as $ifIndex => $port_id) {
 }
 
 
+$ports_found = array ();
 // New interface detection
 foreach ($port_stats as $ifIndex => $port) {
     // Store ifIndex in port entry and prefetch ifName as we'll need it multiple times
@@ -238,6 +239,31 @@ foreach ($port_stats as $ifIndex => $port) {
 
         // Port newly discovered?
         if (! $ports[$port_id]) {
+            /**
+              * When using the ifName or ifDescr as means to map discovered ports to
+              * known ports in the DB (think of port association mode) it's possible
+              * that we're facing the problem that the ifName or ifDescr polled from
+              * the device is unset or an empty string (like when querying some ubnt
+              * devices...). If this happends we have no way to map this port to any
+              * port found in the database. As reported this situation may occur for
+              * the time of one poll and might resolve automagically before the next
+              * poller run happens. Without this special case this would lead to new
+              * ports added to the database each time this situation occurs. To give
+              * the user the choice between »a lot of new ports« and »some poll runs
+              * are missed but ports stay stable« the 'ignore_unmapable_port' option
+              * has been added to configure this behaviour. To skip the port in this
+              * loop is sufficient as the next loop is looping only over ports found
+              * in the database and "maps back". As we did not add a new port to the
+              * DB here, there's no port to be mapped to.
+              *
+              * I'm using the in_array() check here, as I'm not sure if an "ifIndex"
+              * can be legally set to 0, which would yield True when checking if the
+              * value is empty().
+              */
+            if ($config['ignore_unmapable_port'] === True and in_array ($port[$port_association_mode], array ('', Null))) {
+                continue;
+            }
+
             $port_id         = dbInsert(array('device_id' => $device['device_id'], 'ifIndex' => $ifIndex, 'ifName' => $ifName), 'ports');
             dbInsert(array('port_id' => $port_id), 'ports_statistics');
             $ports[$port_id] = dbFetchRow('SELECT * FROM `ports` WHERE `port_id` = ?', array($port_id));
@@ -255,9 +281,18 @@ foreach ($port_stats as $ifIndex => $port) {
             dbInsert(array('port_id' => $port_id), 'ports_statistics');
         }
 
-        // Assure stable mapping
-        $port_stats[$ifIndex]['port_id'] = $port_id;
+        /** Assure stable bidirectional port mapping between DB and polled data
+          *
+          * Store the *current* ifIndex in the port info array containing all port information
+          * fetched from the database, as this is the only means we have to map ports_stats we
+          * just polled from the device to a port in $ports. All code below an includeed below
+          * will and has to map a port using it's ifIndex.
+          */
         $ports[$port_id]['ifIndex'] = $ifIndex;
+        $port_stats[$ifIndex]['port_id'] = $port_id;
+
+	/* Build a list of all ports, identified by their port_id, found within this poller run. */
+	$ports_found[] = $port_id;
     }
 
     // Port vanished (mark as deleted)
@@ -274,11 +309,32 @@ echo "\n";
 // Loop ports in the DB and update where necessary
 foreach ($ports as $port) {
     $port_id = $port['port_id'];
+    $ifIndex = $port['ifIndex'];
 
-    echo 'Port ' . $port['ifName'] . ': ' . $port['ifDescr'] . '(' . $port['ifIndex'] . ') ';
-    if ($port_stats[$port['ifIndex']] && $port['disabled'] != '1') {
+    $port_info_string = 'Port ' . $port['ifName'] . ': ' . $port['ifDescr'] . " ($ifIndex / #$port_id) ";
+
+    /* We don't care for disabled ports, go on */
+    if ($port['disabled'] == 1) {
+        echo "$port_info_string disabled.\n";
+        continue;
+    }
+
+    /**
+     * If this port did not show up in $port_stats before it has been deleted
+     * since the last poller run. Mark it deleted in the database and go on.
+     */
+    if (! in_array ($port_id, $ports_found)) {
+        if ($port['deleted'] != '1') {
+            dbUpdate(array('deleted' => '1'), 'ports', '`device_id` = ? AND `port_id` = ?', array($device['device_id'], $port_id));
+            echo "$port_info_string deleted.\n";
+        }
+        continue;
+    }
+
+    echo $port_info_string;
+    if ($port_stats[$ifIndex]) {
         // Check to make sure Port data is cached.
-        $this_port = &$port_stats[$port['ifIndex']];
+        $this_port = &$port_stats[$ifIndex];
 
         if ($device['os'] == 'vmware' && preg_match('/Device ([a-z0-9]+) at .*/', $this_port['ifDescr'], $matches)) {
             $this_port['ifDescr'] = $matches[1];
@@ -410,6 +466,9 @@ foreach ($ports as $port) {
                     }
                 }
                 $port['update'][$oid] = $this_port[$oid];
+                if ($oid == 'ifOperStatus' || $oid == 'ifAdminStatus') {
+                    $port['update'][$oid.'_prev'] = $port[$oid];
+                }
                 log_event($oid.': '.$port[$oid].' -> '.$this_port[$oid], $device, 'interface', $port['port_id']);
                 if ($debug) {
                     d_echo($oid.': '.$port[$oid].' -> '.$this_port[$oid].' ');
@@ -622,19 +681,8 @@ foreach ($ports as $port) {
             $updated += dbUpdate($port['update_extended'], 'ports_statistics', '`port_id` = ?', array($port_id));
             d_echo("$updated updated");
         }
-
         // End Update Database
     }
-    else if ($port['disabled'] != '1') {
-        echo 'Port Deleted';
-        // Port missing from SNMP cache.
-        if ($port['deleted'] != '1') {
-            dbUpdate(array('deleted' => '1'), 'ports', '`device_id` = ? AND `port_id` = ?', array($device['device_id'], $port_id));
-        }
-    }
-    else {
-        echo 'Port Disabled.';
-    }//end if
 
     echo "\n";
 
@@ -644,3 +692,4 @@ foreach ($ports as $port) {
 
 // Clear Variables Here
 unset($port_stats);
+unset($ports_found);

includes/snmp.inc.php

@@ -776,8 +776,8 @@ function snmp_gen_auth(&$device) {
         }
     }
     else if ($device['snmpver'] === 'v2c' or $device['snmpver'] === 'v1') {
-        $cmd  = ' -'.$device['snmpver'];
-        $cmd .= ' -c '.$device['community'];
+        $cmd  = " -".$device['snmpver'];
+        $cmd .= " -c '".$device['community']."'";
     }
     else {
         if ($debug) {

sql-schema/099.sql

@@ -0,0 +1,2 @@
+ALTER TABLE  `ports` ADD  `ifOperStatus_prev` VARCHAR( 16 ) NULL AFTER  `ifOperStatus` ;
+ALTER TABLE  `ports` ADD  `ifAdminStatus_prev` VARCHAR( 16 ) NULL AFTER  `ifAdminStatus` ;

sql-schema/100.sql

@@ -0,0 +1 @@
+INSERT INTO port_association_mode (name) values ('ifAlias');

validate.php

@@ -24,6 +24,7 @@ if (isset($options['h'])) {
         -m Any sub modules you want to run, comma separated:
           - mail: this will test your email settings  (uses default_mail option even if default_only is not set).
           - dist-poller: this will test for the install running as a distributed poller.
+          - rrdcheck: this will check to see if your rrd files are corrupt
 
         Example: ./validate.php -m mail.
 
@@ -267,6 +268,46 @@ foreach ($modules as $module) {
                 }
             }
         }
+        break;
+    case 'rrdcheck':
+
+        // Loop through the rrd_dir
+        $rrd_directory = new RecursiveDirectoryIterator($config['rrd_dir']);
+        // Filter out any non rrd files
+        $rrd_directory_filter = new RRDRecursiveFilterIterator($rrd_directory);
+        $rrd_iterator = new RecursiveIteratorIterator($rrd_directory_filter);
+        $rrd_total = iterator_count($rrd_iterator);
+        $rrd_iterator->rewind(); // Rewind iterator in case iterator_count left iterator in unknown state
+
+        echo "\nScanning ".$rrd_total." rrd files in ".$config['rrd_dir']."...\n";
+
+        // Count loops so we can push status to the user
+        $loopcount = 0;
+        $screenpad = 0;
+
+        foreach ($rrd_iterator as $filename => $file) {
+
+                $rrd_test_result = rrdtest($filename, $output, $error);
+
+                $loopcount++;
+                if (($loopcount % 50) == 0 ) {
+                        //This lets us update the previous status update without spamming in most consoles
+                        echo "\033[".$screenpad."D";
+                        $test_status = 'Status: '.$loopcount.'/'.$rrd_total;
+                        echo $test_status;
+                        $screenpad = strlen($test_status);
+                }
+
+                // A non zero result means there was some kind of error
+                if ($rrd_test_result > 0)  {
+                        echo "\033[".$screenpad."D";
+                        print_fail('Error parsing "'.$filename.'" RRD '.trim($error));
+                        $screenpad = 0;
+                }
+        }
+        echo "\033[".$screenpad."D";
+        echo "Status: ".$loopcount."/".$rrd_total." - Complete\n";
+
         break;
     }//end switch
 }//end foreach