feature: Optional partial port polling (#5805)

* feature: Added ability to switch port polling mode

* update, speed things up slightly by walking certain tables

* more fixes

* updates

* updates

* update to exclude ports that are down

* wrong variable used

* update to set previous port status when null

* small update to port disco

doc/Support/Performance.md

@@ -69,3 +69,23 @@ The default 16 threads that `poller-wrapper.py` runs as isn't necessarily the op
 
 If your install uses hostnames for devices and you have quite a lot then it's advisable to setup a local recursive dns instance on the 
 LibreNMS server. Something like pdns-recursor can be used and then configure `/etc/resolv.conf` to use 127.0.0.1 for queries.
+
+#### Per port polling - experimental
+
+By default the polling ports module will walk ifXEntry + some items from ifEntry regardless of the port. So if a port is marked as deleted because you don't want to see them 
+or it's disabled then we still collect data. For the most part this is fine as the walks are quite quick. However for devices with a lot of ports and good % of those are 
+either deleted or disabled then this approach isn't optimal. So to counter this you can enable 'selected port polling' per device within the edit device -> misc section or by
+globally enabling it (not recommended): `$config['polling']['selected_ports'] = true;`.
+
+If you would like to see if you should turn this on then run this query in MySQL: `select device_id, count(*) as total from ports where deleted=1 group by device_id order by total desc;`. You will see output like the following:
+
++-----------+-------+
+| device_id | total |
++-----------+-------+
+|       128 |   339 |
+|        92 |    56 |
+|        41 |    41 |
+|        38 |     3 |
+|        81 |     2 |
+
+Here device id 128 and potentially 92 and 41 are likely candidates for this feature to be enabled on. Turn it on and then closely monitor the device for the next 15-30 minutes.

html/pages/device/edit/misc.inc.php

@@ -26,5 +26,11 @@ echo '
             '.dynamic_override_config('checkbox', 'override_rrdtool_tune', $device).'
         </div>
     </div>
+    <div class="form-group">
+        <label for="selected_ports" class="col-sm-4 control-label">Enable selected port polling?</label>
+        <div class="col-sm-8">
+            '.dynamic_override_config('checkbox', 'selected_ports', $device).'
+        </div>
+    </div>
 </form>
 ';

includes/discovery/ports.inc.php

@@ -4,6 +4,7 @@
 $port_stats = array();
 $port_stats = snmpwalk_cache_oid($device, 'ifDescr', $port_stats, 'IF-MIB');
 $port_stats = snmpwalk_cache_oid($device, 'ifName', $port_stats, 'IF-MIB');
+$port_stats = snmpwalk_cache_oid($device, 'ifAlias', $port_stats, 'IF-MIB');
 $port_stats = snmpwalk_cache_oid($device, 'ifType', $port_stats, 'IF-MIB');
 
 // End Building SNMP Cache Array
@@ -42,14 +43,15 @@ foreach ($port_stats as $ifIndex => $port) {
     // Store ifIndex in port entry and prefetch ifName as we'll need it multiple times
     $port['ifIndex'] = $ifIndex;
     $ifName = $port['ifName'];
+    $ifAlias = $port['ifAlias'];
+    $ifDescr = $port['ifDescr'];
 
     // Get port_id according to port_association_mode used for this device
     $port_id = get_port_id($ports_mapped, $port, $port_association_mode);
-
     if (is_port_valid($port, $device)) {
         // Port newly discovered?
         if (! is_array($ports_db[$port_id])) {
-            $port_id         = dbInsert(array('device_id' => $device['device_id'], 'ifIndex' => $ifIndex, 'ifName' => $ifName), 'ports');
+            $port_id         = dbInsert(array('device_id' => $device['device_id'], 'ifIndex' => $ifIndex, 'ifName' => $ifName, 'ifAlias' => $ifAlias, 'ifDescr' => $ifDescr), 'ports');
             $ports[$port_id] = dbFetchRow('SELECT * FROM `ports` WHERE `device_id` = ? AND `port_id` = ?', array($device['device_id'], $port_id));
             echo 'Adding: '.$ifName.'('.$ifIndex.')('.$port_id.')';
         } // Port re-discovered after previous deletion?
@@ -67,7 +69,7 @@ foreach ($port_stats as $ifIndex => $port) {
     else {
         if (is_array($ports_db[$port_id])) {
             if ($ports_db[$port_id]['deleted'] != '1') {
-                dbUpdate(array('deleted' => '1'), 'ports', '`port_id` = ?', array($port_id));
+                dbUpdate(array('deleted' => '1'), 'ports', "`port_id` = ?, `ifName` => '?', `ifAlias` => '?', `ifDescr` => '?'", array($port_id, $ifName, $ifAlias, $ifDescr));
                 $ports_db[$port_id]['deleted'] = '1';
                 echo '-';
             }

includes/polling/ports.inc.php

@@ -122,21 +122,107 @@ $ifmib_oids = array(
     'ifOutDiscards',
 );
 
+$table_base_oids = array(
+    'ifName',
+    'ifAlias',
+    'ifDescr',
+    'ifHighSpeed',
+    'ifOperStatus',
+    'ifAdminStatus',
+);
+
+$hc_oids = array(
+    'ifInMulticastPkts',
+    'ifInBroadcastPkts',
+    'ifOutMulticastPkts',
+    'ifOutBroadcastPkts',
+    'ifHCInOctets',
+    'ifHCInUcastPkts',
+    'ifHCInMulticastPkts',
+    'ifHCInBroadcastPkts',
+    'ifHCOutOctets',
+    'ifHCOutUcastPkts',
+    'ifHCOutMulticastPkts',
+    'ifHCOutBroadcastPkts',
+    'ifPromiscuousMode',
+    'ifConnectorPresent',
+);
+
+$nonhc_oids = array(
+    'ifSpeed',
+    'ifInOctets',
+    'ifInUcastPkts',
+    'ifInUnknownProtos',
+    'ifOutOctets',
+    'ifOutUcastPkts',
+);
+
+$shared_oids = array(
+    'ifInErrors',
+    'ifOutErrors',
+    'ifInNUcastPkts',
+    'ifOutNUcastPkts',
+    'ifInDiscards',
+    'ifOutDiscards',
+    'ifPhysAddress',
+    'ifLastChange',
+    'ifType',
+    'ifMtu',
+);
+
 echo 'Caching Oids: ';
+$port_stats = array();
+$data       = array();
 
 if ($device['os'] === 'f5' && (version_compare($device['version'], '11.2.0', '>=') && version_compare($device['version'], '11.7', '<'))) {
     require_once 'ports/f5.inc.php';
 } else {
-    if (!in_array(strtolower($device['hardware']), array_map('strtolower', $config['os'][$device['os']]['bad_ifXEntry']))) {
-        $port_stats = snmpwalk_cache_oid($device, 'ifXEntry', $port_stats, 'IF-MIB');
-    }
-    $hc_test = array_slice($port_stats, 0, 1);
-    if (!isset($hc_test[0]['ifHCInOctets']) && !is_numeric($hc_test[0]['ifHCInOctets'])) {
-        $port_stats = snmpwalk_cache_oid($device, 'ifEntry', $port_stats, 'IF-MIB', null, '-OQUst');
+    if ($config['polling']['selected_ports'] === true || $device['attribs']['selected_ports'] == 'true') {
+        echo('Select ports polling');
+        foreach ($table_base_oids as $oid) {
+            $data = snmpwalk_cache_oid($device, $oid, $data, 'IF-MIB');
+        }
+        unset(
+            $oid,
+            $table_base_oids
+        );
+        $lports = dbFetchRows("SELECT * FROM `ports` where `device_id` = ? AND `deleted` = 0 AND `disabled` = 0", array($device['device_id']));
+        foreach ($lports as $lport) {
+            if (is_port_valid($lport, $device)) {
+                $i = $lport['ifIndex'];
+                if ($lport['ifAdminStatus_prev'] === 'down' && $data[$i]['ifAdminStatus'] === 'down') {
+                    echo 'port is still admin down';
+                } elseif ($lport['ifOperStatus_prev'] === 'down' && $data[$i]['ifOperStatus'] === 'down') {
+                    echo 'port is still down';
+                } else {
+                    echo 'valid';
+                    if (is_numeric($data[$i]['ifHighSpeed'])) {
+                        $full_oids = array_merge($hc_oids, $shared_oids);
+                    } else {
+                        $full_oids = array_merge($nonhc_oids, $shared_oids);
+                    }
+                    $oids = implode(".$i ", $full_oids) . ".$i";
+                    unset($full_oids);
+                    if (is_array($data[$i])) {
+                        $port_stats[$i] = $data[$i];
+                    }
+                    $port_stats = snmp_get_multi($device, $oids, '-OQUst', 'IF-MIB', null, $port_stats);
+                }
+            }
+        }
+        unset($data);
     } else {
-        foreach ($ifmib_oids as $oid) {
-            echo "$oid ";
-            $port_stats = snmpwalk_cache_oid($device, $oid, $port_stats, 'IF-MIB', null, '-OQUst');
+        if (!in_array(strtolower($device['hardware']), array_map('strtolower', $config['os'][$device['os']]['bad_ifXEntry']))) {
+            $port_stats = snmpwalk_cache_oid($device, 'ifXEntry', $port_stats, 'IF-MIB');
+        }
+        $hc_test = array_slice($port_stats, 0, 1);
+        if (!isset($hc_test[0]['ifHCInOctets']) && !is_numeric($hc_test[0]['ifHCInOctets'])) {
+            $port_stats = snmpwalk_cache_oid($device, 'ifEntry', $port_stats, 'IF-MIB', null, '-OQUst');
+        } else {
+            foreach ($ifmib_oids as $oid) {
+                echo "$oid ";
+                $port_stats = snmpwalk_cache_oid($device, $oid, $port_stats, 'IF-MIB', null, '-OQUst');
+            }
         }
     }
 }
@@ -510,6 +596,12 @@ foreach ($ports as $port) {
                 } else {
                     echo $oid.' ';
                 }
+            } else {
+                if ($oid == 'ifOperStatus' || $oid == 'ifAdminStatus') {
+                    if ($port[$oid.'_prev'] == null) {
+                        $port['update'][$oid.'_prev'] = $this_port[$oid];
+                    }
+                }
             }
         }//end foreach
 
@@ -554,7 +646,7 @@ foreach ($ports as $port) {
 
             if ($config['slow_statistics'] == true) {
                 $port[$port_update][$oid]         = set_numeric($this_port[$oid]);
-                $port[$port_update][$oid.'_prev'] = $port[$oid];
+                $port[$port_update][$oid.'_prev'] = set_numeric($port[$oid]);
             }
 
             $oid_prev = $oid.'_prev';

includes/snmp.inc.php

@@ -184,7 +184,7 @@ function gen_snmp_cmd($cmd, $device, $oids, $options = null, $mib = null, $mibdi
     return $cmd;
 } // end gen_snmp_cmd()
 
-function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir = null)
+function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir = null, $array = array())
 {
     $time_start = microtime(true);
 
@@ -195,8 +195,6 @@ function snmp_get_multi($device, $oids, $options = '-OQUs', $mib = null, $mibdir
     $cmd = gen_snmpget_cmd($device, $oids, $options, $mib, $mibdir);
     $data = trim(external_exec($cmd));
 
-
-    $array = array();
     foreach (explode("\n", $data) as $entry) {
         list($oid,$value)  = explode('=', $entry, 2);
         $oid               = trim($oid);