Added ability to record traceroutes for devices down due to ICMP (#9457)

* Added ability to record traceroutes for devices down due to ICMP

* Update Templates.md

* Updated schema

* Update dev-overview-data.inc.php

LibreNMS/Config.php

@@ -454,7 +454,7 @@ class Config
         self::deprecatedVariable('oxidized.group', 'oxidized.maps.group');
 
         // make sure we have full path to binaries in case PATH isn't set
-        foreach (array('fping', 'fping6', 'snmpgetnext', 'rrdtool') as $bin) {
+        foreach (array('fping', 'fping6', 'snmpgetnext', 'rrdtool', 'traceroute', 'traceroute6') as $bin) {
             if (!is_executable(self::get($bin))) {
                 self::set($bin, self::locateBinary($bin), $persist, $bin, "Path to $bin", 'external', 'paths');
             }

doc/Alerting/Templates.md

@@ -50,6 +50,9 @@ Placeholders are special variables that if used within the template will be repl
 - ping min (if icmp enabled): `$alert->ping_min`
 - ping max (if icmp enabled): `$alert->ping_max`
 - ping avg (if icmp enabled): `$alert->ping_avg`
+- debug (array) If `$config['debug']['run_trace] = true;` is set then this will contain:
+  - traceroute (if enabled you will receive traceroute output): `$alert->debug['traceroute']`
+  - output (if the traceroute fails this will contain why): `$alert->debug['output']`
 - Title for the Alert: `$alert->title`
 - Time Elapsed, Only available on recovery (`$alert->state == 0`): `$alert->elapsed`
 - Rule Builder (the actual rule) (use `{!! $alert->builder !!}`): `$alert->builder`
@@ -108,7 +111,7 @@ More info: https://laravel.com/docs/5.4/blade#extending-a-layout
 
 ## Examples
 
-Default Template:
+#### Default Template:
 ```text
 {{ $alert->title }}
 Severity: {{ $alert->severity }}
@@ -126,7 +129,7 @@ Alert sent to:
   {{ $value }} <{{ $key }}>
 @endforeach
 ```
-Ports Utilization Template:
+#### Ports Utilization Template:
 ```text
 {{ $alert->title }}
 Device Name: {{ $alert->hostname }}
@@ -143,7 +146,7 @@ Outbound Utilization: {{ (($value['ifOutOctets_rate']*8)/$value['ifSpeed'])*100
 @endforeach
 ```
 
-Storage:
+#### Storage:
 ```text
 {{ $alert->title }}
 
@@ -165,7 +168,7 @@ Percent Utilized: {{ $value['storage_perc'] }}
 @endforeach
 ```
 
-Temperature Sensors:
+#### Temperature Sensors:
 ```text
 {{ $alert->title }}
 
@@ -191,7 +194,7 @@ High Temperature Limit: {{ $value['sensor_limit'] }} °C
 @endif
 ```
 
-Value Sensors:
+#### Value Sensors:
 ```text
 {{ $alert->title }}
 
@@ -217,7 +220,7 @@ Limit: {{ $value['sensor_limit'] }}
 @endif
 ```
 
-Memory Alert:
+#### Memory Alert:
 ```text
 {{ $alert->title }}
 
@@ -237,7 +240,9 @@ Percent Utilized: {{ $value['mempool_perc'] }}
 @endforeach 
 ```
 
+### Advanced options
 
+#### Conditional formatting
 Conditional formatting example, will display a link to the host in email or just the hostname in any other transport:
 ```text
 @if ($alert->transport == mail)<a href="https://my.librenms.install/device/device={{ $alert->hostname }}/">{{ $alert->hostname }}</a>
@@ -246,6 +251,15 @@ Conditional formatting example, will display a link to the host in email or just
 @endif
 ```
 
+#### Traceroute debugs
+```text
+@if ($alert->status == 0)
+    @if ($alert->status == icmp)
+        {{ $alert->debug['traceroute'] }}
+    @endif
+@endif
+```
+
 ## Examples HTML
 
 Note: To use HTML emails you must set HTML email to Yes in the WebUI under Global Settings > Alerting Settings > Email transport > Use HTML emails
@@ -277,7 +291,7 @@ Service Alert:
 </div>
 ```
 
-Processor Alert with Graph:
+#### Processor Alert with Graph:
 ```
 {{ $alert->title }} <br>
 Severity: {{ $alert->severity }}  <br>

doc/Support/Configuration.md

@@ -100,6 +100,15 @@ $config['icmp_check'] = false;
 
 If you would like to do this on a per device basis then you can do so under Device -> Edit -> Misc -> Disable ICMP Test? On
 
+#### traceroute
+LibreNMS uses traceroute / traceroute6 to record debug information when a device is down due to icmp AND you have `$config['debug']['run_trace] = true;`
+set.
+
+```php
+$config['traceroute']  = '/usr/bin/traceroute';
+$config['traceroute6'] = '/usr/bin/traceroute6';
+```
+
 #### SNMP
 
 ```php

html/includes/dev-overview-data.inc.php

@@ -1,5 +1,6 @@
 <?php
 
+use App\Models\DevicePerf;
 use LibreNMS\Exceptions\InvalidIpException;
 use LibreNMS\Util\IP;
 
@@ -126,12 +127,30 @@ if (is_array($loc)) {
 if ($uptime) {
     echo "<tr>
         <td>$uptime_text</td>
-        <td>".$uptime."</td>
+        <td>$uptime</td>
       </tr>";
 }
 
 echo '</table>
-      </div>
       </div>
       </div>
       </div>';
+
+$perf_info = DevicePerf::where('device_id', $device['device_id'])->latest('timestamp')->first();
+$perf_debug = json_decode($perf_info['debug'], true);
+if ($perf_debug['traceroute']) {
+    echo "<div class='row'>
+     <div class='col-md-12'>
+         <div class='panel panel-default'>
+             <div class='panel-heading'>
+                 <h3 class='panel-title'>Traceroute ({$perf_info['timestamp']})</h3>
+             </div>
+             <div class='panel-body'>
+                 <pre>{$perf_debug['traceroute']}</pre>
+            </div>
+         </div>
+     </div>
+ </div>";
+}
+
+echo '</div>';

includes/alerts.inc.php

@@ -22,6 +22,7 @@
  * @subpackage Alerts
  */
 
+use App\Models\DevicePerf;
 use LibreNMS\Alert\Template;
 use LibreNMS\Alert\AlertData;
 use LibreNMS\Alerting\QueryBuilderParser;
@@ -414,36 +415,37 @@ function DescribeAlert($alert)
 {
     $obj         = array();
     $i           = 0;
-    $device      = dbFetchRow('SELECT hostname, sysName, sysDescr, sysContact, os, type, ip, hardware, version, location, purpose, notes, uptime FROM devices WHERE device_id = ?', array($alert['device_id']));
+    $device      = dbFetchRow('SELECT hostname, sysName, sysDescr, sysContact, os, type, ip, hardware, version, location, purpose, notes, uptime, status, status_reason FROM devices WHERE device_id = ?', array($alert['device_id']));
     $attribs     = get_dev_attribs($alert['device_id']);
-    if (can_ping_device($attribs)) {
-        $ping_stats = dbFetchRow('SELECT `timestamp`, `loss`, `min`, `max`, `avg` FROM `device_perf` WHERE `device_id` = ? ORDER BY `timestamp` LIMIT 1', [$alert['device_id']]);
-    }
 
-    $obj['hostname']     = $device['hostname'];
-    $obj['sysName']      = $device['sysName'];
-    $obj['sysDescr']     = $device['sysDescr'];
-    $obj['sysContact']   = $device['sysContact'];
-    $obj['os']           = $device['os'];
-    $obj['type']         = $device['type'];
-    $obj['ip']           = inet6_ntop($device['ip']);
-    $obj['hardware']     = $device['hardware'];
-    $obj['version']      = $device['version'];
-    $obj['location']     = $device['location'];
-    $obj['uptime']       = $device['uptime'];
-    $obj['uptime_short'] = formatUptime($device['uptime'], 'short');
-    $obj['uptime_long']  = formatUptime($device['uptime']);
-    $obj['description']  = $device['purpose'];
-    $obj['notes']        = $device['notes'];
-    $obj['alert_notes']  = $alert['note'];
-    $obj['device_id']    = $alert['device_id'];
-    $obj['rule_id']      = $alert['rule_id'];
+    $obj['hostname']      = $device['hostname'];
+    $obj['sysName']       = $device['sysName'];
+    $obj['sysDescr']      = $device['sysDescr'];
+    $obj['sysContact']    = $device['sysContact'];
+    $obj['os']            = $device['os'];
+    $obj['type']          = $device['type'];
+    $obj['ip']            = inet6_ntop($device['ip']);
+    $obj['hardware']      = $device['hardware'];
+    $obj['version']       = $device['version'];
+    $obj['location']      = $device['location'];
+    $obj['uptime']        = $device['uptime'];
+    $obj['uptime_short']  = formatUptime($device['uptime'], 'short');
+    $obj['uptime_long']   = formatUptime($device['uptime']);
+    $obj['description']   = $device['purpose'];
+    $obj['notes']         = $device['notes'];
+    $obj['alert_notes']   = $alert['note'];
+    $obj['device_id']     = $alert['device_id'];
+    $obj['rule_id']       = $alert['rule_id'];
+    $obj['status']        = $device['status'];
+    $obj['status_reason'] = $device['status_reason'];
     if (can_ping_device($attribs)) {
-        $obj['ping_timestamp'] = $ping_stats['template'];
-        $obj['ping_loss']      = $ping_stats['loss'];
-        $obj['ping_min']       = $ping_stats['min'];
-        $obj['ping_max']       = $ping_stats['max'];
-        $obj['ping_avg']       = $ping_stats['avg'];
+        $ping_stats = DevicePerf::where('device_id', $alert['device_id'])->latest('timestamp')->first();
+        $obj['ping_timestamp'] = $ping_stats->template;
+        $obj['ping_loss']      = $ping_stats->loss;
+        $obj['ping_min']       = $ping_stats->min;
+        $obj['ping_max']       = $ping_stats->max;
+        $obj['ping_avg']       = $ping_stats->avg;
+        $obj['debug']          = json_decode($ping_stats->debug, true);
     }
     $extra               = $alert['details'];
 

includes/functions.php

@@ -2208,6 +2208,19 @@ function recordSnmpStatistic($stat, $start_time)
     return $runtime;
 }
 
+function runTraceroute($device)
+{
+    $address_family = snmpTransportToAddressFamily($device['transport']);
+    $trace_name = $address_family == 'ipv6' ? 'traceroute6' : 'traceroute';
+    $trace_path = Config::get($trace_name, $trace_name);
+    $process = new Process([$trace_path, '-q', '1', '-w', '1', $device['hostname']]);
+    $process->run();
+    if ($process->isSuccessful()) {
+        return ['traceroute' => $process->getOutput()];
+    }
+    return ['output' => $process->getErrorOutput()];
+}
+
 /**
  * @param $device
  * @param bool $record_perf
@@ -2221,7 +2234,12 @@ function device_is_up($device, $record_perf = false)
     $device_perf['device_id'] = $device['device_id'];
     $device_perf['timestamp'] = array('NOW()');
 
-    if ($record_perf === true && can_ping_device($device['attribs']) === true) {
+    if ($record_perf === true && can_ping_device($device['attribs'])) {
+        $trace_debug = [];
+        if ($ping_response['result'] === false && Config::get('debug.run_trace', false)) {
+            $trace_debug = runTraceroute($device);
+        }
+        $device_perf['debug'] = json_encode($trace_debug);
         dbInsert($device_perf, 'device_perf');
     }
     $response              = array();

misc/db_schema.yaml

@@ -555,6 +555,7 @@ device_perf:
     - { Field: min, Type: 'double(8,2)', 'Null': false, Extra: '' }
     - { Field: max, Type: 'double(8,2)', 'Null': false, Extra: '' }
     - { Field: avg, Type: 'double(8,2)', 'Null': false, Extra: '' }
+    - { Field: debug, Type: text, 'Null': true, Extra: '' }
   Indexes:
     PRIMARY: { Name: PRIMARY, Columns: [id], Unique: true, Type: BTREE }
     device_id: { Name: device_id, Columns: [device_id], Unique: false, Type: BTREE }

sql-schema/271.sql

@@ -0,0 +1 @@
+ALTER TABLE `device_perf` ADD `debug` TEXT NULL;