diff --git a/LibreNMS/Config.php b/LibreNMS/Config.php index e461369aaf..e10be7eb3e 100644 --- a/LibreNMS/Config.php +++ b/LibreNMS/Config.php @@ -454,7 +454,7 @@ class Config self::deprecatedVariable('oxidized.group', 'oxidized.maps.group'); // make sure we have full path to binaries in case PATH isn't set - foreach (array('fping', 'fping6', 'snmpgetnext', 'rrdtool') as $bin) { + foreach (array('fping', 'fping6', 'snmpgetnext', 'rrdtool', 'traceroute', 'traceroute6') as $bin) { if (!is_executable(self::get($bin))) { self::set($bin, self::locateBinary($bin), $persist, $bin, "Path to $bin", 'external', 'paths'); } diff --git a/doc/Alerting/Templates.md b/doc/Alerting/Templates.md index dd022f936e..92f66eaa2e 100644 --- a/doc/Alerting/Templates.md +++ b/doc/Alerting/Templates.md @@ -50,6 +50,9 @@ Placeholders are special variables that if used within the template will be repl - ping min (if icmp enabled): `$alert->ping_min` - ping max (if icmp enabled): `$alert->ping_max` - ping avg (if icmp enabled): `$alert->ping_avg` +- debug (array) If `$config['debug']['run_trace] = true;` is set then this will contain: + - traceroute (if enabled you will receive traceroute output): `$alert->debug['traceroute']` + - output (if the traceroute fails this will contain why): `$alert->debug['output']` - Title for the Alert: `$alert->title` - Time Elapsed, Only available on recovery (`$alert->state == 0`): `$alert->elapsed` - Rule Builder (the actual rule) (use `{!! $alert->builder !!}`): `$alert->builder` @@ -108,7 +111,7 @@ More info: https://laravel.com/docs/5.4/blade#extending-a-layout ## Examples -Default Template: +#### Default Template: ```text {{ $alert->title }} Severity: {{ $alert->severity }} @@ -126,7 +129,7 @@ Alert sent to: {{ $value }} <{{ $key }}> @endforeach ``` -Ports Utilization Template: +#### Ports Utilization Template: ```text {{ $alert->title }} Device Name: {{ $alert->hostname }} @@ -143,7 +146,7 @@ Outbound Utilization: {{ (($value['ifOutOctets_rate']*8)/$value['ifSpeed'])*100 @endforeach ``` -Storage: +#### Storage: ```text {{ $alert->title }} @@ -165,7 +168,7 @@ Percent Utilized: {{ $value['storage_perc'] }} @endforeach ``` -Temperature Sensors: +#### Temperature Sensors: ```text {{ $alert->title }} @@ -191,7 +194,7 @@ High Temperature Limit: {{ $value['sensor_limit'] }} °C @endif ``` -Value Sensors: +#### Value Sensors: ```text {{ $alert->title }} @@ -217,7 +220,7 @@ Limit: {{ $value['sensor_limit'] }} @endif ``` -Memory Alert: +#### Memory Alert: ```text {{ $alert->title }} @@ -237,7 +240,9 @@ Percent Utilized: {{ $value['mempool_perc'] }} @endforeach ``` +### Advanced options +#### Conditional formatting Conditional formatting example, will display a link to the host in email or just the hostname in any other transport: ```text @if ($alert->transport == mail){{ $alert->hostname }} @@ -246,6 +251,15 @@ Conditional formatting example, will display a link to the host in email or just @endif ``` +#### Traceroute debugs +```text +@if ($alert->status == 0) + @if ($alert->status == icmp) + {{ $alert->debug['traceroute'] }} + @endif +@endif +``` + ## Examples HTML Note: To use HTML emails you must set HTML email to Yes in the WebUI under Global Settings > Alerting Settings > Email transport > Use HTML emails @@ -277,7 +291,7 @@ Service Alert: ``` -Processor Alert with Graph: +#### Processor Alert with Graph: ``` {{ $alert->title }}
Severity: {{ $alert->severity }}
diff --git a/doc/Support/Configuration.md b/doc/Support/Configuration.md index 704723b108..67d7efa950 100644 --- a/doc/Support/Configuration.md +++ b/doc/Support/Configuration.md @@ -100,6 +100,15 @@ $config['icmp_check'] = false; If you would like to do this on a per device basis then you can do so under Device -> Edit -> Misc -> Disable ICMP Test? On +#### traceroute +LibreNMS uses traceroute / traceroute6 to record debug information when a device is down due to icmp AND you have `$config['debug']['run_trace] = true;` +set. + +```php +$config['traceroute'] = '/usr/bin/traceroute'; +$config['traceroute6'] = '/usr/bin/traceroute6'; +``` + #### SNMP ```php diff --git a/html/includes/dev-overview-data.inc.php b/html/includes/dev-overview-data.inc.php index d9bfcee193..822d9b0dc7 100644 --- a/html/includes/dev-overview-data.inc.php +++ b/html/includes/dev-overview-data.inc.php @@ -1,5 +1,6 @@ $uptime_text - ".$uptime." + $uptime "; } echo ' - '; + +$perf_info = DevicePerf::where('device_id', $device['device_id'])->latest('timestamp')->first(); +$perf_debug = json_decode($perf_info['debug'], true); +if ($perf_debug['traceroute']) { + echo "
+
+
+
+

Traceroute ({$perf_info['timestamp']})

+
+
+ 
{$perf_debug['traceroute']}
+
+
+
+
"; +} + +echo ''; diff --git a/includes/alerts.inc.php b/includes/alerts.inc.php index 8532dea161..572072d456 100644 --- a/includes/alerts.inc.php +++ b/includes/alerts.inc.php @@ -22,6 +22,7 @@ * @subpackage Alerts */ +use App\Models\DevicePerf; use LibreNMS\Alert\Template; use LibreNMS\Alert\AlertData; use LibreNMS\Alerting\QueryBuilderParser; @@ -414,36 +415,37 @@ function DescribeAlert($alert) { $obj = array(); $i = 0; - $device = dbFetchRow('SELECT hostname, sysName, sysDescr, sysContact, os, type, ip, hardware, version, location, purpose, notes, uptime FROM devices WHERE device_id = ?', array($alert['device_id'])); + $device = dbFetchRow('SELECT hostname, sysName, sysDescr, sysContact, os, type, ip, hardware, version, location, purpose, notes, uptime, status, status_reason FROM devices WHERE device_id = ?', array($alert['device_id'])); $attribs = get_dev_attribs($alert['device_id']); - if (can_ping_device($attribs)) { - $ping_stats = dbFetchRow('SELECT `timestamp`, `loss`, `min`, `max`, `avg` FROM `device_perf` WHERE `device_id` = ? ORDER BY `timestamp` LIMIT 1', [$alert['device_id']]); - } - $obj['hostname'] = $device['hostname']; - $obj['sysName'] = $device['sysName']; - $obj['sysDescr'] = $device['sysDescr']; - $obj['sysContact'] = $device['sysContact']; - $obj['os'] = $device['os']; - $obj['type'] = $device['type']; - $obj['ip'] = inet6_ntop($device['ip']); - $obj['hardware'] = $device['hardware']; - $obj['version'] = $device['version']; - $obj['location'] = $device['location']; - $obj['uptime'] = $device['uptime']; - $obj['uptime_short'] = formatUptime($device['uptime'], 'short'); - $obj['uptime_long'] = formatUptime($device['uptime']); - $obj['description'] = $device['purpose']; - $obj['notes'] = $device['notes']; - $obj['alert_notes'] = $alert['note']; - $obj['device_id'] = $alert['device_id']; - $obj['rule_id'] = $alert['rule_id']; + $obj['hostname'] = $device['hostname']; + $obj['sysName'] = $device['sysName']; + $obj['sysDescr'] = $device['sysDescr']; + $obj['sysContact'] = $device['sysContact']; + $obj['os'] = $device['os']; + $obj['type'] = $device['type']; + $obj['ip'] = inet6_ntop($device['ip']); + $obj['hardware'] = $device['hardware']; + $obj['version'] = $device['version']; + $obj['location'] = $device['location']; + $obj['uptime'] = $device['uptime']; + $obj['uptime_short'] = formatUptime($device['uptime'], 'short'); + $obj['uptime_long'] = formatUptime($device['uptime']); + $obj['description'] = $device['purpose']; + $obj['notes'] = $device['notes']; + $obj['alert_notes'] = $alert['note']; + $obj['device_id'] = $alert['device_id']; + $obj['rule_id'] = $alert['rule_id']; + $obj['status'] = $device['status']; + $obj['status_reason'] = $device['status_reason']; if (can_ping_device($attribs)) { - $obj['ping_timestamp'] = $ping_stats['template']; - $obj['ping_loss'] = $ping_stats['loss']; - $obj['ping_min'] = $ping_stats['min']; - $obj['ping_max'] = $ping_stats['max']; - $obj['ping_avg'] = $ping_stats['avg']; + $ping_stats = DevicePerf::where('device_id', $alert['device_id'])->latest('timestamp')->first(); + $obj['ping_timestamp'] = $ping_stats->template; + $obj['ping_loss'] = $ping_stats->loss; + $obj['ping_min'] = $ping_stats->min; + $obj['ping_max'] = $ping_stats->max; + $obj['ping_avg'] = $ping_stats->avg; + $obj['debug'] = json_decode($ping_stats->debug, true); } $extra = $alert['details']; diff --git a/includes/functions.php b/includes/functions.php index db24f52186..da5819a0c1 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -2208,6 +2208,19 @@ function recordSnmpStatistic($stat, $start_time) return $runtime; } +function runTraceroute($device) +{ + $address_family = snmpTransportToAddressFamily($device['transport']); + $trace_name = $address_family == 'ipv6' ? 'traceroute6' : 'traceroute'; + $trace_path = Config::get($trace_name, $trace_name); + $process = new Process([$trace_path, '-q', '1', '-w', '1', $device['hostname']]); + $process->run(); + if ($process->isSuccessful()) { + return ['traceroute' => $process->getOutput()]; + } + return ['output' => $process->getErrorOutput()]; +} + /** * @param $device * @param bool $record_perf @@ -2221,7 +2234,12 @@ function device_is_up($device, $record_perf = false) $device_perf['device_id'] = $device['device_id']; $device_perf['timestamp'] = array('NOW()'); - if ($record_perf === true && can_ping_device($device['attribs']) === true) { + if ($record_perf === true && can_ping_device($device['attribs'])) { + $trace_debug = []; + if ($ping_response['result'] === false && Config::get('debug.run_trace', false)) { + $trace_debug = runTraceroute($device); + } + $device_perf['debug'] = json_encode($trace_debug); dbInsert($device_perf, 'device_perf'); } $response = array(); diff --git a/misc/db_schema.yaml b/misc/db_schema.yaml index 2f319f0faa..6062927c61 100644 --- a/misc/db_schema.yaml +++ b/misc/db_schema.yaml @@ -555,6 +555,7 @@ device_perf: - { Field: min, Type: 'double(8,2)', 'Null': false, Extra: '' } - { Field: max, Type: 'double(8,2)', 'Null': false, Extra: '' } - { Field: avg, Type: 'double(8,2)', 'Null': false, Extra: '' } + - { Field: debug, Type: text, 'Null': true, Extra: '' } Indexes: PRIMARY: { Name: PRIMARY, Columns: [id], Unique: true, Type: BTREE } device_id: { Name: device_id, Columns: [device_id], Unique: false, Type: BTREE } diff --git a/sql-schema/271.sql b/sql-schema/271.sql new file mode 100644 index 0000000000..e41946a9f4 --- /dev/null +++ b/sql-schema/271.sql @@ -0,0 +1 @@ +ALTER TABLE `device_perf` ADD `debug` TEXT NULL;