From 8f8872520a66b4b81e04a176e47b42944259343e Mon Sep 17 00:00:00 2001 From: Henne Van Och Date: Thu, 13 Jan 2022 18:19:27 +0100 Subject: [PATCH] Add supervisord application (#13673) --- doc/Extensions/Applications.md | 28 ++++ .../application/supervisord_totals.inc.php | 43 ++++++ .../application/supervisord_uptime.inc.php | 38 ++++++ includes/html/pages/apps.inc.php | 4 + .../pages/device/apps/supervisord.inc.php | 26 ++++ .../polling/applications/supervisord.inc.php | 52 ++++++++ tests/data/linux_supervisord-v1.json | 123 ++++++++++++++++++ tests/snmpsim/linux_supervisord-v1.snmprec | 8 ++ 8 files changed, 322 insertions(+) create mode 100644 includes/html/graphs/application/supervisord_totals.inc.php create mode 100644 includes/html/graphs/application/supervisord_uptime.inc.php create mode 100644 includes/html/pages/device/apps/supervisord.inc.php create mode 100644 includes/polling/applications/supervisord.inc.php create mode 100644 tests/data/linux_supervisord-v1.json create mode 100644 tests/snmpsim/linux_supervisord-v1.snmprec diff --git a/doc/Extensions/Applications.md b/doc/Extensions/Applications.md index 22ebca835e..ae41d70a10 100644 --- a/doc/Extensions/Applications.md +++ b/doc/Extensions/Applications.md @@ -137,6 +137,7 @@ by following the steps under the `SNMP Extend` heading. 1. [Seafile](#seafile) - SNMP extend 1. [SMART](#smart) - SNMP extend 1. [Squid](#squid) - SNMP proxy +1. [Supervisord](#supervisord) - SNMP extend 1. [TinyDNS/djbdns](#tinydns-aka-djbdns) - Agent 1. [Unbound](#unbound) - SNMP extend, Agent 1. [UPS-nut](#ups-nut) - SNMP extend @@ -2182,6 +2183,33 @@ for net-snmp, please see the links below. +## Supervisord + +It shows you the totals per status and also the uptime per process. That way you can add alerts for instance when there are process in state `FATAL`. + +### SNMP Extend + +1. Copy the python script to the desired host. +``` +wget https://github.com/librenms/librenms-agent/raw/master/snmp/supervisord.py -O /etc/snmp/supervisord.py +``` +Notice that this will use the default unix socket path. Modify the `unix_socket_path` variable in the script if your path differs from the default. + +2. Make the script executable +``` +chmod +x /etc/snmp/supervisord.py +``` + +3. Edit your snmpd.conf file (usually /etc/snmp/snmpd.conf) and add: +``` +extend supervisord /etc/snmp/supervisord.py +``` + +4. Restart snmpd on your host +``` +systemctl restart snmpd +``` + ## TinyDNS aka djbdns ### Agent diff --git a/includes/html/graphs/application/supervisord_totals.inc.php b/includes/html/graphs/application/supervisord_totals.inc.php new file mode 100644 index 0000000000..9dcf804755 --- /dev/null +++ b/includes/html/graphs/application/supervisord_totals.inc.php @@ -0,0 +1,43 @@ + ['descr' => 'STOPPED', 'colour' => 'FFC107'], + 'STARTING' => ['descr' => 'STARTING', 'colour' => '28774F'], + 'RUNNING' => ['descr' => 'RUNNING', 'colour' => '4CAf50'], + 'BACKOFF' => ['descr' => 'BACKOFF', 'colour' => 'CDDC39'], + 'STOPPING' => ['descr' => 'STOPPING', 'colour' => 'D46A6A'], + 'EXITED' => ['descr' => 'EXITED', 'colour' => 'E74B00'], + 'FATAL' => ['descr' => 'FATAL', 'colour' => 'E91E63'], + 'UNKNOWN' => ['descr' => 'UNKNOWN', 'colour' => 'CCCCCC'], +]; + +$i = 0; +if (Rrd::checkRrdExists($rrd_filename)) { + foreach ($array as $ds => $var) { + $rrd_list[$i]['filename'] = $rrd_filename; + $rrd_list[$i]['descr'] = $var['descr']; + $rrd_list[$i]['ds'] = $ds; + $rrd_list[$i]['colour'] = $var['colour']; + $i++; + } +} else { + echo "file missing: $rrd_filename"; +} + +require 'includes/html/graphs/generic_v3_multiline.inc.php'; diff --git a/includes/html/graphs/application/supervisord_uptime.inc.php b/includes/html/graphs/application/supervisord_uptime.inc.php new file mode 100644 index 0000000000..de513df493 --- /dev/null +++ b/includes/html/graphs/application/supervisord_uptime.inc.php @@ -0,0 +1,38 @@ + $rrd_filename, + 'descr' => $process_name, + 'ds' => $rrdVar, + ]; + } + $int++; +} + +require 'includes/html/graphs/generic_multi_line_exact_numbers.inc.php'; diff --git a/includes/html/pages/apps.inc.php b/includes/html/pages/apps.inc.php index 4844a4fc50..66362b6344 100644 --- a/includes/html/pages/apps.inc.php +++ b/includes/html/pages/apps.inc.php @@ -367,6 +367,10 @@ $graphs['docker'] = [ 'mem_used', 'mem_perc', ]; +$graphs['supervisord'] = [ + 'totals', + 'uptime', +]; $graphs['chronyd'] = [ 'time', 'frequency', diff --git a/includes/html/pages/device/apps/supervisord.inc.php b/includes/html/pages/device/apps/supervisord.inc.php new file mode 100644 index 0000000000..e0c35c1d8d --- /dev/null +++ b/includes/html/pages/device/apps/supervisord.inc.php @@ -0,0 +1,26 @@ + 'Totals', + 'supervisord_uptime' => 'Process uptime', +]; + +foreach ($graphs as $key => $text) { + $graph_type = $key; + $graph_array['height'] = '100'; + $graph_array['width'] = '215'; + $graph_array['to'] = time(); + $graph_array['id'] = $app['app_id']; + $graph_array['type'] = 'application_' . $key; + + echo '
+
+

' . $text . '

+
+
+
'; + include 'includes/html/print-graphrow.inc.php'; + echo '
'; + echo '
'; + echo '
'; +} diff --git a/includes/polling/applications/supervisord.inc.php b/includes/polling/applications/supervisord.inc.php new file mode 100644 index 0000000000..b07957c1e9 --- /dev/null +++ b/includes/polling/applications/supervisord.inc.php @@ -0,0 +1,52 @@ +getCode() . ':' . $e->getMessage() . PHP_EOL; + update_application($app, 'ERROR', []); // Set empty metrics and error message + + return; +} + +$rrd_name = ['app', $name, $app_id]; + +$metrics = []; +$rrd_def = RrdDefinition::make(); +foreach ($supervisord_data['total'] as $status => $value) { + $rrd_def->addDataset($status, 'GAUGE', 0); +} + +$fields = $supervisord_data['total']; + +$metrics['total'] = $fields; +$tags = ['name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name]; +data_update($device, 'app', $tags, $fields); + +$rrd_def = RrdDefinition::make() + ->addDataset('state', 'GAUGE', 0) + ->addDataset('uptime', 'GAUGE', 0); + +foreach ($supervisord_data['processes'] as $data) { + $process = $data['name']; + + $rrd_name = ['app', $name, $app_id, $process]; + + $fields = [ + 'state' => $data['state'], + 'uptime' => $data['uptime'], + ]; + + $metrics['process_' . $process] = $fields; + $tags = ['name' => $process, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name]; + data_update($device, 'app', $tags, $fields); +} + +update_application($app, $output, $metrics); diff --git a/tests/data/linux_supervisord-v1.json b/tests/data/linux_supervisord-v1.json new file mode 100644 index 0000000000..ca1f4e38a8 --- /dev/null +++ b/tests/data/linux_supervisord-v1.json @@ -0,0 +1,123 @@ +{ + "applications": { + "discovery": { + "applications": [ + { + "app_type": "supervisord", + "app_state": "UNKNOWN", + "discovered": 1, + "app_state_prev": null, + "app_status": "", + "app_instance": "" + } + ] + }, + "poller": { + "applications": [ + { + "app_type": "supervisord", + "app_state": "OK", + "discovered": 1, + "app_state_prev": "UNKNOWN", + "app_status": "", + "app_instance": "" + } + ], + "application_metrics": [ + { + "metric": "process_test-program_00_state", + "value": 20, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "process_test-program_00_uptime", + "value": 70, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "process_test-program_01_state", + "value": 200, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "process_test-program_01_uptime", + "value": 0, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_BACKOFF", + "value": 0, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_EXITED", + "value": 0, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_FATAL", + "value": 1, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_RUNNING", + "value": 1, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_STARTING", + "value": 0, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_STOPPED", + "value": 0, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_STOPPING", + "value": 0, + "value_prev": null, + "app_type": "supervisord" + }, + { + "metric": "total_UNKNOWN", + "value": 0, + "value_prev": null, + "app_type": "supervisord" + } + ] + } + }, + "os": { + "discovery": { + "devices": [ + { + "sysName": "", + "sysObjectID": ".1.3.6.1.4.1.8072.3.2.10", + "sysDescr": "Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64", + "sysContact": "", + "version": "3.10.0-693.5.2.el7.x86_64", + "hardware": "Generic x86 64-bit", + "features": null, + "os": "linux", + "type": "server", + "serial": null, + "icon": "linux.svg", + "location": "" + } + ] + }, + "poller": "matches discovery" + } +} diff --git a/tests/snmpsim/linux_supervisord-v1.snmprec b/tests/snmpsim/linux_supervisord-v1.snmprec new file mode 100644 index 0000000000..dee95cdf6a --- /dev/null +++ b/tests/snmpsim/linux_supervisord-v1.snmprec @@ -0,0 +1,8 @@ +1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64 +1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10 +1.3.6.1.2.1.1.3.0|67|77550514 +1.3.6.1.2.1.1.4.0|4| +1.3.6.1.2.1.1.5.0|4| +1.3.6.1.2.1.1.6.0|4| +1.3.6.1.4.1.8072.1.3.2.2.1.21.11.115.117.112.101.114.118.105.115.111.114.100|2|1 +1.3.6.1.4.1.8072.1.3.2.3.1.2.11.115.117.112.101.114.118.105.115.111.114.100|4|{"version":1,"data":{"processes":[{"start":1641984698,"state":20,"now":1641984768,"group":"test","name":"test-program_00","error":null,"statename":"RUNNING","stop":1641984697,"uptime":70},{"start":1641982882,"state":200,"now":1641984768,"group":"test","name":"test-program_01","error":"Exited too quickly (process log may have details)","statename":"FATAL","stop":1641982882,"uptime":0}],"total":{"RUNNING":1,"STOPPED":0,"STOPPING":0,"UNKNOWN":0,"FATAL":1,"STARTING":0,"EXITED":0,"BACKOFF":0}},"errorString":0,"error":0}