mirror/librenms-librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-10-07 16:52:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Logsize monitoring for LibreNMS (#15137)

Browse Source 
* add logsize poller

* add set size info

* add more stats

* add min size

* more logsize files

* add logsize

* fix creating log file links

* add two new graphs

* add some graphs

* add no_minus_d

* add no minus d to app page

* add count of log files

* save logsizes in app data

* rename title log sizes

* only show top 12

* add mean median and mode

* cleanup naming a bit

* the values for the files hash is now a int instead of another hash

* add new stats and correct median name

* now properly handle unseen

* fix logsize bits

* add combined

* update metrics and add some tests

* add logsize graphs to apps page

* add the logsize bit

* add logszie docs

* add some example alert rules

* style cleanup

* more style cleanup

* fix a few things for metrics and update the test

* fix a variable name in the test and update the data
This commit is contained in:
Zane C. Bowers-Hadley
2023-07-19 22:02:23 -05:00
committed by GitHub
parent b3574ee282
commit 2ac9a634b4
17 changed files with 624 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
doc/Extensions/Applications.md
View File

@@ -961,6 +961,74 @@ The application should be auto-discovered as described at the top of
the page. If it is not, please follow the steps set out under `SNMP
Extend` heading top of page.
## Logsize
### SNMP Extend
1. Download the script and make it executable.
```
wget https://raw.githubusercontent.com/librenms/librenms-agent/master/snmp/logsize -O /etc/snmp/logsize
chmod +x /etc/snmp/logsize
```
2. Install the requirements.
```
# FreeBSD
pkg install p5-File-Find-Rule p5-JSON p5-TOML p5-Time-Piece p5-MIME-Base64 p5-File-Slurp p5-Statistics-Lite
# Debian
apt-get install cpanminus
cpanm File::Find::Rule JSON TOML Time::Piece MIME::Base64 File::Slurp Statistics::Lite
```
3. Configure the config at `/usr/local/etc/logsize.conf`. You can find
the documentation for the config file in the extend. Below is a
small example.
```
# monitor log sizes of logs directly udner /var/log
[sets.var_log]
dir="/var/log/"
# monitor remote logs from network devices
[sets.remote_network]
dir="/var/log/remote/network/"
# monitor remote logs from windows sources
[sets.remote_windows]
dir="/var/log/remote/windows/"
# monitor suricata flows logs sizes
[sets.suricata_flows]
dir="/var/log/suricata/flows/current"
```
4. If the directories all readable via SNMPD, this script can be ran
via snmpd. Otherwise it needs setup in cron. Similarly is
processing a large number of files, it may also need setup in cron
if it takes the script awhile to run.
```
*/5 * * * * /etc/snmp/logsize -b 2> /dev/null > /dev/null
```
5. Make sure that `/var/cache/logsize_extend` exists and is writable
by the user running the extend.
```
mkdir -p /var/cache/logsize_extend
```
6. Configure it in the SNMPD config.
```
# if not using cron
extend logsize /etc/snmp/logsize -b
# if using cron
extend logsize /bin/cat /var/cache/logsize_extend/extend_return
```
## linux_config_files
linux_config_files is an application intended to monitor a Linux distribution's configuration files via that distribution's configuration management tool/system. At this time, ONLY RPM-based (Fedora/RHEL) SYSTEMS ARE SUPPORTED utilizing the rpmconf tool. The linux_config_files application collects and graphs the total count of configuration files that are out of sync and graphs that number.

11
includes/html/graphs/application/logsize-common.inc.php Normal file
View File

@@ -0,0 +1,11 @@
<?php
$name = 'logsize';
if (isset($vars['log_set']) && isset($vars['log_file'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, $vars['log_set'] . '_____-_____' . $vars['log_file']]);
} elseif (isset($vars['log_set'])) {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, $vars['log_set']]);
} else {
$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
}

46
includes/html/graphs/application/logsize_combined_stats.inc.php Normal file
View File

@@ -0,0 +1,46 @@
<?php
$name = 'logsize';
$app_id = $app['app_id'];
$unit_text = 'Bytes';
$colours = 'psychedelic';
$dostack = 0;
$printtotal = 1;
$addarea = 0;
$transparency = 0;
$float_precision = 3;
require 'logsize-common.inc.php';
$rrd_list = [];
if (Rrd::checkRrdExists($filename)) {
$rrd_list[] = [
'filename' => $filename,
'descr' => 'Max Size',
'ds' => 'max_size',
];
$rrd_list[] = [
'filename' => $filename,
'descr' => 'Mean Size',
'ds' => 'mean_size',
];
$rrd_list[] = [
'filename' => $filename,
'descr' => 'Median Size',
'ds' => 'median_size',
];
$rrd_list[] = [
'filename' => $filename,
'descr' => 'Mode Size',
'ds' => 'mode_size',
];
$rrd_list[] = [
'filename' => $filename,
'descr' => 'Min Size',
'ds' => 'min_size',
];
} else {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_multi_line.inc.php';

31
includes/html/graphs/application/logsize_log_sizes.inc.php Normal file
View File

@@ -0,0 +1,31 @@
<?php
$name = 'logsize';
$app_id = $app['app_id'];
$unit_text = 'Bytes';
$colours = 'rainbow';
$dostack = 0;
$printtotal = 1;
$addarea = 0;
$transparency = 0;
$float_precision = 3;
$log_files_sizes = $app->data['sets'][$vars['log_set']]['log_sizes'] ?? [];
$log_files = array_slice(array_keys($log_files_sizes), 0, 12);
$rrd_list = [];
foreach ($log_files as $index => $log_file) {
$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app['app_id'], $vars['log_set'] . '_____-_____' . $log_file]);
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => $log_file,
'ds' => 'size',
];
}
if (sizeof($rrd_list)) {
d_echo('No relevant log file RRDs found');
}
require 'includes/html/graphs/generic_multi_line.inc.php';

13
includes/html/graphs/application/logsize_max_size.inc.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
$unit_text = 'Bytes';
$descr = 'Max Size';
$ds = 'max_size';
require 'logsize-common.inc.php';
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

13
includes/html/graphs/application/logsize_mean_size.inc.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
$unit_text = 'Bytes';
$descr = 'Mean Size';
$ds = 'mean_size';
require 'logsize-common.inc.php';
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

13
includes/html/graphs/application/logsize_median_size.inc.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
$unit_text = 'Bytes';
$descr = 'Median Size';
$ds = 'median_size';
require 'logsize-common.inc.php';
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

13
includes/html/graphs/application/logsize_min_size.inc.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
$unit_text = 'Bytes';
$descr = 'Min Size';
$ds = 'min_size';
require 'logsize-common.inc.php';
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

13
includes/html/graphs/application/logsize_mode_size.inc.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
$unit_text = 'Bytes';
$descr = 'Mode Size';
$ds = 'mode_size';
require 'logsize-common.inc.php';
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

31
includes/html/graphs/application/logsize_set_sizes.inc.php Normal file
View File

@@ -0,0 +1,31 @@
<?php
$name = 'logsize';
$app_id = $app['app_id'];
$unit_text = 'Bytes';
$colours = 'psychedelic';
$dostack = 0;
$printtotal = 1;
$addarea = 0;
$transparency = 0;
$float_precision = 3;
$log_sets = Rrd::getRrdApplicationArrays($device, $app['app_id'], 'logsize');
$rrd_list = [];
foreach ($log_sets as $index => $log_set) {
if (! preg_match('/\_\_\_\_\_\-\_\_\_\_\_/', $log_set)) {
$rrd_filename = Rrd::name($device['hostname'], ['app', $name, $app['app_id'], $log_set]);
$rrd_list[] = [
'filename' => $rrd_filename,
'descr' => $log_set,
'ds' => 'size',
];
}
}
if (sizeof($rrd_list)) {
d_echo('No relevant log set RRDs found');
}
require 'includes/html/graphs/generic_multi_line.inc.php';

13
includes/html/graphs/application/logsize_size.inc.php Normal file
View File

@@ -0,0 +1,13 @@
<?php
$unit_text = 'Bytes';
$descr = 'Size';
$ds = 'size';
require 'logsize-common.inc.php';
if (! Rrd::checkRrdExists($filename)) {
d_echo('RRD "' . $filename . '" not found');
}
require 'includes/html/graphs/generic_stats.inc.php';

9
includes/html/pages/apps.inc.php
View File

@@ -457,6 +457,15 @@ $graphs['wireguard'] = [
'traffic',
'time',
];
$graphs['logsize'] = [
'size',
'set_sizes',
'max_size',
'mean_size',
'median_size',
'mode_size',
'min_size',
];
$graphs['linux_config_files'] = [
'number_of_confs',
];

106
includes/html/pages/device/apps/logsize.inc.php Normal file
View File

@@ -0,0 +1,106 @@
<?php
$link_array = [
'page' => 'device',
'device' => $device['device_id'],
'tab' => 'apps',
'app' => 'logsize',
];
$no_minus_d = $app->data['no_minus_d'] ?? false;
print_optionbar_start();
echo generate_link('Basics', $link_array);
echo ' | Sets:';
$sets = $app->data['sets'] ?? [];
$sets_list = array_keys($sets);
sort($sets_list);
foreach ($sets_list as $index => $log_set) {
$label = $vars['log_set'] == $log_set
? '<span class="pagemenu-selected">' . $log_set . '</span>'
: $log_set;
echo generate_link($label, $link_array, ['log_set' => $log_set]) . "\n";
if ($index < (count($sets_list) - 1)) {
echo ', ';
}
}
if (isset($vars['log_set']) && isset($sets[$vars['log_set']])) {
$log_files = $sets[$vars['log_set']]['files'];
$log_count = count($log_files);
echo "<br>\nFiles Count: " . $log_count . "<br>\nFiles: \n";
sort($log_files);
foreach ($log_files as $index => $log_file) {
$label = $vars['log_file'] == $log_file
? '<span class="pagemenu-selected">' . $log_file . '</span>'
: $log_file;
echo generate_link($label, $link_array, ['log_set' => $vars['log_set'], 'log_file'=>$log_file]) . "\n";
if ($index < (count($log_files) - 1)) {
echo ', ';
}
}
}
print_optionbar_end();
if (isset($vars['log_file']) && isset($vars['log_set'])) {
$graphs = [
'logsize_size' => 'Log Size',
];
} elseif (isset($vars['log_set'])) {
$graphs = [
'logsize_size' => 'Set Size',
'logsize_log_sizes' => 'Log Sizes, Top 12',
'logsize_combined_stats' => 'Combined Log Stats',
'logsize_max_size' => 'Max Log Size',
'logsize_mean_size' => 'Mean Log Size',
'logsize_median_size' => 'Median Log Size',
'logsize_mode_size' => 'Mode Log Size',
'logsize_min_size' => 'Min Log Size',
];
} else {
$graphs = [
'logsize_size' => 'Total Size',
'logsize_set_sizes' => 'Set Sizes',
'logsize_combined_stats' => 'Combined Set Stats',
'logsize_max_size' => 'Max Set Size',
'logsize_mean_size' => 'Mean Set Size',
'logsize_median_size' => 'Median Set Size',
'logsize_mode_size' => 'Mode Set Size',
'logsize_min_size' => 'Min Set Size',
];
}
foreach ($graphs as $key => $text) {
$graph_type = $key;
$graph_array['height'] = '100';
$graph_array['width'] = '215';
$graph_array['to'] = \LibreNMS\Config::get('time.now');
$graph_array['id'] = $app['app_id'];
$graph_array['type'] = 'application_' . $key;
if (isset($vars['log_set'])) {
$graph_array['log_set'] = $vars['log_set'];
}
if (isset($vars['log_file'])) {
$graph_array['log_file'] = $vars['log_file'];
}
echo '<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">' . $text . '</h3>
</div>
<div class="panel-body">
<div class="row">';
include 'includes/html/print-graphrow.inc.php';
echo '</div>';
echo '</div>';
echo '</div>';
}

115
includes/polling/applications/logsize.inc.php Normal file
View File

@@ -0,0 +1,115 @@
<?php
use LibreNMS\Exceptions\JsonAppException;
use LibreNMS\RRD\RrdDefinition;
$name = 'logsize';
try {
$returned = json_app_get($device, $name, 1);
} catch (JsonAppException $e) {
echo PHP_EOL . $name . ':' . $e->getCode() . ':' . $e->getMessage() . PHP_EOL;
update_application($app, $e->getCode() . ':' . $e->getMessage(), []); // Set empty metrics and error message
return;
}
$data = $returned['data'];
$rrd_def = RrdDefinition::make()
->addDataset('size', 'GAUGE');
$set_rrd_def = RrdDefinition::make()
->addDataset('max_size', 'GAUGE')
->addDataset('mean_size', 'GAUGE')
->addDataset('median_size', 'GAUGE')
->addDataset('mode_size', 'GAUGE')
->addDataset('min_size', 'GAUGE')
->addDataset('size', 'GAUGE');
$app_data = ['sets'=>[], 'no_minus_d'=>$data['no_minus_d']];
$rrd_name = ['app', $name, $app->app_id];
$fields = [
'max_size' => $data['max'],
'mean_size' => $data['mean'],
'median_size' => $data['median'],
'mode_size' => $data['mode'],
'min_size' => $data['min'],
'size' => $data['size'],
];
$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $set_rrd_def, 'rrd_name' => $rrd_name];
data_update($device, 'app', $tags, $fields);
$metrics = $fields;
foreach ($data['sets'] as $set_name => $set_data) {
$app_data['sets'][$set_name] = [
'files' => array_keys($set_data['files']),
'max_size' => $set_data['max'],
'mean_size' => $set_data['mean'],
'median_size' => $set_data['median'],
'mode_size' => $set_data['mode'],
'min_size' => $set_data['min'],
'size' => $set_data['size'],
'log_sizes' => [],
];
$metrics['set_' . $set_name . '_max_size'] = $set_data['max'];
$metrics['set_' . $set_name . '_mean_size'] = $set_data['mean'];
$metrics['set_' . $set_name . '_median_size'] = $set_data['median'];
$metrics['set_' . $set_name . '_mode_size'] = $set_data['mode'];
$metrics['set_' . $set_name . '_min_size'] = $set_data['min'];
$metrics['set_' . $set_name . '_size'] = $set_data['size'];
$rrd_name = ['app', $name, $app->app_id, $set_name];
$fields = [
'max_size' => $set_data['max'],
'mean_size' => $set_data['mean'],
'median_size' => $set_data['median'],
'mode_size' => $set_data['mode'],
'min_size' => $set_data['min'],
'size' => $set_data['size'],
];
$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $set_rrd_def, 'rrd_name' => $rrd_name];
data_update($device, 'app', $tags, $fields);
foreach ($set_data['files'] as $log_name => $log_size) {
$rrd_name = ['app', $name, $app->app_id, $set_name . '_____-_____' . $log_name];
$fields = [
'size' => $log_size,
];
$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
data_update($device, 'app', $tags, $fields);
$app_data['sets'][$set_name]['log_sizes'][$log_name] = $log_size;
//$metrics['set_' . $set_name . '_files_' .$log_name] = $log_size;
}
foreach ($set_data['unseen'] as $log_name) {
$rrd_name = ['app', $name, $app->app_id, $set_name . '_____-_____' . $log_name];
$fields = [
'size' => 0,
];
$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
data_update($device, 'app', $tags, $fields);
$app_data['sets'][$set_name]['log_sizes'][$log_name] = 0;
$app_data['sets'][$set_name]['files'][] = $log_name;
//$metrics['set_' . $set_name . '_files_' .$log_name] = 0;
}
uasort($app_data['sets'][$set_name]['log_sizes'], function ($a, $b) {
if ($a == $b) {
return 0;
}
return ($a > $b) ? -1 : 1;
});
}
$app->data = $app_data;
update_application($app, 'OK', $metrics);

15
misc/alert_rules.json
View File

@@ -713,6 +713,21 @@
"name": "Systemd Services Failed > 0",
"severity": "warning"
},
{
"rule": "applications.app_type = \"logsize\" && application_metrics.metric = \"max_size\" && application_metrics.value > \"5000000000\"",
"name": "Logsize: set(s) have a size > 5GB",
"severity": "warning"
},
{
"rule": "applications.app_type = \"logsize\" && application_metrics.metric like \"set_%_max_size\" && application_metrics.value > \"5000000000\"",
"name": "Logsize: set(s) has a log file(s) with a size > 5GB",
"severity": "warning"
},
{
"rule": "applications.app_type = \"logsize\" && application_metrics.metric = \"size\" && application_metrics.value > \"5000000000\"",
"name": "Logsize: total size > 5GB",
"severity": "warning"
},
{
"rule": "applications.app_type = \"zfs\" && application_metrics.metric = \"l2_errors\" && application_metrics.value >= \"1\"",
"name": "ZFS L2 errors present",

104
tests/data/linux_logsize-v1.json Normal file
View File

@@ -0,0 +1,104 @@
{
"applications": {
"discovery": {
"applications": [
{
"app_type": "logsize",
"app_state": "UNKNOWN",
"discovered": 1,
"app_state_prev": null,
"app_status": "",
"app_instance": "",
"data": null
}
]
},
"poller": {
"applications": [
{
"app_type": "logsize",
"app_state": "OK",
"discovered": 1,
"app_state_prev": "UNKNOWN",
"app_status": "",
"app_instance": "",
"data": "{\"sets\":{\"var_log\":{\"files\":[\"daemon\",\"mount\",\"devd\",\"utx\",\"setuid\",\"maillog\",\"Xorg.0\",\"xferlog\",\"mail\",\"snmpd\",\"bsdisks\",\"userlog\",\"ppp\",\"cron\",\"messages\",\"xdm\",\"dmesg\",\"security\",\"debug\",\"auth\"],\"max_size\":212661,\"mean_size\":32019.9,\"median_size\":2273.5,\"mode_size\":\"62\",\"min_size\":0,\"size\":640398,\"log_sizes\":{\"mail\":212661,\"maillog\":164271,\"Xorg.0\":64927,\"setuid\":48325,\"daemon\":46208,\"bsdisks\":38084,\"dmesg\":30308,\"cron\":25531,\"snmpd\":3796,\"userlog\":3488,\"xdm\":1059,\"auth\":724,\"mount\":371,\"messages\":331,\"xferlog\":64,\"ppp\":64,\"devd\":62,\"security\":62,\"debug\":62,\"utx\":0}}},\"no_minus_d\":null}"
}
],
"application_metrics": [
{
"metric": "max_size",
"value": 640398,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "mean_size",
"value": 640398,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "median_size",
"value": 640398,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "min_size",
"value": 640398,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "mode_size",
"value": 640398,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "set_var_log_max_size",
"value": 212661,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "set_var_log_mean_size",
"value": 32019.9,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "set_var_log_median_size",
"value": 2273.5,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "set_var_log_min_size",
"value": 0,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "set_var_log_mode_size",
"value": 62,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "set_var_log_size",
"value": 640398,
"value_prev": null,
"app_type": "logsize"
},
{
"metric": "size",
"value": 640398,
"value_prev": null,
"app_type": "logsize"
}
]
}
}
}

10
tests/snmpsim/linux_logsize-v1.snmprec Normal file
View File

@@ -0,0 +1,10 @@
1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64
1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10
1.3.6.1.2.1.1.3.0|67|77550514
1.3.6.1.2.1.1.4.0|4|<private>
1.3.6.1.2.1.1.5.0|4|<private>
1.3.6.1.2.1.1.6.0|4|<private>
1.3.6.1.2.1.25.1.1.0|67|77552962
1.3.6.1.4.1.8072.1.3.2.2.1.21.6.100.105.115.116.114.111|2|1
1.3.6.1.4.1.8072.1.3.2.2.1.21.7.108.111.103.115.105.122.101|2|1
1.3.6.1.4.1.8072.1.3.2.3.1.2.7.108.111.103.115.105.122.101|4x|7b2276657273696f6e223a312c2264617461223a7b226d6f6465223a3634303339382c226d656469616e223a3634303339382c226d65616e223a3634303339382c226661696c65645f73657473223a7b7d2c2273697a65223a3634303339382c2273657473223a7b227661725f6c6f67223a7b22756e7365656e223a5b5d2c226d656469616e223a323237332e352c226d6f6465223a223632222c226d696e223a302c2273697a65223a3634303339382c226d6178223a3231323636312c2266696c6573223a7b226461656d6f6e223a34363230382c226d6f756e74223a3337312c2264657664223a36322c22757478223a302c22736574756964223a34383332352c226d61696c6c6f67223a3136343237312c22586f72672e30223a36343932372c22786665726c6f67223a36342c226d61696c223a3231323636312c22736e6d7064223a333739362c2262736469736b73223a33383038342c22757365726c6f67223a333438382c22707070223a36342c2263726f6e223a32353533312c226d65737361676573223a3333312c2278646d223a313035392c22646d657367223a33303330382c227365637572697479223a36322c226465627567223a36322c2261757468223a3732347d2c226d65616e223a33323031392e397d7d2c226d696e223a3634303339382c226d6178223a3634303339387d2c226572726f72223a302c226572726f72537472696e67223a22227d0a