add Suricata Extract submission stats app (#15105)

* add suricata_extract * convert from dervive to gauge and use delta * add suricata_extract to includes/html/pages/apps.inc.php * graph cleanup * add sub_size * add sub_size graph * add docs for suricata extract * add tests for suricata_extract * add rules for suricata extract * minor test tweaks
2024-10-07 16:52:45 +00:00 · 2023-06-25 13:48:26 -05:00
parent 17f503b40a
commit 00cf300d1a
22 changed files with 472 additions and 0 deletions
--- a/doc/Extensions/Applications.md
+++ b/doc/Extensions/Applications.md
@@ -2806,6 +2806,20 @@ setup. If the default does not work, check the docs for it at
 suricata_stat_check](https://metacpan.org/dist/Suricata-Monitoring/view/bin/suricata_stat_check)


+## Suricata Extract
+
+### SNMP
+
+1. Add the following to your snmpd config and restart. Path may have
+to be adjusted depending on where `suricata_extract_submit_extend` is
+installed to.
+```
+extend suricata_extract /usr/local/bin/suricata_extract_submit_extend
+```
+
+Then just wait for the system to be rediscovered or enable it manually
+for the server in question.
+
 ## Systemd

 The systemd application polls systemd and scrapes systemd units' load, activation, and sub states.
--- a/includes/html/graphs/application/suricata_extract_errors.inc.php
+++ b/includes/html/graphs/application/suricata_extract_errors.inc.php
@@ -0,0 +1,11 @@
+<?php
+
+$ds = 'errors';
+$unit_text = 'Errors';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_ignored_host.inc.php
+++ b/includes/html/graphs/application/suricata_extract_ignored_host.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'ignored_host';
+$unit_text = 'Ignored';
+$descr = 'By Host';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_ignored_ip.inc.php
+++ b/includes/html/graphs/application/suricata_extract_ignored_ip.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'ignored_ip';
+$unit_text = 'Ignored';
+$descr = 'By IP';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_ignored_ip_dest.inc.php
+++ b/includes/html/graphs/application/suricata_extract_ignored_ip_dest.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'ignored_ip_dest';
+$unit_text = 'Ignored';
+$descr = 'By IP Dest';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_ignored_ip_src.inc.php
+++ b/includes/html/graphs/application/suricata_extract_ignored_ip_src.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'ignored_ip_src';
+$unit_text = 'Ignored';
+$descr = 'By IP Src';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub.inc.php
@@ -0,0 +1,11 @@
+<?php
+
+$ds = 'sub';
+$unit_text = 'Submissions';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub_2xx.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub_2xx.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'sub_2xx';
+$unit_text = 'HTML Status';
+$descr = '2xx';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub_3xx.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub_3xx.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'sub_3xx';
+$unit_text = 'HTML Status';
+$descr = '3xx';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub_4xx.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub_4xx.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'sub_4xx';
+$unit_text = 'HTML Status';
+$descr = '4xx';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub_5xx.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub_5xx.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'sub_5xx';
+$unit_text = 'HTML Status';
+$descr = '5xx';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub_codes.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub_codes.inc.php
@@ -0,0 +1,35 @@
+<?php
+
+require 'includes/html/graphs/common.inc.php';
+
+$scale_min = 0;
+$nototal = 1;
+$unit_text = 'HTTP Code';
+$unitlen = 15;
+$bigdescrlen = 20;
+$smalldescrlen = 15;
+$colours = 'rainbow';
+
+$array = [
+    'sub_2xx' => '2xx',
+    'sub_3xx' => '3xx',
+    'sub_4xx' => '4xx',
+    'sub_5xx' => '5xx',
+];
+
+$rrd_filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+$rrd_list = [];
+if (Rrd::checkRrdExists($rrd_filename)) {
+    $i = 0;
+    foreach ($array as $ds => $descr) {
+        $rrd_list[$i]['filename'] = $rrd_filename;
+        $rrd_list[$i]['descr'] = $descr;
+        $rrd_list[$i]['ds'] = $ds;
+        $i++;
+    }
+} else {
+    echo "file missing: $rrd_filename";
+}
+
+require 'includes/html/graphs/generic_multi_line_exact_numbers.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub_fail.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub_fail.inc.php
@@ -0,0 +1,11 @@
+<?php
+
+$ds = 'sub_fail';
+$unit_text = 'Sub Fails';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_sub_size.inc.php
+++ b/includes/html/graphs/application/suricata_extract_sub_size.inc.php
@@ -0,0 +1,11 @@
+<?php
+
+$ds = 'sub_size';
+$unit_text = 'Bytes';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_truncated.inc.php
+++ b/includes/html/graphs/application/suricata_extract_truncated.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'truncated';
+$unit_text = 'Files';
+$descr = 'Truncated';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/graphs/application/suricata_extract_zero_sized.inc.php
+++ b/includes/html/graphs/application/suricata_extract_zero_sized.inc.php
@@ -0,0 +1,12 @@
+<?php
+
+$ds = 'zero_sized';
+$unit_text = 'Files';
+$descr = 'Zero Sized';
+$filename = Rrd::name($device['hostname'], ['app', 'suricata_extract', $app->app_id]);
+
+if (! Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';
--- a/includes/html/pages/apps.inc.php
+++ b/includes/html/pages/apps.inc.php
@@ -460,6 +460,23 @@ $graphs['wireguard'] = [
 $graphs['linux_config_files'] = [
    'number_of_confs',
 ];
+$graphs['suricata_extract'] = [
+    'errors',
+    'ignored_host',
+    'ignored_ip',
+    'ignored_ip_dest',
+    'ignored_ip_src',
+    'sub',
+    'sub_2xx',
+    'sub_3xx',
+    'sub_4xx',
+    'sub_5xx',
+    'sub_codes',
+    'sub_fail',
+    'truncated',
+    'zero_sized',
+    'sub_size',
+];
 $graphs['linux_softnet_stat'] = [
    'packets',
    'time_squeeze',
--- a/includes/html/pages/device/apps/suricata_extract.inc.php
+++ b/includes/html/pages/device/apps/suricata_extract.inc.php
@@ -0,0 +1,39 @@
+<?php
+
+$graphs = [
+    'suricata_extract_sub' => 'Submission',
+    'suricata_extract_ignored_host'  => 'Ignored By Host',
+    'suricata_extract_ignored_ip'  => 'Ignored By IP',
+    'suricata_extract_ignored_ip_src'  => 'Ignored By IP Source',
+    'suricata_extract_ignored_ip_dest'  => 'Ignored By IP Destination',
+    'suricata_extract_sub_fail'  => 'Submission Failure',
+    'suricata_extract_errors'  => 'Errors',
+    'suricata_extract_truncated'  => 'File Truncated',
+    'suricata_extract_zero_sized'  => 'File Zero Sized',
+    'suricata_extract_sub_size'  => 'Total Size Of Submissions',
+    'suricata_extract_sub_codes'  => 'HTTP Submission Result Codes',
+    'suricata_extract_sub_2xx'  => 'HTTP Submission Result Code, 2xx',
+    'suricata_extract_sub_3xx'  => 'HTTP Submission Result Code, 3xx',
+    'suricata_extract_sub_4xx'  => 'HTTP Submission Result Code, 4xx',
+    'suricata_extract_sub_5xx'  => 'HTTP Submission Result Code, 5xx',
+];
+
+foreach ($graphs as $key => $text) {
+    $graph_type = $key;
+    $graph_array['height'] = '100';
+    $graph_array['width'] = '215';
+    $graph_array['to'] = \LibreNMS\Config::get('time.now');
+    $graph_array['id'] = $app['app_id'];
+    $graph_array['type'] = 'application_' . $key;
+
+    echo '<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">' . $text . '</h3>
+    </div>
+    <div class="panel-body">
+    <div class="row">';
+    include 'includes/html/print-graphrow.inc.php';
+    echo '</div>';
+    echo '</div>';
+    echo '</div>';
+}
--- a/includes/polling/applications/suricata_extract.inc.php
+++ b/includes/polling/applications/suricata_extract.inc.php
@@ -0,0 +1,57 @@
+<?php
+
+use LibreNMS\Exceptions\JsonAppException;
+use LibreNMS\RRD\RrdDefinition;
+
+$name = 'suricata_extract';
+
+try {
+    $data = json_app_get($device, $name)['data'];
+} catch (JsonAppException $e) {
+    echo PHP_EOL . $name . ':' . $e->getCode() . ':' . $e->getMessage() . PHP_EOL;
+    update_application($app, $e->getCode() . ':' . $e->getMessage(), []); // Set empty metrics and error message
+
+    return;
+}
+
+$rrd_name = ['app', $name, $app->app_id];
+$rrd_def = RrdDefinition::make()
+    ->addDataset('errors', 'GAUGE', 0)
+    ->addDataset('ignored_host', 'GAUGE', 0)
+    ->addDataset('ignored_ip', 'GAUGE', 0)
+    ->addDataset('ignored_ip_dest', 'GAUGE', 0)
+    ->addDataset('ignored_ip_src', 'GAUGE', 0)
+    ->addDataset('sub', 'GAUGE', 0)
+    ->addDataset('sub_2xx', 'GAUGE', 0)
+    ->addDataset('sub_3xx', 'GAUGE', 0)
+    ->addDataset('sub_4xx', 'GAUGE', 0)
+    ->addDataset('sub_5xx', 'GAUGE', 0)
+    ->addDataset('sub_fail', 'GAUGE', 0)
+    ->addDataset('truncated', 'GAUGE', 0)
+    ->addDataset('zero_sized', 'GAUGE', 0)
+    ->addDataset('sub_size', 'GAUGE', 0);
+
+$fields = [
+    'errors' => $data['errors_delta'],
+    'ignored_host' => $data['ignored_host_delta'],
+    'ignored_ip' => $data['ignored_ip_delta'],
+    'ignored_ip_dest' => $data['ignored_ip_dest_delta'],
+    'ignored_ip_src' => $data['ignored_ip_src_delta'],
+    'sub' => $data['sub_delta'],
+    'sub_2xx' => $data['sub_2xx_delta'],
+    'sub_3xx' => $data['sub_3xx_delta'],
+    'sub_4xx' => $data['sub_4xx_delta'],
+    'sub_5xx' => $data['sub_5xx_delta'],
+    'sub_fail' => $data['sub_fail_delta'],
+    'truncated' => $data['truncated_delta'],
+    'zero_sized' => $data['zero_sized_delta'],
+    'sub_size' => $data['sub_size_delta'],
+];
+
+if (isset($data['last_errors']) && isset($data['last_errors'][0])) {
+    log_event('suricata_extract_submit errors found: ' . json_encode($data['last_errors']), $device, 'application', 5);
+}
+
+$tags = compact('name', 'app_id', 'rrd_name', 'rrd_def');
+data_update($device, 'app', $tags, $fields);
+update_application($app, 'OK', $fields);
--- a/misc/alert_rules.json
+++ b/misc/alert_rules.json
@@ -727,5 +727,25 @@
    "rule": "applications.app_type = \"linux_config_files\" && application_metrics.metric = \"number_of_confs\" && application_metrics.value > \"0\"",
    "name": "linux_config_files Configuration Files Out-Of-Sync > 0",
    "severity": "warning"
+  },
+  {
+    "rule": "applications.app_type = \"suricata_extract\" && application_metrics.metric = \"errors\" && application_metrics.value > \"0\"",
+    "name": "Suricata Extract Submit errors found > 0",
+    "severity": "warning"
+  },
+  {
+    "rule": "(applications.app_type = \"suricata_extract\" && application_metrics.metric = \"sub_fail\" && application_metrics.value > \"0\") || (applications.app_type = \"suricata_extract\" && application_metrics.metric = \"sub_4xx\" && application_metrics.value > \"0\") || (applications.app_type = \"suricata_extract\" && application_metrics.metric = \"sub_5xx\" && application_metrics.value > \"0\")",
+    "name": "Suricata Extract Submit submission failures found > 0",
+    "severity": "warning"
+  },
+  {
+    "rule": "applications.app_type = \"suricata_extract\" && application_metrics.metric = \"truncated\" && application_metrics.value > \"0\"",
+    "name": "Suricata Extract Submit truncated extracts found > 0",
+    "severity": "warning"
+  },
+  {
+    "rule": "applications.app_type = \"suricata_extract\" && application_metrics.metric = \"zero_sized\" && application_metrics.value > \"0\"",
+    "name": "Suricata Extract Submit zero sized files > 0",
+    "severity": "warning"
  }
 ]
--- a/tests/data/linux_suricata_extract-v1.json
+++ b/tests/data/linux_suricata_extract-v1.json
@@ -0,0 +1,116 @@
+{
+    "applications": {
+        "discovery": {
+            "applications": [
+                {
+                    "app_type": "suricata_extract",
+                    "app_state": "UNKNOWN",
+                    "discovered": 1,
+                    "app_state_prev": null,
+                    "app_status": "",
+                    "app_instance": "",
+                    "data": null
+                }
+            ]
+        },
+        "poller": {
+            "applications": [
+                {
+                    "app_type": "suricata_extract",
+                    "app_state": "OK",
+                    "discovered": 1,
+                    "app_state_prev": "UNKNOWN",
+                    "app_status": "",
+                    "app_instance": "",
+                    "data": null
+                }
+            ],
+            "application_metrics": [
+                {
+                    "metric": "errors",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "ignored_host",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "ignored_ip",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "ignored_ip_dest",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "ignored_ip_src",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "sub",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "sub_2xx",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "sub_3xx",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "sub_4xx",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "sub_5xx",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "sub_fail",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "sub_size",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "truncated",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                },
+                {
+                    "metric": "zero_sized",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "suricata_extract"
+                }
+            ]
+        }
+    }
+}
--- a/tests/snmpsim/linux_suricata_extract-v1.snmprec
+++ b/tests/snmpsim/linux_suricata_extract-v1.snmprec
@@ -0,0 +1,10 @@
+1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64
+1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10
+1.3.6.1.2.1.1.3.0|67|77550514
+1.3.6.1.2.1.1.4.0|4|<private>
+1.3.6.1.2.1.1.5.0|4|<private>
+1.3.6.1.2.1.1.6.0|4|<private>
+1.3.6.1.2.1.25.1.1.0|67|77552962
+1.3.6.1.4.1.8072.1.3.2.2.1.21.6.100.105.115.116.114.111|2|1
+1.3.6.1.4.1.8072.1.3.2.2.1.21.16.115.117.114.105.99.97.116.97.95.101.120.116.114.97.99.116|2|1
+1.3.6.1.4.1.8072.1.3.2.3.1.2.16.115.117.114.105.99.97.116.97.95.101.120.116.114.97.99.116|4x|7b0a20202264617461223a207b0a20202020226572726f7273223a20302c0a20202020226572726f72735f64656c7461223a20302c0a202020202269676e6f7265645f686f7374223a20302c0a202020202269676e6f7265645f686f73745f64656c7461223a20302c0a202020202269676e6f7265645f6970223a20302c0a202020202269676e6f7265645f69705f64656c7461223a20302c0a202020202269676e6f7265645f69705f64657374223a20302c0a202020202269676e6f7265645f69705f646573745f64656c7461223a20302c0a202020202269676e6f7265645f69705f737263223a20302c0a202020202269676e6f7265645f69705f7372635f64656c7461223a20302c0a20202020226c6173745f6572726f7273223a205b5d2c0a2020202022737562223a2035392c0a20202020227375625f327878223a2035392c0a20202020227375625f3278785f64656c7461223a20302c0a20202020227375625f337878223a20302c0a20202020227375625f3378785f64656c7461223a20302c0a20202020227375625f347878223a20302c0a20202020227375625f3478785f64656c7461223a20302c0a20202020227375625f357878223a20302c0a20202020227375625f3578785f64656c7461223a20302c0a20202020227375625f64656c7461223a20302c0a20202020227375625f6661696c223a20302c0a20202020227375625f6661696c5f64656c7461223a20302c0a20202020227375625f73697a65223a2031313533353338342c0a20202020227375625f73697a655f64656c7461223a20302c0a202020202274696d657374616d70223a20313638363830353530312c0a20202020227472756e6361746564223a20302c0a20202020227472756e63617465645f64656c7461223a20302c0a20202020227a65726f5f64656c7461223a20302c0a20202020227a65726f5f73697a6564223a203132362c0a20202020227a65726f5f73697a65645f64656c7461223a20300a20207d2c0a2020226572726f72223a20302c0a2020226572726f72537472696e67223a2022222c0a20202276657273696f6e223a20310a7d0a