From a166df006ad5d56c93a32604071c1b1f80483774 Mon Sep 17 00:00:00 2001 From: "Zane C. Bowers-Hadley" Date: Fri, 21 Oct 2022 10:05:49 -0500 Subject: [PATCH] base64 gzip compression support for json_app_get (#14169) * add lnms_return_optimizer * add compression test using zfs-v1 * minor style fix * save the original output if not json * replace gzinflate with gzdecode as apparently that does not require yanking the header * Minor comment cleanup. Also note it in the application notes as well. * update docs on how it is called * update the spelling of it in a few places * and a few more * dev docs updated a bit * the suricata extend has native support for this now * add exception handling for base64 and gzip decoding failure * minor cleanup for new exceptions * minor misc changes * minor formatting fix * more phpdoc tweaks * minor formatting tweak * remember to actually include the new exceptions * more phpdoc tweaking * correct name in JsonAppGzipDecodeException * add debug and verbose output * style fix * not base64 is it starts with a line with only a integer --- .../JsonAppBase64DecodeException.php | 33 ++ .../Exceptions/JsonAppGzipDecodeException.php | 33 ++ doc/Developing/Application-Notes.md | 9 + doc/Extensions/Applications.md | 84 ++++ includes/polling/functions.inc.php | 53 ++- tests/data/linux_zfs-v1-compressed.json | 413 ++++++++++++++++++ tests/snmpsim/linux_zfs-v1-compressed.snmprec | 10 + 7 files changed, 626 insertions(+), 9 deletions(-) create mode 100644 LibreNMS/Exceptions/JsonAppBase64DecodeException.php create mode 100644 LibreNMS/Exceptions/JsonAppGzipDecodeException.php create mode 100644 tests/data/linux_zfs-v1-compressed.json create mode 100644 tests/snmpsim/linux_zfs-v1-compressed.snmprec diff --git a/LibreNMS/Exceptions/JsonAppBase64DecodeException.php b/LibreNMS/Exceptions/JsonAppBase64DecodeException.php new file mode 100644 index 0000000000..0ce690904f --- /dev/null +++ b/LibreNMS/Exceptions/JsonAppBase64DecodeException.php @@ -0,0 +1,33 @@ +output = $output; + } + + /** + * @return string + */ + public function getOutput() + { + return $this->output; + } +} diff --git a/LibreNMS/Exceptions/JsonAppGzipDecodeException.php b/LibreNMS/Exceptions/JsonAppGzipDecodeException.php new file mode 100644 index 0000000000..7d74acdb1b --- /dev/null +++ b/LibreNMS/Exceptions/JsonAppGzipDecodeException.php @@ -0,0 +1,33 @@ +output = $output; + } + + /** + * @return string + */ + public function getOutput() + { + return $this->output; + } +} diff --git a/doc/Developing/Application-Notes.md b/doc/Developing/Application-Notes.md index 1c9843f01c..e67e82a84c 100644 --- a/doc/Developing/Application-Notes.md +++ b/doc/Developing/Application-Notes.md @@ -56,6 +56,15 @@ try { } ``` +### Compression + +Also worth noting that `json_app_get` supports compressed data via +base64 encoded gzip. If base64 encoding is detected on the the SNMP +return, it will be gunzipped and then parsed. + +`https://github.com/librenms/librenms-agent/blob/master/utils/librenms_return_optimizer` +may be used to optimize JSON returns. + ## Application Data Storage The `$app` model is supplied for each application poller and graph. diff --git a/doc/Extensions/Applications.md b/doc/Extensions/Applications.md index d380ada253..ec48ac2262 100644 --- a/doc/Extensions/Applications.md +++ b/doc/Extensions/Applications.md @@ -51,6 +51,84 @@ like that for proxmox: extend proxmox /usr/bin/sudo /usr/local/bin/proxmox ``` +### JSON Return Optimization Using librenms_return_optimizer + +While the json_app_get does allow for more complex and larger data +to be easily returned by a extend and the data to then be worked +with, this can also sometimes result in large returns that +occasionally don't play nice with SNMP on some networks. + +`librenms_return_optimizer` fixes this via taking the extend output +piped to it, gzipping it, and then converting it to base64. The +later is needed as net-snmp does not play that nice with binary data, +converting most of the non-printable characters to `.`. This does add +a bit of additional overhead to the gzipped data, but still tends to +be result in a return that is usually a third of the size for JSONs +items. + +The change required is fairly simply. So for the portactivity example below... + +``` +extend portactivity /etc/snmp/extends/portactivity smtps,http,imap,imaps,postgresql,https,ldap,ldaps,nfsd,syslog-conn,ssh,matrix,gitea +``` + +Would become this... + +``` +extend portactivity /usr/local/bin/lnms_return_optimizer -- /etc/snmp/extends/portactivity smtps,http,imap,imaps,postgresql,https,ldap,ldaps,nfsd,syslog-conn,ssh,matrix,gitea +``` + +The requirements for this are Perl, MIME::Base64, and Gzip::Faster. + +Installing on FreeBSD... + +``` +pkg install p5-MIME-Base64 p5-Gzip-Faster wget +wget https://raw.githubusercontent.com/librenms/librenms-agent/master/utils/librenms_return_optimizer -O /usr/local/bin/librenms_return_optimizer +chmod +x /usr/local/bin/librenms_return_optimizer +``` + +Installing on Debian... + +``` +apt-get install zlib1g-dev cpanminus wget +cpanm Gzip::Faster +cpanm MIME::Base64 +wget https://raw.githubusercontent.com/librenms/librenms-agent/master/utils/librenms_return_optimizer -O /usr/local/bin/librenms_return_optimizer +chmod +x /usr/local/bin/librenms_return_optimizer +``` + +Currently supported applications as are below. + +- backupninja +- certificate +- chronyd +- dhcp-stats +- docker +- fail2ban +- fbsd-nfs-client +- fbsd-nfs-server +- gpsd +- mailcow-postfix +- mdadm +- ntp-client +- ntp-server +- portactivity +- powerdns +- powermon +- puppet-agent +- pureftpd +- redis +- seafile +- supervisord +- ups-apcups +- zfs + +The following apps have extends that have native support for this, +if congiured to do so. + +- suricata + ## Enable the application discovery module 1. Edit the device for which you want to add this support @@ -2361,6 +2439,12 @@ cpanm Suricata::Monitoring extend suricata-stats /usr/bin/env PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin suricata_stat_check -c ``` +Or if you want to use try compressing the return via Base64+GZIP... + +``` +extend suricata-stats /usr/bin/env PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin suricata_stat_check -c -b +``` + 4. Restart snmpd on your system. You will want to make sure Suricata is set to output the stats diff --git a/includes/polling/functions.inc.php b/includes/polling/functions.inc.php index 6467b484fb..f2be9278fd 100644 --- a/includes/polling/functions.inc.php +++ b/includes/polling/functions.inc.php @@ -4,13 +4,16 @@ use App\Models\DeviceGraph; use Illuminate\Support\Str; use LibreNMS\Config; use LibreNMS\Enum\Alert; +use LibreNMS\Exceptions\JsonAppBase64DecodeException; use LibreNMS\Exceptions\JsonAppBlankJsonException; use LibreNMS\Exceptions\JsonAppExtendErroredException; +use LibreNMS\Exceptions\JsonAppGzipDecodeException; use LibreNMS\Exceptions\JsonAppMissingKeysException; use LibreNMS\Exceptions\JsonAppParsingFailedException; use LibreNMS\Exceptions\JsonAppPollingFailedException; use LibreNMS\Exceptions\JsonAppWrongVersionException; use LibreNMS\RRD\RrdDefinition; +use LibreNMS\Util\Debug; function bulk_sensor_snmpget($device, $sensors) { @@ -586,6 +589,10 @@ function update_application($app, $response, $metrics = [], $status = '') /** * This is to make it easier polling apps. Also to help standardize around JSON. * + * If the data has is in base64, it will be converted and then gunzipped. + * https://github.com/librenms/librenms-agent/blob/master/utils/lnms_return_optimizer + * May be used to convert output from extends to that via piping it through it. + * * The required keys for the returned JSON are as below. * version - The version of the snmp extend script. Should be numeric and at least 1. * error - Error code from the snmp extend script. Should be > 0 (0 will be ignored and negatives are reserved) @@ -601,16 +608,20 @@ function update_application($app, $response, $metrics = [], $status = '') * -4 : Empty JSON parsed, meaning blank JSON was returned. * -5 : Valid json, but missing required keys * -6 : Returned version is less than the min version. + * -7 : Base64 decode failure. + * -8 : Gzip decode failure. * * Error checking may also be done via checking the exceptions listed below. - * JsonAppPollingFailedException, -2 : Empty return from SNMP. - * JsonAppParsingFailedException, -3 : Could not parse the JSON. - * JsonAppBlankJsonException, -4 : Blank JSON. - * JsonAppMissingKeysException, -5 : Missing required keys. - * JsonAppWrongVersionException , -6 : Older version than supported. - * JsonAppExtendErroredException : Polling and parsing was good, but the returned data has an error set. - * This may be checked via $e->getParsedJson() and then checking the - * keys error and errorString. + * JsonAppPollingFailedException, -2 : Empty return from SNMP. + * JsonAppParsingFailedException, -3 : Could not parse the JSON. + * JsonAppBlankJsonException, -4 : Blank JSON. + * JsonAppMissingKeysException, -5 : Missing required keys. + * JsonAppWrongVersionException , -6 : Older version than supported. + * JsonAppExtendErroredException : Polling and parsing was good, but the returned data has an error set. + * This may be checked via $e->getParsedJson() and then checking the + * keys error and errorString. + * JsonAppPollingBase64DecodeException , -7 : Base64 decoding failed. + * JsonAppPollingGzipDecodeException , -8 : Gzip decoding failed. * The error value can be accessed via $e->getCode() * The output can be accessed via $->getOutput() Only returned for code -3 or lower. * The parsed JSON can be access via $e->getParsedJson() @@ -635,17 +646,41 @@ function json_app_get($device, $extend, $min_version = 1) { $output = snmp_get($device, 'nsExtendOutputFull.' . string_to_oid($extend), '-Oqv', 'NET-SNMP-EXTEND-MIB'); + // save for returning if not JSON + $orig_output = $output; + // make sure we actually get something back if (empty($output)) { throw new JsonAppPollingFailedException('Empty return from snmp_get.', -2); } + // checks for base64 decoding and converts it to non-base64 so it can gunzip + if (preg_match('/^[A-Za-z0-9\/\+\n]+\=*\n*$/', $output) && ! preg_match('/^[0-9]+\n/', $output)) { + $output = base64_decode($output); + if (! $output) { + if (Debug::isEnabled()) { + echo "Decoding Base64 Failed...\n\n"; + } + throw new JsonAppBase64DecodeException('Base64 decode failed.', $orig_output, -7); + } + $output = gzdecode($output); + if (! $output) { + if (Debug::isEnabled()) { + echo "Decoding GZip failed...\n\n"; + } + throw new JsonAppGzipDecodeException('Gzip decode failed.', $orig_output, -8); + } + if (Debug::isVerbose()) { + echo 'Decoded Base64+GZip Output: ' . $output . "\n\n"; + } + } + // turn the JSON into a array $parsed_json = json_decode(stripslashes($output), true); // improper JSON or something else was returned. Populate the variable with an error. if (json_last_error() !== JSON_ERROR_NONE) { - throw new JsonAppParsingFailedException('Invalid JSON', $output, -3); + throw new JsonAppParsingFailedException('Invalid JSON', $orig_output, -3); } // There no keys in the array, meaning '{}' was was returned diff --git a/tests/data/linux_zfs-v1-compressed.json b/tests/data/linux_zfs-v1-compressed.json new file mode 100644 index 0000000000..f6a8b78ac3 --- /dev/null +++ b/tests/data/linux_zfs-v1-compressed.json @@ -0,0 +1,413 @@ +{ + "applications": { + "discovery": { + "applications": [ + { + "app_type": "zfs", + "app_state": "UNKNOWN", + "discovered": 1, + "app_state_prev": null, + "app_status": "", + "app_instance": "", + "data": null + } + ] + }, + "poller": { + "applications": [ + { + "app_type": "zfs", + "app_state": "OK", + "discovered": 1, + "app_state_prev": "UNKNOWN", + "app_status": "", + "app_instance": "", + "data": "{\"pools\":[\"arc\"]}" + } + ], + "application_metrics": [ + { + "metric": "actual_hit_per", + "value": 91.264716658306, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "anon_hits", + "value": 1735151, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "anon_hits_per", + "value": 0.87447363662197, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_accesses_total", + "value": 213732964, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_hits", + "value": 198422334, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_misses", + "value": 15310630, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_size", + "value": 4811379336, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_size_per", + "value": 14.893344946427, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "cache_hits_per", + "value": 92.836561233484, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "cache_miss_per", + "value": 7.1634387665162, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "data_demand_per", + "value": 98.185847580587, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "data_pre_per", + "value": 76.243415839277, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "deleted", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_data_hits", + "value": 1295901, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_data_misses", + "value": 23944, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_data_total", + "value": 1319845, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_hits_per", + "value": 0.65310238715366, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_meta_hits", + "value": 189163699, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_meta_misses", + "value": 14348845, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_misses_per", + "value": 0.15638807808692, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_skip", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "freq_used_per", + "value": 30.42081926587, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "meta_hits_per", + "value": 95.333874562729, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "meta_misses_per", + "value": 93.718187951769, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_ghost_hits", + "value": 1032016, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_ghost_per", + "value": 0.5201108056717, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_hits", + "value": 159197014, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_per", + "value": 80.231398749699, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_size", + "value": 1463661012, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "min_size_per", + "value": 12.5, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_ghost_hits", + "value": 592383, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_ghost_per", + "value": 0.29854653357721, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_hits", + "value": 35865770, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_per", + "value": 18.07547027443, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mutex_skip", + "value": 113, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "p", + "value": 3347718324, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_alloc", + "value": 12194087313408, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_cap", + "value": 45, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_dedup", + "value": 76, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_expandsz", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_frag", + "value": -1, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_free", + "value": 3748831289344, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_size", + "value": 15942918602752, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_data_hits", + "value": 32713, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_data_misses", + "value": 10193, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_data_total", + "value": 42906, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_hits_per", + "value": 0.016486551357671, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_hits", + "value": 7930021, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_hits_per", + "value": 3.9965364987593, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_misses", + "value": 927648, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_misses_per", + "value": 6.0588493092707, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_misses_per", + "value": 0.066574660872871, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "real_hits", + "value": 195062784, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "rec_used_per", + "value": 69.579180734129, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "recycle_miss", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size", + "value": 5184353784, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_arat", + "value": 0.16047865661247, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_max", + "value": 32305565696, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_min", + "value": 4038195712, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_per", + "value": 16.047865661247, + "value_prev": null, + "app_type": "zfs" + } + ] + } + }, + "os": { + "discovery": { + "devices": [ + { + "sysName": "", + "sysObjectID": ".1.3.6.1.4.1.8072.3.2.10", + "sysDescr": "Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64", + "sysContact": "", + "version": "3.10.0-693.5.2.el7.x86_64", + "hardware": "Generic x86 64-bit", + "features": null, + "os": "linux", + "type": "server", + "serial": null, + "icon": "linux.svg", + "location": "" + } + ] + }, + "poller": "matches discovery" + } +} diff --git a/tests/snmpsim/linux_zfs-v1-compressed.snmprec b/tests/snmpsim/linux_zfs-v1-compressed.snmprec new file mode 100644 index 0000000000..190599edf9 --- /dev/null +++ b/tests/snmpsim/linux_zfs-v1-compressed.snmprec @@ -0,0 +1,10 @@ +1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64 +1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10 +1.3.6.1.2.1.1.3.0|67|77550514 +1.3.6.1.2.1.1.4.0|4| +1.3.6.1.2.1.1.5.0|4| +1.3.6.1.2.1.1.6.0|4| +1.3.6.1.2.1.25.1.1.0|67|77552962 +1.3.6.1.4.1.8072.1.3.2.2.1.21.6.100.105.115.116.114.111|2|1 +1.3.6.1.4.1.8072.1.3.2.2.1.21.3.122.102.115|2|1 +1.3.6.1.4.1.8072.1.3.2.3.1.2.3.122.102.115|4x|483473494141414141414141413356573232376a4f41783937316345667534596f6b694a31507a47504334576765466f576d4e794738635a644b666f767939744a35626b74486d4a4c5a3151354f4868556436664e70744e4666762b315038592b753734556d322b623672716556722b452f744c647a714f537a4376374a716847562f66783764786f592f4e66767661445a634a464a7a786c6f57653739744430372f45595876702f7362746f586d6267714e46343578335076687141545a394f36504f735a3943555330426b5369514a387630414a78436b5141674b79364c394c4f507637665853397a645136477079527142594c30544e6d3542746b33374771666b373942676130484e444b77654c53674c3944786e6a73514d677062536362743461493637496f717076554d77466f58426a654557384f486e645347724168636773414571575468306c307538497a534d52354d4135394e70502b3339633176527a3374366e474c73393664322f726d46514559594166567243584c44485a7644544b49657574364c6232637436764a33326a66723362615a3653433333746e463358586559372f653037374d3579475443494c562f684b74555574726c523279416351627938342b426d746d7058367230736248376648663536544f747043434437566a44576f5943577849556d6a6134546f4c655a4543314e5954673166526f50464a67496672454e2b326c312f64584b674b4d4f76663858526353554759694e476a3939714e544d656a4646356554356368515a30314145614d38777873456e516375753235583261446657314a43394445676d5766354456694a764277477072394346554b6a532f6b31625274484157574d465a48434733495372787075677746434548497255477a58464d523444794b45697847664c43664f384639786e3174694555485241644f71553763644d6353614f74303769467156755770415773645378414f4474686e6263304f6e5a726c51416764716b4e5642576c547a4c787657496567512b7770434473316a6b654b387a4531454c4236684379546a705a7a6c657a6950673578747871756c45685a6e4a4c6b5243696730575962646f5743377055426a5172545247775a627455636f32706d386c354e77616f56386c646e54366d4e32694a35634c71797448474b676f4650526d424b6939474267794c6c577a4a69617176477050785338434859423133463468674a6f4257475548307951416c6c5545636f7577734f66656133364652717a426e6a78653355485764544d36685868574f775830686b746a64746837465a32656f302f3758376d66474a362b4c487053566f697153704f4544484f757059447671742f72753070515a7851757a55467953626b5a796d58497471444b7157354d7933573236454c5035524b35656b74354f71415479346771373779456d7457694e57397955316b56423453434a634863474f642b586e4b736e797368677973792b59622f706d75486d482f38495031754d5a5849326f425a4454767873324a4a654a66377032534f5a7353696d732f4e594763615254726d31676d375568515a644331514e51634c707250703754583659707a745048302f2b722f65565a52676b4141413d3d