PHP-FPM app update to handle multiple pools (#16122)

* checkpoint work for now

* polling should be good now

* re-work php-fpm some more

* re-work the php-fpm app page

* add php-fpm_v1_combined.inc.php

* add more graphs

* more work

* add last request cpu

* fix stats for pools

* more graph work

* update the docs for php-fpm

* more graph work

* more graph stuff

* graph stuff now done

* some style fixes

* another style fix

* remove a unneeded use line

* add new tests

* add some missing items to the test

* json fix

* remove start time and add data

doc/Extensions/Applications.md

@@ -1905,26 +1905,41 @@ using it as a agent.
 
 1. Copy the shell script, phpfpmsp, to the desired host
 ```
-wget https://github.com/librenms/librenms-agent/raw/master/snmp/phpfpmsp -O /etc/snmp/phpfpmsp
+wget https://github.com/librenms/librenms-agent/raw/master/snmp/php-fpm -O /etc/snmp/php-fpm
 ```
 
 2. Make the script executable
 ```
-chmod +x /etc/snmp/phpfpmsp
+chmod +x /etc/snmp/php-fpm
+```
+
+3. Install the depends.
+```shell
+# FreeBSD
+pkg install p5-File-Slurp p5-JSON p5-String-ShellQuote p5-MIME-Base64
+# Debian
+apt-get install libfile-slurp-perl libjson-perl libstring-shellquote-perl libmime-base64-perl
 ```
 
 3. Edit your snmpd.conf file (usually /etc/snmp/snmpd.conf) and add:
 ```
-extend phpfpmsp /etc/snmp/phpfpmsp
+extend phpfpmsp /etc/snmp/php-fpm
 ```
 
-4. Edit /etc/snmp/phpfpmsp to include the status URL for the PHP-FPM
+5. Create the config file
-   pool you are monitoring.
+   `/usr/local/etc/php-fpm_extend.json`. Alternate locations may be
+   specified using the the `-f` switch. Akin to like below. For more
+   information, see `/etc/snmp/php-fpm --help`.
+```json
+{
+"pools":{
+         "thefrog": "https://thefrog/fpm-status",
+         "foobar": "https://foo.bar/fpm-status"
+    }
+}
+```
 
-5. Restart snmpd on your host
+6. Restart snmpd on the host
-
-It is worth noting that this only monitors a single pool. If you want
-to monitor multiple pools, this won't do it.
 
 The application should be auto-discovered as described at the top of
 the page. If it is not, please follow the steps set out under `SNMP

includes/html/graphs/application/php-fpm-include.php

@@ -0,0 +1,10 @@
+<?php
+
+$name = 'php-fpm';
+$ds = 'data';
+
+if (isset($vars['phpfpm_pool'])) {
+    $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'pools___' . $vars['phpfpm_pool'] . '___' . $stat]);
+} else {
+    $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'totals___' . $stat]);
+}

includes/html/graphs/application/php-fpm_overview_combined.inc.php

@@ -0,0 +1,87 @@
+<?php
+
+$name = 'php-fpm';
+$unit_text = 'Totals';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$descr_len = 20;
+$addarea = 1;
+$transparency = 15;
+
+$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'totals___max_active_processes']);
+
+if (Rrd::checkRrdExists($filename)) {
+    $rrd_list = [];
+
+    $proc_stats = [
+        'max_active_processes' => 'Max Active Processes',
+        'active_processes' => 'Active Processes',
+        'idle_processes' => 'Idle Processes',
+        'max_listen_queue' => 'Max Listen Queue',
+        'listen_queue' => 'Listen Queue',
+        'listen_queue_len' => 'Listen Queue Len',
+        'listen_queue' => 'Listen Queue',
+    ];
+
+    foreach ($proc_stats as $stat => $descr) {
+        $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'totals___' . $stat]);
+        if (Rrd::checkRrdExists($filename)) {
+            $rrd_list[] = [
+                'filename' => $filename,
+                'descr' => $descr,
+                'ds' => 'data',
+            ];
+        }
+    }
+} else {
+    $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+    if (Rrd::checkRrdExists($filename)) {
+        $rrd_list = [
+            [
+                'filename' => $filename,
+                'descr' => 'Listen Queue',
+                'ds' => 'lq',
+            ],
+            [
+                'filename' => $filename,
+                'descr' => 'Max Listen Queue',
+                'ds' => 'mlq',
+            ],
+            [
+                'filename' => $filename,
+                'descr' => 'Idle Procs',
+                'ds' => 'ip',
+            ],
+            [
+                'filename' => $filename,
+                'descr' => 'Active Procs',
+                'ds' => 'ap',
+            ],
+            [
+                'filename' => $filename,
+                'descr' => 'Total Procs',
+                'ds' => 'tp',
+            ],
+            [
+                'filename' => $filename,
+                'descr' => 'Max Active Procs',
+                'ds' => 'map',
+            ],
+            [
+                'filename' => $filename,
+                'descr' => 'Max Children Reached',
+                'ds' => 'mcr',
+            ],
+            [
+                'filename' => $filename,
+                'descr' => 'Slow Reqs',
+                'ds' => 'sr',
+            ],
+        ];
+    } else {
+        echo 'file missing: ' . $filename;
+    }
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/php-fpm_overview_max_children_reached.inc.php

@@ -0,0 +1,19 @@
+<?php
+
+$name = 'php-fpm';
+
+$descr = 'Max Chldrn Reached';
+$unit_text = 'Per Second';
+$ds = 'data';
+
+$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'totals___max_children_reached']);
+
+if (! Rrd::checkRrdExists($filename)) {
+    $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+    $ds = 'mcr';
+    if (! Rrd::checkRrdExists($filename)) {
+        echo 'file missing: ' . $filename;
+    }
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_overview_slow_requests.inc.php

@@ -0,0 +1,20 @@
+<?php
+
+$name = 'php-fpm';
+
+$descr = 'Slow Requests';
+$stat = 'slow_requests';
+$unit_text = 'Requests/S';
+$ds = 'data';
+
+$filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'totals___slow_requests']);
+
+if (! Rrd::checkRrdExists($filename)) {
+    $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id]);
+    $ds = 'sr';
+    if (! Rrd::checkRrdExists($filename)) {
+        echo 'file missing: ' . $filename;
+    }
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_accepted_conn.inc.php

@@ -0,0 +1,13 @@
+<?php
+
+$descr = 'Accepted Conns';
+$stat = 'accepted_conn';
+$unit_text = 'Conns/S';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_active_processes.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'Active Procs';
+$stat = 'active_processes';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_combined.inc.php

@@ -0,0 +1,39 @@
+<?php
+
+$name = 'php-fpm';
+$unit_text = 'Totals';
+$colours = 'psychedelic';
+$dostack = 0;
+$printtotal = 0;
+$descr_len = 20;
+$addarea = 1;
+$transparency = 15;
+
+$rrd_list = [];
+
+$proc_stats = [
+    'max_active_processes' => 'Max Active Processes',
+    'active_processes' => 'Active Processes',
+    'idle_processes' => 'Idle Processes',
+    'max_listen_queue' => 'Max Listen Queue',
+    'listen_queue' => 'Listen Queue',
+    'listen_queue_len' => 'Listen Queue Len',
+    'listen_queue' => 'Listen Queue',
+];
+
+foreach ($proc_stats as $stat => $descr) {
+    if (isset($vars['phpfpm_pool'])) {
+        $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'pools___' . $vars['phpfpm_pool'] . '___' . $stat]);
+    } else {
+        $filename = Rrd::name($device['hostname'], ['app', $name, $app->app_id, 'totals___' . $stat]);
+    }
+    if (Rrd::checkRrdExists($filename)) {
+        $rrd_list[] = [
+            'filename' => $filename,
+            'descr' => $descr,
+            'ds' => 'data',
+        ];
+    }
+}
+
+require 'includes/html/graphs/generic_multi_line.inc.php';

includes/html/graphs/application/php-fpm_v1_idle_processes.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'Idle Procs';
+$stat = 'idle_processes';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_last_request_cpu.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'CPU Time %';
+$stat = 'last_request_cpu';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_listen_queue.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'Listen Queue';
+$stat = 'listen_queue';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_listen_queue_len.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'Listen Queue Len';
+$stat = 'listen_queue_len';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_max_active_processes.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'Max Act Procs';
+$stat = 'max_active_processes';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_max_children_reached.inc.php

@@ -0,0 +1,13 @@
+<?php
+
+$descr = 'Max Children Reached';
+$stat = 'max_children_reached';
+$unit_text = 'Per Second';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_max_listen_queue.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'Max Listen Queue';
+$stat = 'max_listen_queue';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_slow_requests.inc.php

@@ -0,0 +1,13 @@
+<?php
+
+$descr = 'Slow Requests';
+$stat = 'slow_requests';
+$unit_text = 'Requests/S';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_start_since.inc.php

@@ -0,0 +1,15 @@
+<?php
+
+$descr = 'Uptime';
+$stat = 'start_since';
+
+$munge = true;
+$unit_text = 'days';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/graphs/application/php-fpm_v1_total_processes.inc.php

@@ -0,0 +1,12 @@
+<?php
+
+$descr = 'Total Procs';
+$stat = 'total_processes';
+
+require 'php-fpm-include.php';
+
+if (Rrd::checkRrdExists($filename)) {
+    d_echo('RRD "' . $filename . '" not found');
+}
+
+require 'includes/html/graphs/generic_stats.inc.php';

includes/html/pages/apps.inc.php

@@ -184,7 +184,10 @@ $graphs['exim-stats'] = [
     'queue',
 ];
 $graphs['php-fpm'] = [
-    'stats',
+    'overview_combined',
+    'overview_slow_requests',
+    'overview_max_childen_reached',
+    'v1_last_request_cpu',
 ];
 $graphs['nvidia'] = [
     'sm',

includes/html/pages/device/apps/php-fpm.inc.php

@@ -1,9 +1,62 @@
 <?php
 
-$graphs = [
+$app_data = $app->data;
-    'php-fpm_stats' => 'PHP-FPM',
+
+$link_array = [
+    'page' => 'device',
+    'device' => $device['device_id'],
+    'tab' => 'apps',
+    'app' => 'php-fpm',
 ];
 
+if ($app_data['version'] == 'legacy') {
+    $graphs = [
+        'php-fpm_stats' => 'PHP-FPM',
+    ];
+} else {
+    print_optionbar_start();
+
+    // print the link to the totals
+    $total_label = isset($vars['phpfpm_pool'])
+        ? 'Totals'
+        : '<span class="pagemenu-selected">Totals</span>';
+    echo generate_link($total_label, $link_array);
+
+    // print links to the pools
+    echo ' | Pools: ';
+    $pools = $app->data['pools'] ?? [];
+    sort($pools);
+    foreach ($pools as $index => $pool_name) {
+        $label = $vars['phpfpm_pool'] == $pool_name
+            ? '<span class="pagemenu-selected">' . $pool_name . '</span>'
+            : $pool_name;
+
+        echo generate_link($label, $link_array, ['phpfpm_pool' => $pool_name]);
+
+        if ($index < (count($pools) - 1)) {
+            echo ', ';
+        }
+    }
+
+    print_optionbar_end();
+
+    $graphs = [
+        'php-fpm_v1_combined' => 'Combined',
+        'php-fpm_v1_accepted_conn' => 'Connections Per Second',
+        'php-fpm_v1_last_request_cpu' => 'Last Request CPU',
+        'php-fpm_v1_slow_requests' => 'Slow Requests',
+        'php-fpm_v1_active_processes' => 'Active Procs',
+        'php-fpm_v1_idle_processes' => 'Idle Procs',
+        'php-fpm_v1_total_processes' => 'Total Procs',
+        'php-fpm_v1_max_active_processes' => 'Max Active Procs',
+        'php-fpm_v1_listen_queue' => 'Listen Queue',
+        'php-fpm_v1_max_listen_queue' => 'Max Listen Queue',
+        'php-fpm_v1_listen_queue_len' => 'Listen Queue Len',
+        'php-fpm_v1_max_children_reached' => 'Max Children Reached',
+        'php-fpm_v1_start_since' => 'Uptime',
+    ];
+}
+
 foreach ($graphs as $key => $text) {
     $graph_type = $key;
     $graph_array['height'] = '100';
@@ -12,6 +65,10 @@ foreach ($graphs as $key => $text) {
     $graph_array['id'] = $app['app_id'];
     $graph_array['type'] = 'application_' . $key;
 
+    if (isset($vars['phpfpm_pool'])) {
+        $graph_array['phpfpm_pool'] = $vars['phpfpm_pool'];
+    }
+
     echo '<div class="panel panel-default">
     <div class="panel-heading">
         <h3 class="panel-title">' . $text . '</h3>

includes/polling/applications/php-fpm.inc.php

@@ -1,42 +1,135 @@
 <?php
 
+use LibreNMS\Exceptions\JsonAppParsingFailedException;
 use LibreNMS\RRD\RrdDefinition;
 
 $name = 'php-fpm';
 
-if (! empty($agent_data['app'][$name])) {
+try {
-    $phpfpm = $agent_data['app'][$name];
+    // phpfpmsp is the original name and being kept for compatbility purposes... sp stood for single pool
-} else {
+    $extend_return = json_app_get($device, 'phpfpmsp');
-    $options = '-Oqv';
+} catch (JsonAppParsingFailedException $e) {
-    $oid = '.1.3.6.1.4.1.8072.1.3.2.3.1.2.8.112.104.112.102.112.109.115.112';
+    // Legacy script, build compatible array
-    $phpfpm = snmp_walk($device, $oid, $options);
+    $phpfpm = $e->getOutput();
+
+    [$pool,$start_time,$start_since,$accepted_conn,$listen_queue,$max_listen_queue,$listen_queue_len,$idle_processes,
+        $active_processes,$total_processes,$max_active_processes,$max_children_reached,$slow_requests] = explode("\n", $phpfpm);
+
+    $rrd_name = ['app', $name, $app->app_id];
+    $rrd_def = RrdDefinition::make()
+        ->addDataset('lq', 'GAUGE', 0)
+        ->addDataset('mlq', 'GAUGE', 0)
+        ->addDataset('ip', 'GAUGE', 0)
+        ->addDataset('ap', 'GAUGE', 0)
+        ->addDataset('tp', 'GAUGE', 0)
+        ->addDataset('map', 'GAUGE', 0)
+        ->addDataset('mcr', 'GAUGE', 0)
+        ->addDataset('sr', 'GAUGE', 0);
+
+    $fields = [
+        'lq' => $listen_queue,
+        'mlq' => $max_listen_queue,
+        'ip' => $idle_processes,
+        'ap' => $active_processes,
+        'tp' => $total_processes,
+        'map' => $max_active_processes,
+        'mcr' => $max_children_reached,
+        'sr' => $slow_requests,
+    ];
+
+    $app->data = ['version' => 'legacy'];
+
+    $tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+    data_update($device, 'app', $tags, $fields);
+
+    update_application($app, 'OK', $fields);
+
+    return;
 }
 
-[$pool,$start_time,$start_since,$accepted_conn,$listen_queue,$max_listen_queue,$listen_queue_len,$idle_processes,
+$var_mappings = [
-    $active_processes,$total_processes,$max_active_processes,$max_children_reached,$slow_requests] = explode("\n", $phpfpm);
+    'accepted_conn' => 'accepted conn',
-
+    'active_processes' => 'active processes',
-$rrd_name = ['app', $name, $app->app_id];
+    'idle_processes' => 'idle processes',
-$rrd_def = RrdDefinition::make()
+    'listen_queue' => 'listen queue',
-    ->addDataset('lq', 'GAUGE', 0)
+    'listen_queue_len' => 'listen queue len',
-    ->addDataset('mlq', 'GAUGE', 0)
+    'max_active_processes' => 'max active processes',
-    ->addDataset('ip', 'GAUGE', 0)
+    'max_children_reached' => 'max children reached',
-    ->addDataset('ap', 'GAUGE', 0)
+    'max_listen_queue' => 'max listen queue',
-    ->addDataset('tp', 'GAUGE', 0)
+    'slow_requests' => 'slow requests',
-    ->addDataset('map', 'GAUGE', 0)
+    'start_since' => 'start since',
-    ->addDataset('mcr', 'GAUGE', 0)
+    'total_processes' => 'total processes',
-    ->addDataset('sr', 'GAUGE', 0);
+    'last_request_cpu' => 'last request cpu',
-
-$fields = [
-    'lq' => $listen_queue,
-    'mlq' => $max_listen_queue,
-    'ip' => $idle_processes,
-    'ap' => $active_processes,
-    'tp' => $total_processes,
-    'map' => $max_active_processes,
-    'mcr' => $max_children_reached,
-    'sr' => $slow_requests,
 ];
 
-$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+$new_app_data = [
-data_update($device, 'app', $tags, $fields);
+    'version' => $extend_return['version'],
-update_application($app, $phpfpm, $fields);
+    'pools' => [],
+];
+
+$metrics = [
+    'errored' => $extend_return['data']['errored'],
+];
+
+$old_app_data = $app->data;
+
+$counter_rrd_def = RrdDefinition::make()
+    ->addDataset('data', 'DERIVE', 0);
+
+$gauge_rrd_def = RrdDefinition::make()
+    ->addDataset('data', 'GAUGE', 0);
+
+// process pools
+foreach ($extend_return['data']['pools'] as $pool => $pool_stats) {
+    $new_app_data['pools'][] = $pool;
+    foreach ($var_mappings as $stat => $stat_key) {
+        $rrd_name = ['app', $name, $app->app_id, 'pools___' . $pool . '___' . $stat];
+        $fields = ['data' => $extend_return['data']['pools'][$pool][$stat_key]];
+
+        $metrics['pools___' . $pool . '___' . $stat] = $extend_return['data']['pools'][$pool][$stat_key];
+
+        if ($stat == 'accepted_conn' || $stat == 'slow_requests' || $stat == 'max_children_reached') {
+            $rrd_def = $counter_rrd_def;
+        } else {
+            $rrd_def = $gauge_rrd_def;
+        }
+
+        $tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+        data_update($device, 'app', $tags, $fields);
+    }
+}
+
+// process totals
+foreach ($var_mappings as $stat => $stat_key) {
+    $rrd_name = ['app', $name, $app->app_id, 'totals___' . $stat];
+    $fields = ['data' => $extend_return['data']['totals'][$stat_key]];
+
+    $metrics['totals_' . $stat] = $extend_return['data']['totals'][$stat_key];
+
+    if ($stat == 'accepted_conn' || $stat == 'slow_requests' || $stat == 'max_children_reached') {
+        $rrd_def = $counter_rrd_def;
+    } else {
+        $rrd_def = $gauge_rrd_def;
+    }
+
+    $tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name];
+    data_update($device, 'app', $tags, $fields);
+}
+
+// check for added or removed pools
+$old_pools = $old_app_data['pools'] ?? [];
+$new_pools = $new_app_data['pools'] ?? [];
+$added_pools = array_diff($new_pools, $old_pools);
+$removed_pools = array_diff($old_pools, $new_pools);
+
+// if we have any changes in pools, log it
+if (count($added_pools) > 0 || count($removed_pools) > 0) {
+    $log_message = 'Suricata Instance Change:';
+    $log_message .= count($added_pools) > 0 ? ' Added ' . implode(',', $added_pools) : '';
+    $log_message .= count($removed_pools) > 0 ? ' Removed ' . implode(',', $added_pools) : '';
+    log_event($log_message, $device, 'application');
+}
+
+$app->data = $new_app_data;
+
+update_application($app, 'OK', $metrics);

tests/data/linux_php-fpm-v1.json

@@ -0,0 +1,184 @@
+{
+    "applications": {
+        "discovery": {
+            "applications": [
+                {
+                    "app_type": "php-fpm",
+                    "app_state": "UNKNOWN",
+                    "discovered": 1,
+                    "app_state_prev": null,
+                    "app_status": "",
+                    "app_instance": "",
+                    "data": null,
+                    "deleted_at": null
+                }
+            ]
+        },
+        "poller": {
+            "applications": [
+                {
+                    "app_type": "php-fpm",
+                    "app_state": "OK",
+                    "discovered": 1,
+                    "app_state_prev": "UNKNOWN",
+                    "app_status": "",
+                    "app_instance": "",
+                    "data":  "{\"version\":1,\"pools\":[\"thefrog\"]}",
+                    "deleted_at": null
+                }
+            ],
+            "application_metrics": [
+                {
+                    "metric": "errored",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___accepted_conn",
+                    "value": 6839,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___active_processes",
+                    "value": 1,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___idle_processes",
+                    "value": 4,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___last_request_cpu",
+                    "value": 50.75,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___listen_queue",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___listen_queue_len",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___max_active_processes",
+                    "value": 5,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___max_children_reached",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___max_listen_queue",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___slow_requests",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___start_since",
+                    "value": 31334,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "pools___thefrog___total_processes",
+                    "value": 5,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_accepted_conn",
+                    "value": 33127,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_active_processes",
+                    "value": 5,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_idle_processes",
+                    "value": 18,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_last_request_cpu",
+                    "value": 472.63,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_listen_queue",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_listen_queue_len",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_max_active_processes",
+                    "value": 17,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_max_children_reached",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_max_listen_queue",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_slow_requests",
+                    "value": 0,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_start_since",
+                    "value": 31334,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                },
+                {
+                    "metric": "totals_total_processes",
+                    "value": 23,
+                    "value_prev": null,
+                    "app_type": "php-fpm"
+                }
+            ]
+        }
+    }
+}

tests/snmpsim/linux_php-fpm-v1.snmprec

@@ -0,0 +1,10 @@
+1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64
+1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10
+1.3.6.1.2.1.1.3.0|67|77550514
+1.3.6.1.2.1.1.4.0|4|<private>
+1.3.6.1.2.1.1.5.0|4|<private>
+1.3.6.1.2.1.1.6.0|4|<private>
+1.3.6.1.2.1.25.1.1.0|67|77552962
+1.3.6.1.4.1.8072.1.3.2.2.1.21.6.100.105.115.116.114.111|2|1
+1.3.6.1.4.1.8072.1.3.2.2.1.21.8.112.104.112.102.112.109.115.112|2|1
+1.3.6.1.4.1.8072.1.3.2.3.1.2.8.112.104.112.102.112.109.115.112|4x|7b0a20202264617461223a207b0a20202020226572726f726564223a20302c0a2020202022706f6f6c5f6572726f7273223a207b7d2c0a2020202022706f6f6c73223a207b0a2020202020202274686566726f67223a207b0a202020202020202022616363657074656420636f6e6e223a20363833392c0a2020202020202020226163746976652070726f636573736573223a20312c0a20202020202020202269646c652070726f636573736573223a20342c0a2020202020202020226c617374207265717565737420637075223a2035302e37352c0a2020202020202020226c697374656e207175657565223a20302c0a2020202020202020226c697374656e207175657565206c656e223a20302c0a2020202020202020226d6178206163746976652070726f636573736573223a20352c0a2020202020202020226d6178206368696c6472656e2072656163686564223a20302c0a2020202020202020226d6178206c697374656e207175657565223a20302c0a202020202020202022706f6f6c223a202274686566726f67222c0a20202020202020202270726f63657373206d616e61676572223a202264796e616d6963222c0a202020202020202022736c6f77207265717565737473223a20302c0a20202020202020202273746172742073696e6365223a2033313333342c0a20202020202020202273746172742074696d65223a20313731383438313939342c0a202020202020202022746f74616c2070726f636573736573223a20350a2020202020207d0a202020207d2c0a2020202022746f74616c73223a207b0a20202020202022616363657074656420636f6e6e223a2033333132372c0a202020202020226163746976652070726f636573736573223a20352c0a2020202020202269646c652070726f636573736573223a2031382c0a202020202020226c617374207265717565737420637075223a203437322e36332c0a202020202020226c697374656e207175657565223a20302c0a202020202020226c697374656e207175657565206c656e223a20302c0a202020202020226d6178206163746976652070726f636573736573223a2031372c0a202020202020226d6178206368696c6472656e2072656163686564223a20302c0a202020202020226d6178206c697374656e207175657565223a20302c0a20202020202022736c6f77207265717565737473223a20302c0a2020202020202273746172742073696e6365223a2033313333342c0a20202020202022746f74616c2070726f636573736573223a2032330a202020207d0a20207d2c0a2020226572726f72223a20302c0a2020226572726f72537472696e67223a2022222c0a20202276657273696f6e223a20310a7d0a