From af271957469c6b1cf904c2a94ed15f438409aa94 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Fri, 17 Nov 2017 03:10:37 -0600 Subject: [PATCH] feature: Script to collect port polling data and compare full walk vs selective port (#7626) * Script to collect port polling data and compare full walk vs selective port polling. * Select hosts. Colors. Percent. Units (s). Enable sel poll. * Moved the colors. Disable selective if needed. Report set status and count. * Style fix. * CLI help. * Docs. * Style fix. * Allow comma separated device_id list. Better color formatting for output Fix issue when using &$device by reference --- doc/Support/Performance.md | 20 ++-- scripts/collect-port-polling.php | 178 +++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 11 deletions(-) create mode 100755 scripts/collect-port-polling.php diff --git a/doc/Support/Performance.md b/doc/Support/Performance.md index 9e11f7cbee..89d795c8c4 100644 --- a/doc/Support/Performance.md +++ b/doc/Support/Performance.md @@ -67,9 +67,11 @@ The default 16 threads that `poller-wrapper.py` runs as isn't necessarily the op This can be changed by going to the cron job for librenms. Usually in /etc/cron.d/librenms and changing the "16" +``` */5 * * * * librenms /opt/librenms/cronic /opt/librenms/poller-wrapper.py 16 +``` -KEEP in MIND that this dosnt always help, it depnds on your system and CPU. So Be careful. +KEEP in MIND that this doesn't always help, it depends on your system and CPU. So be careful. #### Recursive DNS @@ -84,17 +86,13 @@ or it's disabled then we still collect data. For the most part this is fine as t either deleted or disabled then this approach isn't optimal. So to counter this you can enable 'selected port polling' per device within the edit device -> misc section or by globally enabling it (not recommended): `$config['polling']['selected_ports'] = true;`. You can also set it for a specific OS: `$config['os']['ios']['polling']['selected_ports'] = true;`. -If you would like to see if you should turn this on then run this query in MySQL: `select device_id, count(*) as total from ports where deleted=1 group by device_id order by total desc;`. You will see output like the following: +Running `./scripts/collect-port-polling.php` will poll your devices with both full and selective polling, display a table with the difference and optionally enable or disable selected ports polling for devices which would benefit from a change. +Note that it doesn't continously re-evaluate this, it will only be updated when the script is run. There are a number of options: -| device_id | total | -| --------: | ----: | -| 128 | 339 | -| 92 | 56 | -| 41 | 41 | -| 38 | 3 | -| 81 | 2 | - -Here device id 128 and potentially 92 and 41 are likely candidates for this feature to be enabled on. Turn it on and then closely monitor the device for the next 15-30 minutes. +``` +-h | Poll single device or wildcard hostname +-e Enable/disable selected ports polling for devices which would benefit from a change +``` ### Web interface diff --git a/scripts/collect-port-polling.php b/scripts/collect-port-polling.php new file mode 100755 index 0000000000..655a69f9c6 --- /dev/null +++ b/scripts/collect-port-polling.php @@ -0,0 +1,178 @@ +#!/usr/bin/env php + | Poll single device, wildcard hostname, or comma separated list\n"; + echo "-e Enable/disable selected ports polling for devices which would benefit from a change\n"; + echo "\n"; +} + +if (isset($options['d'])) { + $debug = true; +} + +if (isset($options['help'])) { + print_help(); + exit(0); +} + +if (isset($options['h'])) { + if (is_numeric($options['h'])) { + $where = "AND `device_id` = ?"; + $params = array($options['h']); + } elseif (str_contains($options['h'], ',')) { + $device_ids = array_map('trim', explode(',', $options['h'])); + $device_ids = array_filter($device_ids, 'is_numeric'); + $where = 'AND `device_id` in ' . dbGenPlaceholders(count($device_ids)); + $params = $device_ids; + } else { + $where = "AND `hostname` LIKE ?"; + $params = array(str_replace('*', '%', mres($options['h']))); + } + $devices = dbFetch("SELECT * FROM `devices` WHERE status = 1 AND disabled = 0 $where ORDER BY `hostname` ASC", $params); +} else { + $devices = get_all_devices(); +} + +if (isset($options['e'])) { + if (!is_numeric($options['e']) || $options['e'] < 0) { + print_help(); + exit(1); + } + $enable_sel_value = $options['e']; +} + +echo "Full Polling: "; +Config::set('polling.selected_ports', false); +foreach ($devices as $index => $device) { + echo $device['device_id'] . ' '; + if (!$debug) { + ob_start(); + } + + $port_test_start = microtime(true); + include $install_dir . '/includes/polling/ports.inc.php'; + $devices[$index]['full_time_sec'] = microtime(true) - $port_test_start; + ob_end_clean(); +} +echo PHP_EOL; + +Config::set('polling.selected_ports', true); +echo "Selective Polling: "; +foreach ($devices as $index => $device) { + echo $device['device_id'] . ' '; + if (!$debug) { + ob_start(); + } + + $port_test_start = microtime(true); + include $install_dir . '/includes/polling/ports.inc.php'; + $devices[$index]['selective_time_sec'] = microtime(true) - $port_test_start; + ob_end_clean(); +} +echo PHP_EOL; + + +// collect port counts +$inactive_sql = "`deleted` = 1 OR `ifAdminStatus` != 'up' OR `disabled` = 1"; +$set_count = 0; +foreach ($devices as &$device) { + $count = dbFetchCell('SELECT COUNT(*) FROM `ports` WHERE `device_id`=?', array($device['device_id'])); + $inactive = dbFetchCell( + "SELECT COUNT(*) FROM `ports` WHERE `device_id`=? AND ($inactive_sql)", + array($device['device_id']) + ); + + $device['port_count'] = $count; + $device['inactive_ratio'] = ($inactive == 0 ? 0 : ($inactive / $count)); + $device['diff_sec'] = $device['selective_time_sec'] - $device['full_time_sec']; + $device['diff_perc'] = ($device['diff_sec'] / $device['full_time_sec']) * 100; + + // $enable_sel_value is negative and we want to enable it for all devices with an even lower value. + // It also has to save more than 1 s, or we might enable it for devices with i.e. 100ms vs 50ms, which isn't needed. + $device['set'] = "none"; + if (isset($enable_sel_value) && $device['diff_perc'] < ($enable_sel_value * -1) && $device['diff_sec'] < -1) { + set_dev_attrib($device, 'selected_ports', "true"); + $device['set'] = "true"; + $set_count++; + } + if (isset($enable_sel_value) && $device['diff_perc'] > $enable_sel_value && $device['diff_sec'] > 1) { + set_dev_attrib($device, 'selected_ports', "false"); + $device['set'] = "false"; + $set_count++; + } +} +unset($device); // will edit the wrong thing after using $device by reference + +// print out the results +$stats = array( + 'device_id', + 'os', + 'port_count', + 'inactive_ratio', + 'full_time', + 'selective_time', + 'diff', + 'diff', + 'set' +); + +echo PHP_EOL; +$header = "| %9.9s | %-11.11s | %10.10s | %14.14s | %10.10s | %14.14s | %8.10s | %5.9s | %5.5s |\n"; +call_user_func_array('printf', array_merge(array($header), $stats)); + +$mask = "| %9.9s | %-11.11s | %10.10s | %14.3f | %9.3fs | %13.3fs | %s%+7.3fs\e[0m | %s%+4.0f%%\e[0m | %5.5s |\n"; +foreach ($devices as $device) { + $diff_color = ($device['diff_sec'] > 0 ? "\033[0;31m" : "\033[0;32m"); + printf( + $mask, + $device['device_id'], + $device['os'], + $device['port_count'], + $device['inactive_ratio'], + $device['full_time_sec'], + $device['selective_time_sec'], + $diff_color, + $device['diff_sec'], + $diff_color, + $device['diff_perc'], + $device['set'] + ); +} + +$total_ports = array_sum(array_column($devices, 'port_count')); +$inactive_ratio = array_sum(array_column($devices, 'inactive_ratio')) / count($devices); +$total_full_time = array_sum(array_column($devices, 'full_time_sec')); +$total_selective_time = array_sum(array_column($devices, 'selective_time_sec')); +$difference = $total_selective_time - $total_full_time; +$difference_perc = ($difference / $total_full_time) * 100; +$total_diff_color = ($difference > 0 ? "\033[0;31m" : "\033[0;32m"); + +printf( + $mask, + 'Totals:', + '', + $total_ports, + $inactive_ratio, + $total_full_time, + $total_selective_time, + $total_diff_color, + $difference, + $total_diff_color, + $difference_perc, + $set_count +);