From 494c29aefaaa0aa872ec9be8279d38ca8065ae85 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Sat, 20 Jan 2018 20:22:47 -0600 Subject: [PATCH] Split save-test-data.php into two scripts, allow mass update (#8115) * Add mass update to save-test-data. Fail when snmpsim doesn't start and try to give a hint. * Split save-test-data.php into two scripts One for updating snmprec files. One for updating db dump files. --- LibreNMS/Util/ModuleTestHelper.php | 80 +++++++++++++++++- doc/Developing/os/Test-Units.md | 26 +++--- scripts/collect-snmp-data.php | 116 ++++++++++++++++++++++++++ scripts/save-test-data.php | 125 ++++++++++++----------------- tests/OSModulesTest.php | 44 +++------- 5 files changed, 277 insertions(+), 114 deletions(-) create mode 100755 scripts/collect-snmp-data.php diff --git a/LibreNMS/Util/ModuleTestHelper.php b/LibreNMS/Util/ModuleTestHelper.php index f019345932..07d8783ba1 100644 --- a/LibreNMS/Util/ModuleTestHelper.php +++ b/LibreNMS/Util/ModuleTestHelper.php @@ -101,6 +101,16 @@ class ModuleTestHelper $this->quiet = $quiet; } + public function setSnmprecSavePath($path) + { + $this->snmprec_file = $path; + } + + public function setJsonSavePath($path) + { + $this->json_file = $path; + } + public function captureFromDevice($device_id, $write = true, $prefer_new = false) { $snmp_oids = $this->collectOids($device_id); @@ -184,6 +194,66 @@ class ModuleTestHelper return $snmp_oids; } + /** + * Generate a list of os containing test data for $modules (an empty array means all) + * + * Returns an array indexed by the basename ($os or $os_$variant) + * Each entry contains [$os, $variant, $valid_modules] + * $valid_modules is an array of selected modules this os has test data for + * + * @param array $modules + * @return array + */ + public static function findOsWithData($modules = array()) + { + $os_list = array(); + + foreach (glob(Config::get('install_dir') . "/tests/data/*.json") as $file) { + $base_name = basename($file, '.json'); + list($os, $variant) = self::extractVariant($file); + + // calculate valid modules + $data_modules = array_keys(json_decode(file_get_contents($file), true)); + + if (empty($modules)) { + $valid_modules = $data_modules; + } else { + $valid_modules = array_intersect($modules, $data_modules); + } + + if (empty($valid_modules)) { + continue; // no test data for selected modules + } + + $os_list[$base_name] = array( + $os, + $variant, + $valid_modules, + ); + } + + return $os_list; + } + + /** + * Given a json filename or basename, extract os and variant + * + * @param string $os_file Either a filename or the basename + * @return array [$os, $variant] + */ + public static function extractVariant($os_file) + { + $full_name = basename($os_file, '.json'); + + if (!str_contains($full_name, '_')) { + return [$full_name, '']; + } elseif (is_file(Config::get('install_dir') . "/includes/definitions/$full_name.yaml")) { + return [$full_name, '']; + } else { + list($rvar, $ros) = explode('_', strrev($full_name), 2); + return [strrev($ros), strrev($rvar)]; + } + } /** * Generate a module list. Try to take dependencies into account. @@ -581,8 +651,16 @@ class ModuleTestHelper return empty($this->modules) ? $this->getSupportedModules() : $this->modules; } - public function fetchTestData() + public function getTestData() { return json_decode(file_get_contents($this->json_file), true); } + + public function getJsonFilepath($short = false) + { + if ($short) { + return ltrim(str_replace(Config::get('install_dir'), '', $this->json_file), '/'); + } + return $this->json_file; + } } diff --git a/doc/Developing/os/Test-Units.md b/doc/Developing/os/Test-Units.md index d24dab2165..d3b3f4ca41 100644 --- a/doc/Developing/os/Test-Units.md +++ b/doc/Developing/os/Test-Units.md @@ -15,12 +15,15 @@ consistent manner. ## Capturing test data -`./scripts/save-test-data.php` is provided to make it easy to collect data for tests. Running save-test-data.php with -the --hostname (-h) allows you to capture all data used to discover and poll a device already added to LibreNMS. Make sure to -re-run the script if you add additional support. Check the command-line help for more options. +`./scripts/collect-snmp-data.php` is provided to make it easy to collect data for tests. Running collect-snmp-data.php + with the --hostname (-h) allows you to capture all data used to discover and poll a device already added to LibreNMS. + Make sure to re-run the script if you add additional support. Check the command-line help for more options. -Generally, you will only need to capture data (-h) once. After you have the data you need in the snmprec file, you can -update the json using -o (and -v if using a variant) after that. +After you have collected snmp data, run `./scripts/save-test-data.php` with the --os (-o) option to dump the post discovery +and post poll database entries to json files. + +Generally, you will only need to capture data once. After you have the data you need in the snmprec file, you can +just use save-test-data.php to update the database dump (json) after that. ### OS Variants @@ -44,7 +47,11 @@ To run the full suite of tests enable database and snmpsim reliant tests: `./scr ## Using snmpsim for testing +<<<<<<< HEAD You can run snmpsim to access test data by running `./scripts/save-test-data.php --snmpsim` +======= +You can run snmpsim to access test data by running `./scripts/collct-snmp-data.php --snmpsim` +>>>>>>> Split save-test-data.php into two scripts You may then run snmp queries against it using the os (and variant) as the community and 127.1.6.1:1161 as the host. ``` @@ -98,18 +105,19 @@ data, you must use a variant to store your test data (-v). ### Add initial detection 1. Add device to LibreNMS. It is generic and device_id = 42 -2. Run `./scripts/save-test-data.php -h 42 -m os`, initial snmprec will be created +2. Run `./scripts/collect-snmp-data.php -h 42 -m os`, initial snmprec will be created 3. [Add initial detection](Initial-Detection.md) for `example-os` 4. Run discovery to make sure it detects properly `./discovery.php -h 42` 5. Add any additional os items like version, hardware, features, or serial. -6. If there is additional snmp data required, run `./scripts/save-test-data.php -h 42 -m os` otherwise, run `./scripts/save-test-data.php -o example-os -m os` -7. Review data. If you modified the snmprec (don't modify json manually) run `./scripts/save-test-data.php -o example-os -m os` +6. If there is additional snmp data required, run `./scripts/collect-snmp-data.php -h 42 -m os` +7. Run `./scripts/save-test-data.php -o example-os -m os` to update the dumped database data. +7. Review data. If you modified the snmprec or code (don't modify json manually) run `./scripts/save-test-data.php -o example-os -m os` 8. Run `./scripts/pre-commit.php --db --snmpsim` 9. If the tests succeed submit a pull request ### Additional module support or test data 1. Add code to support module or support already exists. -2. `./scripts/save-test-data.php -h 42 -m `, this will add more data to the snmprec file +2. `./scripts/collect-snmp-data.php -h 42 -m `, this will add more data to the snmprec file 3. Review data. If you modified the snmprec (don't modify json manually) run `./scripts/save-test-data.php -o example-os -m ` 4. Run `./scripts/pre-commit.php --db --snmpsim` 5. If the tests succeed submit a pull request diff --git a/scripts/collect-snmp-data.php b/scripts/collect-snmp-data.php new file mode 100755 index 0000000000..a60ccd622e --- /dev/null +++ b/scripts/collect-snmp-data.php @@ -0,0 +1,116 @@ +#!/usr/bin/env php +run(); + exit; +} + +// check for hostname +if (isset($options['h'])) { + $hostname = $options['h']; +} elseif (isset($options['hostname'])) { + $hostname = $options['hostname']; +} + +if (isset($hostname)) { + if (is_numeric($hostname)) { + $device = device_by_id_cache($hostname); + } elseif (!empty($hostname)) { + $device = device_by_name($hostname); + } + + if (isset($device['os']) && $device['os'] != 'generic') { + $target_os = $device['os']; + } else { + echo "OS (-o, --os) required because device is generic.\n"; + exit; + } +} + +if (isset($options['help']) || empty($target_os)) { + echo "Script to collect snmp data from devices to be used for testing. +Snmp data is saved in tests/snmpsim. + +Usage: + You must specify an existing device to collect data from. +Required + -h, --hostname ID, IP, or hostname of the device to collect data from +Optional: + -m, --modules The discovery/poller module(s) to collect data for, comma delimited + -n, --prefer-new Prefer new snmprec data over existing data + -o, --os Name of the OS to save test data for (only used if device is generic) + -v, --variant The variant of the OS to use, usually the device model + -f, --file Save data to file instead of the standard location + -d, --debug Enable debug output + --snmpsim Run snmpsimd.py using the collected data for manual testing. +"; + exit; +} + +$debug = (isset($options['d']) || isset($options['debug'])); + +if (isset($options['m'])) { + $modules_input = $options['m']; + $modules = explode(',', $modules_input); +} elseif (isset($options['modules'])) { + $modules_input = $options['modules']; + $modules = explode(',', $modules_input); +} else { + $modules_input = 'all'; + $modules = array(); +} + +$variant = ''; +if (isset($options['v'])) { + $variant = $options['v']; +} elseif (isset($options['variant'])) { + $variant = $options['variant']; +} + +echo "OS: $target_os\n"; +echo "Module(s): $modules_input\n"; +if ($variant) { + echo "Variant: $variant\n"; +} +echo PHP_EOL; + +$capture = new ModuleTestHelper($modules, $target_os, $variant); + + +if (isset($options['f'])) { + $capture->setSnmprecSavePath($options['f']); +} elseif (isset($options['file'])) { + $capture->setSnmprecSavePath($options['file']); +} + +$prefer_new_snmprec = isset($options['n']) || isset($options['prefer-new']); + + +echo "Capturing Data: "; +$capture->captureFromDevice($device['device_id'], true, $prefer_new_snmprec); diff --git a/scripts/save-test-data.php b/scripts/save-test-data.php index 013a9200b4..98e80f6f55 100755 --- a/scripts/save-test-data.php +++ b/scripts/save-test-data.php @@ -12,19 +12,16 @@ $install_dir = realpath(__DIR__ . '/..'); chdir($install_dir); $options = getopt( - 'h:dncm:o:v:f:', + 'o:v:m:nf:dh', array( - 'debug', - 'collect-only', - 'no-save', - 'prefer-new', - 'hostname:', - 'help', - 'module:', 'os:', 'variant:', + 'modules:', + 'no-save', 'file:', + 'debug', 'snmpsim', + 'help', ) ); @@ -41,54 +38,34 @@ if (isset($options['snmpsim'])) { exit; } -if (isset($options['h'])) { - $hostname = $options['h']; -} elseif (isset($options['hostname'])) { - $hostname = $options['hostname']; -} -$target_os = ''; -if (isset($options['o'])) { - $target_os = $options['o']; -} elseif (isset($options['os'])) { - $target_os = $options['os']; -} - -if (isset($hostname)) { - if (is_numeric($hostname)) { - $device = device_by_id_cache($hostname); - } elseif (!empty($hostname)) { - $device = device_by_name($hostname); - } - - if (isset($device['os']) && $device['os'] != 'generic') { - $target_os = $device['os']; - } else { - echo "OS (-o, --os) required because device is generic.\n"; - exit; - } -} - -if (isset($options['help']) || empty($target_os)) { - echo "Script to extract test data from devices or update test data. -Snmp data is saved in tests/snmpsim and database data is saved in tests/data. +if (isset($options['h']) + || isset($options['help']) + || !(isset($options['o']) || isset($options['os']) || isset($options['m']) || isset($options['modules'])) +) { + echo "Script to update test data. Database data is saved in tests/data. Usage: - You must specify a valid hostname or os. - -h, --hostname ID, IP, or hostname of the device to extract data from - If this is not given, the existing snmp data will be used + You must specify a valid OS and/or module(s). + -o, --os Name of the OS to save test data for -v, --variant The variant of the OS to use, usually the device model -m, --modules The discovery/poller module(s) to collect data for, comma delimited - -c, --collect-only Only collect snmpsim data (does not require snmpsim) + -n, --no-save Don't save database entries, print them out instead + -f, --file Save data to file instead of the standard location -d, --debug Enable debug output - -n, --prefer-new Prefer new snmprec data over existing data - --no-save Don't save database entries, print them out instead - --snmpsim Just run snmpsimd.py for manual testing. + --snmpsim Run snmpsimd.py using the collected data for manual testing. "; exit; } +$os_name = false; +if (isset($options['o'])) { + $os_name = $options['o']; +} elseif (isset($os_list['os'])) { + $os_name = $options['os']; +} + if (isset($options['m'])) { $modules_input = $options['m']; $modules = explode(',', $modules_input); @@ -100,35 +77,22 @@ if (isset($options['m'])) { $modules = array(); } +$full_os_name = $os_name; $variant = ''; if (isset($options['v'])) { $variant = $options['v']; + $full_os_name = $os_name . '_' . $variant; } elseif (isset($options['variant'])) { $variant = $options['variant']; + $full_os_name = $os_name . '_' . $variant; } -echo "OS: $target_os\n"; -echo "Module: $modules_input\n"; -if ($variant) { - echo "Variant: $variant\n"; -} -echo PHP_EOL; +$os_list = array(); -$tester = new ModuleTestHelper($modules, $target_os, $variant); - - -// Capture snmp data -if ($device) { - echo "Capturing Data: "; - $prefer_new_snmprec = isset($options['n']) || isset($options['prefer-new']); - $tester->captureFromDevice($device['device_id'], true, $prefer_new_snmprec); - - if (isset($options['c']) || isset($options['collect-only'])) { - // just capture, don't generate json - exit; - } - - echo PHP_EOL; +if ($os_name) { + $os_list = [$full_os_name => [$os_name, $variant]]; +} else { + $os_list = ModuleTestHelper::findOsWithData($modules); } @@ -138,10 +102,27 @@ $snmpsim->fork(); $snmpsim_ip = $snmpsim->getIp(); $snmpsim_port = $snmpsim->getPort(); - -$no_save = isset($options['no-save']); -$test_data = $tester->generateTestData($snmpsim, $no_save); - -if ($no_save) { - print_r($test_data); +if (!$snmpsim->isRunning()) { + echo "Failed to start snmpsim, make sure it is installed, working, and there are no bad snmprec files.\n"; + exit; +} + + +$no_save = isset($options['n']) || isset($options['no-save']); +foreach ($os_list as $full_os_name => $parts) { + list($target_os, $target_variant) = $parts; + echo "OS: $target_os\n"; + echo "Module: $modules_input\n"; + if ($target_variant) { + echo "Variant: $target_variant\n"; + } + echo PHP_EOL; + + $tester = new ModuleTestHelper($modules, $target_os, $target_variant); + + $test_data = $tester->generateTestData($snmpsim, $no_save); + + if ($no_save) { + print_r($test_data); + } } diff --git a/tests/OSModulesTest.php b/tests/OSModulesTest.php index f940197aeb..29589b1f46 100644 --- a/tests/OSModulesTest.php +++ b/tests/OSModulesTest.php @@ -40,40 +40,37 @@ class OSModulesTest extends DBTestCase * @param string $filename file name of the json data * @param array $modules modules to test for this os */ - public function testOS($target_os, $filename, $modules) + public function testOS($os, $variant, $modules) { $this->requreSnmpsim(); // require snmpsim for tests global $snmpsim; - $file = Config::get('install_dir') . '/' . $filename; - $expected_data = json_decode(file_get_contents($file), true); - - list($os, $variant) = explode('_', $target_os, 2); - $helper = new ModuleTestHelper($modules, $os, $variant); $helper->setQuiet(); + $filename = $helper->getJsonFilepath(true); + $expected_data = $helper->getTestData(); $results = $helper->generateTestData($snmpsim, true); if (is_null($results)) { - $this->fail("$target_os: Failed to collect data."); + $this->fail("$os: Failed to collect data."); } foreach ($modules as $module) { $this->assertEquals( $expected_data[$module]['discovery'], $results[$module]['discovery'], - "OS $target_os: Discovered $module data does not match that found in $filename\n" + "OS $os: Discovered $module data does not match that found in $filename\n" . $helper->getLastDiscoveryOutput() - . "\nOS $target_os: Polled $module data does not match that found in $filename" + . "\nOS $os: Polled $module data does not match that found in $filename" ); $this->assertEquals( $expected_data[$module]['poller'] == 'matches discovery' ? $expected_data[$module]['discovery'] : $expected_data[$module]['poller'], $results[$module]['poller'], - "OS $target_os: Polled $module data does not match that found in $filename\n" + "OS $os: Polled $module data does not match that found in $filename\n" . $helper->getLastPollerOutput() - . "\nOS $target_os: Polled $module data does not match that found in $filename" + . "\nOS $os: Polled $module data does not match that found in $filename" ); } } @@ -81,29 +78,12 @@ class OSModulesTest extends DBTestCase public function dumpedDataProvider() { - $install_dir = Config::get('install_dir'); - $dump_files = glob("$install_dir/tests/data/*.json"); - $data = array(); - $modules = getenv('TEST_MODULES') ? explode(',', getenv('TEST_MODULES')) : array(); + $modules = array(); - foreach ($dump_files as $file) { - $os = basename($file, '.json'); - $short_file = str_replace($install_dir.'/', '', $file); - $data_modules = array_keys(json_decode(file_get_contents($file), true)); - - if (count(array_diff($modules, $data_modules)) !== 0) { - continue; // no test data for selected modules - } - - $test_modules = empty($modules) ? $data_modules : array_intersect($modules, $data_modules); - - $data[$os] = array( - $os, - $short_file, - $test_modules, - ); + if (getenv('TEST_MODULES')) { + $modules = explode(',', getenv('TEST_MODULES')); } - return $data; + return ModuleTestHelper::findOsWithData($modules); } }