Added debug options to wrapper scripts to record output to logs/ dir (#8811)

* Ability to enable debug output with wrappers
Output is redirected to a per-device file in the log directory.

* Add warning

* Added some docs to help with debugging using this new option

discovery-wrapper.py

@@ -40,6 +40,7 @@ try:
     import sys
     import threading
     import time
+    import argparse
 
 except:
     print "ERROR: missing one or more of the following python modules:"
@@ -85,6 +86,7 @@ except:
     sys.exit(2)
 
 discovery_path = config['install_dir'] + '/discovery.php'
+log_dir = config['log_dir']
 db_username = config['db_user']
 db_password = config['db_pass']
 db_port = int(config['db_port'])
@@ -188,13 +190,16 @@ discovered_devices = 0
     Take the amount of threads we want to run in parallel from the commandline
     if None are given or the argument was garbage, fall back to default of 16
 """
-try:
+parser = argparse.ArgumentParser(description='Spawn multiple discovery.php processes in parallel.',
-    amount_of_workers = int(sys.argv[1])
+                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    if amount_of_workers == 0:
+parser.add_argument('-d', '--debug', action='store_true', default=False,
-        print "ERROR: 0 threads is not a valid value"
+                    help="Enable debug output. WARNING: Leaving this enabled will consume a lot of disk space.")
-        sys.exit(2)
+parser.add_argument('workers', metavar='N', type=int, default=1, nargs='?',
-except:
+                    help='The max number of workers allowed to run at one time. If too high this can overwelm your server.')
-    amount_of_workers = 1
+args = parser.parse_args()
+
+amount_of_workers = args.workers
+debug = args.debug
 
 devices_list = []
 
@@ -304,8 +309,11 @@ def poll_worker():
 # EOC5
             try:
                 start_time = time.time()
-                command = "/usr/bin/env php %s -h %s >> /dev/null 2>&1" % (discovery_path, device_id)
+
+                output = "-d >> %s/discover_device_%s.log" % (log_dir, device_id) if debug else ">> /dev/null"
+                command = "/usr/bin/env php %s -h %s %s 2>&1" % (discovery_path, device_id, output)
                 subprocess.check_call(command, shell=True)
+
                 elapsed_time = int(time.time() - start_time)
                 print_queue.put([threading.current_thread().name, device_id, elapsed_time])
             except (KeyboardInterrupt, SystemExit):

doc/Extensions/Services.md

@@ -11,6 +11,8 @@ to LibreNMS - localhost is a good one. This is needed in order for alerting to w
 > Service checks is now distributed aware. If you run a distributed setup then you can now run 
 `services-wrapper.py` in cron instead of `check-services.php` across all polling nodes.
 
+If you need to debug the output of services-wrapper.py then you can add `-d` to the end of the command - it is NOT recommended to do this in cron.
+
 Firstly, install Nagios plugins however you would like, this could be via yum, apt-get or direct from source.
 
 Next, you need to enable the services within config.php with the following:
@@ -30,9 +32,9 @@ For example:
 chmod +x /usr/lib/nagios/plugins/*
 ```
 
-Finally, you now need to add check-services.php to the current cron file (/etc/cron.d/librenms typically) like:
+Finally, you now need to add services-wrapper.py to the current cron file (/etc/cron.d/librenms typically) like:
 ```bash
-*/5 * * * * librenms /opt/librenms/check-services.php >> /dev/null 2>&1
+*/5 * * * * librenms /opt/librenms/services-wrapper.py 1
 ```
 
 Now you can add services via the main Services link in the navbar, or via the 'Add Service' link within the device, services page.

doc/Support/Discovery Support.md

@@ -39,6 +39,8 @@ new will poll only those devices that have recently been added or have been sele
 
 We have a `discovery-wrapper.py` script which is based on `poller-wrapper.py` by [Job Snijders](https://github.com/job). This script is currently the default.
 
+If you need to debug the output of discovery-wrapper.py then you can add `-d` to the end of the command - it is NOT recommended to do this in cron.
+
 If you want to switch back to discovery.php then you can replace:
 
 `33  */6   * * *   librenms    /opt/librenms/discovery-wrapper.py 1 >> /dev/null 2>&1`

doc/Support/Poller Support.md

@@ -36,6 +36,12 @@ even. all will run poller against all devices.
 
 `-m` This enables you to specify the module you want to run for poller.
 
+#### Poller Wrapper
+
+We have a `poller-wrapper.py` script by [Job Snijders](https://github.com/job). This script is currently the default.
+
+If you need to debug the output of poller-wrapper.py then you can add `-d` to the end of the command - it is NOT recommended to do this in cron.
+
 #### Poller config
 
 These are the default poller config items. You can globally disable a module by setting it to 0. If you just want to

poller-wrapper.py

@@ -30,6 +30,7 @@ try:
     import sys
     import threading
     import time
+    import argparse
 
 except:
     print "ERROR: missing one or more of the following python modules:"
@@ -75,6 +76,7 @@ except:
     sys.exit(2)
 
 poller_path = config['install_dir'] + '/poller.php'
+log_dir = config['log_dir']
 db_username = config['db_user']
 db_password = config['db_pass']
 db_port = int(config['db_port'])
@@ -194,13 +196,16 @@ polled_devices = 0
     Take the amount of threads we want to run in parallel from the commandline
     if None are given or the argument was garbage, fall back to default of 16
 """
-try:
+parser = argparse.ArgumentParser(description='Spawn multiple poller.php processes in parallel.',
-    amount_of_workers = int(sys.argv[1])
+                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    if amount_of_workers == 0:
+parser.add_argument('-d', '--debug', action='store_true', default=False,
-        print "ERROR: 0 threads is not a valid value"
+                    help="Enable debug output. WARNING: Leaving this enabled will consume a lot of disk space.")
-        sys.exit(2)
+parser.add_argument('workers', metavar='N', type=int, default=16, nargs='?',
-except:
+                    help='The max number of workers allowed to run at one time. If too high this can overwelm your server.')
-    amount_of_workers = 16
+args = parser.parse_args()
+
+amount_of_workers = args.workers
+debug = args.debug
 
 devices_list = []
 
@@ -310,8 +315,11 @@ def poll_worker():
 # EOC5
             try:
                 start_time = time.time()
-                command = "/usr/bin/env php %s -h %s >> /dev/null 2>&1" % (poller_path, device_id)
+
+                output = "-d >> %s/poll_device_%s.log" % (log_dir, device_id) if debug else ">> /dev/null"
+                command = "/usr/bin/env php %s -h %s %s 2>&1" % (poller_path, device_id, output)
                 subprocess.check_call(command, shell=True)
+
                 elapsed_time = int(time.time() - start_time)
                 print_queue.put([threading.current_thread().name, device_id, elapsed_time])
             except (KeyboardInterrupt, SystemExit):

services-wrapper.py

@@ -40,6 +40,7 @@ try:
     import sys
     import threading
     import time
+    import argparse
 
 except:
     print "ERROR: missing one or more of the following python modules:"
@@ -85,6 +86,7 @@ except:
     sys.exit(2)
 
 service_path = config['install_dir'] + '/check-services.php'
+log_dir = config['log_dir']
 db_username = config['db_user']
 db_password = config['db_pass']
 
@@ -192,13 +194,16 @@ service_devices = 0
     Take the amount of threads we want to run in parallel from the commandline
     if None are given or the argument was garbage, fall back to default of 16
 """
-try:
+parser = argparse.ArgumentParser(description='Spawn multiple check-services.php processes in parallel.',
-    amount_of_workers = int(sys.argv[1])
+                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    if amount_of_workers == 0:
+parser.add_argument('-d', '--debug', action='store_true', default=False,
-        print "ERROR: 0 threads is not a valid value"
+                    help="Enable debug output. WARNING: Leaving this enabled will consume a lot of disk space.")
-        sys.exit(2)
+parser.add_argument('workers', metavar='N', type=int, default=1, nargs='?',
-except:
+                    help='The max number of workers allowed to run at one time. If too high this can overwelm your server.')
-    amount_of_workers = 1
+args = parser.parse_args()
+
+amount_of_workers = args.workers
+debug = args.debug
 
 devices_list = []
 
@@ -302,8 +307,11 @@ def poll_worker():
 # EOC5
             try:
                 start_time = time.time()
-                command = "/usr/bin/env php %s -h %s >> /dev/null 2>&1" % (service_path, device_id)
+
+                output = "-d >> %s/services_device_%s.log" % (log_dir, device_id) if debug else ">> /dev/null"
+                command = "/usr/bin/env php %s -h %s %s 2>&1" % (service_path, device_id, output)
                 subprocess.check_call(command, shell=True)
+
                 elapsed_time = int(time.time() - start_time)
                 print_queue.put([threading.current_thread().name, device_id, elapsed_time])
             except (KeyboardInterrupt, SystemExit):