Dispatcher option to log output (#15230)

* Dispatcher option to log output
-o --log-output Log output into various files in the log directory
wire up -d option to be passed into scheduled commands
Caution, can fill your disk.

* style fixes

* more silly style fixes (and a typo accidentally added)

* final lint maybe?

* more lint...

* believe it or not, more lint

LibreNMS/queuemanager.py

@@ -362,19 +362,24 @@ class BillingQueueManager(TimedQueueManager):
     def do_work(self, run_type, group):
         if run_type == "poll":
             logger.info("Polling billing")
-            exit_code, output = LibreNMS.call_script("poll-billing.php")
-            if exit_code != 0:
-                logger.warning(
-                    "Error {} in Polling billing:\n{}".format(exit_code, output)
-                )
+            args = ("-d") if self.config.debug else ()
+            exit_code, output = LibreNMS.call_script("poll-billing.php", args)
         else:  # run_type == 'calculate'
             logger.info("Calculating billing")
-            exit_code, output = LibreNMS.call_script("billing-calculate.php")
+            args = ("-d") if self.config.debug else ()
+            exit_code, output = LibreNMS.call_script("billing-calculate.php", args)
+
         if exit_code != 0:
             logger.warning(
-                    "Error {} in Calculating billing:\n{}".format(exit_code, output)
+                "Error {} in {} billing:\n{}".format(exit_code, run_type, output)
             )
 
+        if self.config.log_output:
+            with open(
+                "{}/dispatch_billing-{}.log".format(self.config.logdir, run_type), "a"
+            ) as log_file:
+                log_file.write(output)
+
 
 class PingQueueManager(TimedQueueManager):
     def __init__(self, config, lock_manager):
@@ -401,7 +406,19 @@ class PingQueueManager(TimedQueueManager):
         if self.lock(group, "group", timeout=self.config.ping.frequency):
             try:
                 logger.info("Running fast ping")
-                exit_code, output = LibreNMS.call_script("ping.php", ("-g", group))
+
+                args = ("-d", "-g", group) if self.config.debug else ("-g", group)
+                exit_code, output = LibreNMS.call_script("ping.php", args)
+
+                if self.config.log_output:
+                    with open(
+                        "{}/dispatch_group_{}_ping.log".format(
+                            self.config.logdir, group
+                        ),
+                        "a",
+                    ) as log_file:
+                        log_file.write(output)
+
                 if exit_code != 0:
                     logger.warning(
                         "Running fast ping for {} failed with error code {}: {}".format(
@@ -439,9 +456,18 @@ class ServicesQueueManager(TimedQueueManager):
     def do_work(self, device_id, group):
         if self.lock(device_id, timeout=self.config.services.frequency):
             logger.info("Checking services on device {}".format(device_id))
-            exit_code, output = LibreNMS.call_script(
-                "check-services.php", ("-h", device_id)
-            )
+            args = ("-d", "-h", device_id) if self.config.debug else ("-h", device_id)
+            exit_code, output = LibreNMS.call_script("check-services.php", args)
+
+            if self.config.log_output:
+                with open(
+                    "{}/dispatch_device_{}_services.log".format(
+                        self.config.logdir, device_id
+                    ),
+                    "a",
+                ) as log_file:
+                    log_file.write(output)
+
             if exit_code == 0:
                 self.unlock(device_id)
             else:
@@ -480,7 +506,16 @@ class AlertQueueManager(TimedQueueManager):
 
     def do_work(self, device_id, group):
         logger.info("Checking alerts")
-        exit_code, output = LibreNMS.call_script("alerts.php")
+        args = ("-d") if self.config.debug else ()
+        exit_code, output = LibreNMS.call_script("alerts.php", args)
+
+        if self.config.log_output:
+            with open(
+                "{}/dispatch_alerts.log".format(self.config.logdir),
+                "a",
+            ) as log_file:
+                log_file.write(output)
+
         if exit_code != 0:
             if exit_code == 1:
                 logger.warning("There was an error issuing alerts: {}".format(output))
@@ -504,7 +539,18 @@ class PollerQueueManager(QueueManager):
         if self.lock(device_id, timeout=self.config.poller.frequency):
             logger.info("Polling device {}".format(device_id))
 
-            exit_code, output = LibreNMS.call_script("poller.php", ("-h", device_id))
+            args = ("-d", "-h", device_id) if self.config.debug else ("-h", device_id)
+            exit_code, output = LibreNMS.call_script("poller.php", args)
+
+            if self.config.log_output:
+                with open(
+                    "{}/dispatch_device_{}_poller.log".format(
+                        self.config.logdir, device_id
+                    ),
+                    "a",
+                ) as log_file:
+                    log_file.write(output)
+
             if exit_code == 0:
                 self.unlock(device_id)
             else:
@@ -557,7 +603,19 @@ class DiscoveryQueueManager(TimedQueueManager):
             device_id, timeout=LibreNMS.normalize_wait(self.config.discovery.frequency)
         ):
             logger.info("Discovering device {}".format(device_id))
-            exit_code, output = LibreNMS.call_script("discovery.php", ("-h", device_id))
+
+            args = ("-d", "-h", device_id) if self.config.debug else ("-h", device_id)
+            exit_code, output = LibreNMS.call_script("discovery.php", args)
+
+            if self.config.log_output:
+                with open(
+                    "{}/dispatch_device_{}_discovery.log".format(
+                        self.config.logdir, device_id
+                    ),
+                    "a",
+                ) as log_file:
+                    log_file.write(output)
+
             if exit_code == 0:
                 self.unlock(device_id)
             else:

LibreNMS/service.py

@@ -100,6 +100,9 @@ class ServiceConfig(DBConfig):
     redis_sentinel_service = None
     redis_timeout = 60
 
+    log_output = False
+    logdir = "logs"
+
     watchdog_enabled = False
     watchdog_logfile = "logs/librenms.log"
 
@@ -248,7 +251,8 @@ class ServiceConfig(DBConfig):
         self.watchdog_enabled = config.get(
             "service_watchdog_enabled", ServiceConfig.watchdog_enabled
         )
-        self.watchdog_logfile = config.get("log_file", ServiceConfig.watchdog_logfile)
+        self.logdir = config.get("log_dir", ServiceConfig.BASE_DIR + "/logs")
+        self.watchdog_logfile = config.get("log_file", self.logdir + "/librenms.log")
 
         # set convenient debug variable
         self.debug = logging.getLogger().isEnabledFor(logging.DEBUG)

librenms-service.py

@@ -5,11 +5,10 @@ import logging
 import os
 import sys
 import threading
+from logging import info
 
 import LibreNMS
 
-from logging import info
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="LibreNMS Service - manages polling and other periodic processes"
@@ -23,6 +22,12 @@ if __name__ == "__main__":
     )
     parser.add_argument("-v", "--verbose", action="count", help="Show verbose output.")
     parser.add_argument("-d", "--debug", action="store_true", help="Show debug output.")
+    parser.add_argument(
+        "-o",
+        "--log-output",
+        action="store_true",
+        help="Log poller ouput to files. Warning: This could use significant disk space!",
+    )
     parser.add_argument(
         "-m",
         "--multiple",
@@ -61,6 +66,7 @@ if __name__ == "__main__":
         sys.exit(2)
 
     service.config.single_instance = args.multiple
+    service.config.log_output = args.log_output
 
     if args.group:
         if isinstance(args.group, list):