diff --git a/discovery-wrapper.py b/discovery-wrapper.py new file mode 100755 index 0000000000..024a9588dd --- /dev/null +++ b/discovery-wrapper.py @@ -0,0 +1,387 @@ +#! /usr/bin/env python2 +""" + discovery-wrapper A small tool which wraps around discovery and tries to + guide the discovery process with a more modern approach with a + Queue and workers. + + Based on the original version of poller-wrapper.py by Job Snijders + + Author: Neil Lathwood + Date: Sep 2016 + + Usage: This program accepts one command line argument: the number of threads + that should run simultaneously. If no argument is given it will assume + a default of 1 thread. + + Ubuntu Linux: apt-get install python-mysqldb + FreeBSD: cd /usr/ports/*/py-MySQLdb && make install clean + + License: This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program. If not, see http://www.gnu.org/licenses/. + + LICENSE.txt contains a copy of the full GPLv3 licensing conditions. +""" +try: + + import json + import os + import Queue + import subprocess + import sys + import threading + import time + +except: + print "ERROR: missing one or more of the following python modules:" + print "threading, Queue, sys, subprocess, time, os, json" + sys.exit(2) + +try: + import MySQLdb +except: + print "ERROR: missing the mysql python module:" + print "On ubuntu: apt-get install python-mysqldb" + print "On FreeBSD: cd /usr/ports/*/py-MySQLdb && make install clean" + sys.exit(2) + +""" + Fetch configuration details from the config_to_json.php script +""" + +install_dir = os.path.dirname(os.path.realpath(__file__)) +config_file = install_dir + '/config.php' + + +def get_config_data(): + config_cmd = ['/usr/bin/env', 'php', '%s/config_to_json.php' % install_dir] + try: + proc = subprocess.Popen(config_cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE) + except: + print "ERROR: Could not execute: %s" % config_cmd + sys.exit(2) + return proc.communicate()[0] + +try: + with open(config_file) as f: + pass +except IOError as e: + print "ERROR: Oh dear... %s does not seem readable" % config_file + sys.exit(2) + +try: + config = json.loads(get_config_data()) +except: + print "ERROR: Could not load or parse configuration, are PATHs correct?" + sys.exit(2) + +discovery_path = config['install_dir'] + '/discovery.php' +db_username = config['db_user'] +db_password = config['db_pass'] + +if config['db_host'][:5].lower() == 'unix:': + db_server = config['db_host'] + db_port = 0 +elif ':' in config['db_host']: + db_server = config['db_host'].rsplit(':')[0] + db_port = int(config['db_host'].rsplit(':')[1]) +else: + db_server = config['db_host'] + db_port = 0 + +db_dbname = config['db_name'] + + +def db_open(): + try: + if db_port == 0: + db = MySQLdb.connect(host=db_server, user=db_username, passwd=db_password, db=db_dbname) + else: + db = MySQLdb.connect(host=db_server, port=db_port, user=db_username, passwd=db_password, db=db_dbname) + return db + except: + print "ERROR: Could not connect to MySQL database!" + sys.exit(2) + +# (c) 2015, GPLv3, Daniel Preussker << << << << << << 0 and nodes is not None: + try: + time.sleep(1) + nodes = memc.get("discovery.nodes") + except: + pass + print "Clearing Locks" + x = minlocks + while x <= maxlocks: + memc.delete('discovery.device.' + str(x)) + x = x + 1 + print "%s Locks Cleared" % x + print "Clearing Nodes" + memc.delete("discovery.master") + memc.delete("discovery.nodes") + else: + memc.decr("discovery.nodes") + print "Finished %s." % time.time() +# EOC6 + +show_stopper = False + +if total_time > 21600: + print "WARNING: the process took more than 6 hours to finish, you need faster hardware or more threads" + print "INFO: in sequential style discovery the elapsed time would have been: %s seconds" % real_duration + for device in per_device_duration: + if per_device_duration[device] > 3600: + print "WARNING: device %s is taking too long: %s seconds" % (device, per_device_duration[device]) + show_stopper = True + if show_stopper: + print "ERROR: Some devices are taking more than 3600 seconds, the script cannot recommend you what to do." + else: + recommend = int(total_time / 300.0 * amount_of_workers + 1) + print "WARNING: Consider setting a minimum of %d threads. (This does not constitute professional advice!)" % recommend + + sys.exit(2) diff --git a/doc/Support/Discovery Support.md b/doc/Support/Discovery Support.md index 3086a25b9a..8034a3ce65 100644 --- a/doc/Support/Discovery Support.md +++ b/doc/Support/Discovery Support.md @@ -35,6 +35,21 @@ new will poll only those devices that have recently been added or have been sele `-m` This enables you to specify the module you want to run for discovery. +#### Discovery wrapper + +We have a `discovery-wrapper.py` script which is based on `poller-wrapper.py` by [Job Snijders](https://github.com/job). +You can enable support for this within cron by replacing: + +`33 */6 * * * librenms /opt/librenms/discovery.php -h all >> /dev/null 2>&1` + +With: + +`33 */6 * * * librenms /opt/librenms/discovery-wrapper.php 1 >> /dev/null 2>&1` + +The default is for discovery wrapper to only use 1 thread so that it mimics the current behaviour. However if your +system is powerful enough and the devices can cope then you can increase the thread count from 1 to a value of your +choosing. + #### Discovery config These are the default discovery config items. You can globally disable a module by setting it to 0. If you just want to