From 7ed34c889ee203c9cb02077c643b4838b1e2cc10 Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Mon, 23 Nov 2020 00:33:56 -0600 Subject: [PATCH] Distributed Poller improved validation (#12269) --- LibreNMS/Exceptions/InvalidNameException.php | 32 +++++++++ LibreNMS/ValidationResult.php | 26 +++++--- LibreNMS/Validations/DistributedPoller.php | 69 +++++++++++++++++--- LibreNMS/Validator.php | 11 ++++ app/Models/PollerCluster.php | 36 +++++++++- config/librenms.php | 10 +++ 6 files changed, 164 insertions(+), 20 deletions(-) create mode 100644 LibreNMS/Exceptions/InvalidNameException.php diff --git a/LibreNMS/Exceptions/InvalidNameException.php b/LibreNMS/Exceptions/InvalidNameException.php new file mode 100644 index 0000000000..12f91f54bc --- /dev/null +++ b/LibreNMS/Exceptions/InvalidNameException.php @@ -0,0 +1,32 @@ +. + * + * @package LibreNMS + * @link http://librenms.org + * @copyright 2020 Tony Murray + * @author Tony Murray + */ + +namespace LibreNMS\Exceptions; + + +class InvalidNameException extends \Exception +{ + +} diff --git a/LibreNMS/ValidationResult.php b/LibreNMS/ValidationResult.php index 13dda9ac87..71a76dc863 100644 --- a/LibreNMS/ValidationResult.php +++ b/LibreNMS/ValidationResult.php @@ -29,6 +29,7 @@ class ValidationResult const FAILURE = 0; const WARNING = 1; const SUCCESS = 2; + const INFO = 3; private $message; private $status; @@ -71,6 +72,16 @@ class ValidationResult return new self($message, self::WARNING, $fix); } + /** + * Create a new informational Validation result + * @param string $message The message to describe this result + * @return ValidationResult + */ + public static function info($message) + { + return new self($message, self::INFO); + } + /** * Create a new failure Validation result * @param string $message The message to describe this result @@ -173,15 +184,14 @@ class ValidationResult */ public static function getStatusText($status) { - if ($status === self::SUCCESS) { - return '%gOK%n'; - } elseif ($status === self::WARNING) { - return '%YWARN%n'; - } elseif ($status === self::FAILURE) { - return '%RFAIL%n'; - } + $table = [ + self::SUCCESS => '%gOK%n', + self::WARNING => '%YWARN%n', + self::FAILURE => '%RFAIL%n', + self::INFO => '%CINFO%n', + ]; - return 'Unknown'; + return $table[$status] ?? 'Unknown'; } public function getListDescription() diff --git a/LibreNMS/Validations/DistributedPoller.php b/LibreNMS/Validations/DistributedPoller.php index 0c10cb176b..f33900f1a3 100644 --- a/LibreNMS/Validations/DistributedPoller.php +++ b/LibreNMS/Validations/DistributedPoller.php @@ -31,12 +31,18 @@ namespace LibreNMS\Validations; +use App\Models\PollerCluster; +use Carbon\Carbon; use LibreNMS\Config; use LibreNMS\Validator; class DistributedPoller extends BaseValidation { - protected static $RUN_BY_DEFAULT = false; + public function isDefault() + { + // run by default if distributed polling is enabled + return Config::get('distributed_poller'); + } /** * Validate this module. @@ -47,11 +53,62 @@ class DistributedPoller extends BaseValidation public function validate(Validator $validator) { if (! Config::get('distributed_poller')) { - $validator->fail('You have not enabled distributed_poller'); + $validator->fail('You have not enabled distributed_poller', 'lnms config:set distributed_poller true'); return; } + if (! Config::get('rrdcached')) { + $validator->fail('You have not configured $config[\'rrdcached\']'); + } elseif (! is_dir(Config::get('rrd_dir'))) { + $validator->fail('You have not configured $config[\'rrd_dir\']'); + } else { + Rrd::checkRrdcached($validator); + } + + if (PollerCluster::exists()) { + if (PollerCluster::isActive()->exists()) { + $validator->info('Detected Dispatcher Service'); + $this->checkDispatcherService($validator); + return; + } + + $validator->warn('Dispatcher Service has been used in your cluster, but not recently. It may take up to 5 minutes to register.'); + } + + $validator->info('Detected Python Wrapper'); + $this->checkPythonWrapper($validator); + } + + private function checkDispatcherService(Validator $validator) + { + $driver = config('cache.default'); + if ($driver != 'redis') { + $validator->warn("Using $driver for distributed locking, you should set CACHE_DRIVER=redis"); + } + + try { + $lock = \Cache::lock('dist_test_validation'); + $lock->get(); + $lock->release(); + } catch (\Exception $e) { + $validator->fail('Locking server issue: ' . $e->getMessage()); + } + + $node = PollerCluster::firstWhere('node_id', config('librenms.node_id')); + if (! $node->exists) { + $validator->fail('Dispatcher service is enabled on your cluster, but not in use on this node'); + return; + } + + if ($node->last_report->lessThan(Carbon::now()->subSeconds($node->getSettingValue('poller_frequency')))) { + $validator->fail('Dispatcher service has not reported stats within the last poller window'); + } + + } + + private function checkPythonWrapper(Validator $validator) + { if (! Config::get('distributed_poller_memcached_host')) { $validator->fail('You have not configured $config[\'distributed_poller_memcached_host\']'); } elseif (! Config::get('distributed_poller_memcached_port')) { @@ -65,13 +122,5 @@ class DistributedPoller extends BaseValidation $validator->ok('Connection to memcached is ok'); } } - - if (! Config::get('rrdcached')) { - $validator->fail('You have not configured $config[\'rrdcached\']'); - } elseif (! is_dir(Config::get('rrd_dir'))) { - $validator->fail('You have not configured $config[\'rrd_dir\']'); - } else { - Rrd::checkRrdcached($validator); - } } } diff --git a/LibreNMS/Validator.php b/LibreNMS/Validator.php index 541cc188a2..bd98c5236b 100644 --- a/LibreNMS/Validator.php +++ b/LibreNMS/Validator.php @@ -218,6 +218,17 @@ class Validator $this->result(new ValidationResult($message, ValidationResult::FAILURE, $fix), $group); } + /** + * Submit an informational validation result. + * + * @param string $message + * @param string $group manually specify the group, otherwise this will be inferred from the callers class name + */ + public function info($message, $group = null) + { + $this->result(new ValidationResult($message, ValidationResult::INFO), $group); + } + /** * Get version_info() array. This will cache the result and add remote data if requested and not already existing. * diff --git a/app/Models/PollerCluster.php b/app/Models/PollerCluster.php index c640fe578e..ce9c276bd5 100644 --- a/app/Models/PollerCluster.php +++ b/app/Models/PollerCluster.php @@ -25,6 +25,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Model; +use LibreNMS\Exceptions\InvalidNameException; class PollerCluster extends Model { @@ -32,6 +33,9 @@ class PollerCluster extends Model protected $table = 'poller_cluster'; protected $primaryKey = 'id'; protected $fillable = ['poller_name']; + protected $casts = [ + 'last_report' => 'datetime', + ]; // ---- Accessors/Mutators ---- @@ -40,17 +44,45 @@ class PollerCluster extends Model $this->attributes['poller_groups'] = is_array($groups) ? implode(',', $groups) : $groups; } + // ---- Scopes ---- + + public function scopeIsActive($query) + { + $default = (int) \LibreNMS\Config::get('service_poller_frequency'); + $query->where('last_report', '>=', \DB::raw("DATE_SUB(NOW(),INTERVAL COALESCE(`poller_frequency`, $default) SECOND)")); + } + // ---- Helpers ---- + /** + * Get the value of a setting (falls back to the global value if not set on this node) + * + * @param string $name + * @return mixed + * @throws \LibreNMS\Exceptions\InvalidNameException + */ + public function getSettingValue(string $name) + { + $definition = $this->configDefinition(false); + + foreach ($definition as $entry) { + if ($entry['name'] == $name) { + return $entry['value']; + } + } + + throw new InvalidNameException("Poller group setting named \"$name\" is invalid"); + } + /** * Get the frontend config definition for this poller * - * @param \Illuminate\Support\Collection $groups optionally supply full list of poller groups to avoid fetching multiple times + * @param \Illuminate\Support\Collection|bool|null $groups optionally supply full list of poller groups to avoid fetching multiple times * @return array[] */ public function configDefinition($groups = null) { - if (empty($groups)) { + if ($groups === null || $groups === true) { $groups = PollerGroup::list(); } diff --git a/config/librenms.php b/config/librenms.php index 33873d2dc5..bcaf953544 100644 --- a/config/librenms.php +++ b/config/librenms.php @@ -43,4 +43,14 @@ return [ 'install' => env('INSTALL', false), + /* + |-------------------------------------------------------------------------- + | NODE ID + |-------------------------------------------------------------------------- + | + | Unique value to identify this node. Primarily used for distributed polling. + */ + + 'node_id' => env('NODE_ID'), + ];