mirror/librenms-librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-10-07 16:52:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Poll device job (#16306)

Browse Source 
* Split out single device polling to a Laravel Job
Probably totally broken :)

* Add dispatch option to device:poll

* Fix up submodules

* Add missing logger parameter

* Apply fixes from StyleCI

* Update module format in test helper

* Apply fixes from StyleCI

* Use Log facade to match other code

---------

Co-authored-by: Tony Murray <murrant@users.noreply.github.com>
This commit is contained in:
Tony Murray
2024-08-25 19:56:50 -05:00
committed by GitHub
parent 2face4bc4a
commit db5fdf705d
5 changed files with 354 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

322
LibreNMS/Poller.php
View File

@ -1,322 +0,0 @@
<?php
/**
* Poller.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* @link https://www.librenms.org
*
* @copyright 2021 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS;
use App\Events\DevicePolled;
use App\Events\PollingDevice;
use App\Models\Device;
use App\Models\Eventlog;
use App\Polling\Measure\Measurement;
use App\Polling\Measure\MeasurementManager;
use Carbon\Carbon;
use Illuminate\Support\Str;
use LibreNMS\Enum\Severity;
use LibreNMS\Polling\ConnectivityHelper;
use LibreNMS\Polling\Result;
use LibreNMS\RRD\RrdDefinition;
use LibreNMS\Util\Debug;
use LibreNMS\Util\Dns;
use LibreNMS\Util\Module;
use LibreNMS\Util\Version;
use Psr\Log\LoggerInterface;
use Throwable;
class Poller
{
/** @var string */
private $device_spec;
/** @var array */
private $module_override;
/**
* @var Device
*/
private $device;
/**
* @var array
*/
private $deviceArray;
/**
* @var \LibreNMS\OS|\LibreNMS\OS\Generic
*/
private $os;
/**
* @var LoggerInterface
*/
private $logger;
public function __construct(string $device_spec, array $module_override, LoggerInterface $logger)
{
$this->device_spec = $device_spec;
$this->module_override = $module_override;
$this->logger = $logger;
$this->parseModules();
}
public function poll(): Result
{
$results = new Result;
$this->printHeader();
if (Debug::isEnabled() && ! defined('PHPUNIT_RUNNING')) {
\LibreNMS\Util\OS::updateCache(true); // Force update of OS Cache
}
$this->logger->info("Starting polling run:\n");
foreach (Device::whereDeviceSpec($this->device_spec)->pluck('device_id') as $device_id) {
$results->markAttempted();
$this->initDevice($device_id);
PollingDevice::dispatch($this->device);
$this->os = OS::make($this->deviceArray);
$measurement = Measurement::start('poll');
$measurement->manager()->checkpoint(); // don't count previous stats
$helper = new ConnectivityHelper($this->device);
$helper->saveMetrics();
$helper->isUp(); // check and save status
$this->pollModules();
$measurement->end();
// if modules are not overridden, record performance
if (empty($this->module_override)) {
if ($this->device->status) {
$this->recordPerformance($measurement);
}
if ($helper->canPing()) {
$this->os->enableGraph('ping_perf');
}
$this->os->persistGraphs($this->device->status); // save graphs but don't delete any if device is down
$this->logger->info(sprintf("Enabled graphs (%s): %s\n\n",
$this->device->graphs->count(),
$this->device->graphs->pluck('graph')->implode(' ')
));
}
// finalize the device poll
$this->device->save();
$results->markCompleted($this->device->status);
DevicePolled::dispatch($this->device);
$this->logger->info(sprintf("\n>>> Polled %s (%s) in %0.3f seconds <<<",
$this->device->displayName(),
$this->device->device_id,
$measurement->getDuration()));
\Log::channel('single')->alert(sprintf('INFO: device:poll %s (%s) polled in %0.3fs',
$this->device->hostname,
$this->device->device_id,
$measurement->getDuration()));
// check if the poll took too long and log an event
if ($measurement->getDuration() > Config::get('rrd.step')) {
Eventlog::log('Polling took longer than ' . round(Config::get('rrd.step') / 60, 2) .
' minutes! This will cause gaps in graphs.', $this->device, 'system', Severity::Error);
}
}
return $results;
}
private function pollModules(): void
{
// update $device array status
$this->deviceArray['status'] = $this->device->status;
$this->deviceArray['status_reason'] = $this->device->status_reason;
// import legacy garbage
include_once base_path('includes/functions.php');
include_once base_path('includes/common.php');
include_once base_path('includes/polling/functions.inc.php');
include_once base_path('includes/snmp.inc.php');
$datastore = app('Datastore');
foreach (array_keys(Config::get('poller_modules')) as $module) {
$module_status = Module::pollingStatus($module, $this->device, $this->isModuleManuallyEnabled($module));
$should_poll = false;
$start_memory = memory_get_usage();
$module_start = microtime(true);
try {
$instance = Module::fromName($module);
$should_poll = $instance->shouldPoll($this->os, $module_status);
if ($should_poll) {
$this->logger->info("#### Load poller module $module ####\n");
$this->logger->debug($module_status);
$instance->poll($this->os, $datastore);
}
} catch (Throwable $e) {
// isolate module exceptions so they don't disrupt the polling process
$this->logger->error("%rError polling $module module for {$this->device->hostname}.%n $e", ['color' => true]);
Eventlog::log("Error polling $module module. Check log file for more details.", $this->device, 'poller', Severity::Error);
report($e);
}
if ($should_poll) {
$this->logger->info('');
app(MeasurementManager::class)->printChangedStats();
$this->saveModulePerformance($module, $module_start, $start_memory);
$this->logger->info("#### Unload poller module $module ####\n");
}
}
}
private function saveModulePerformance(string $module, float $start_time, int $start_memory): void
{
$module_time = microtime(true) - $start_time;
$module_mem = (memory_get_usage() - $start_memory);
$this->logger->info(sprintf(">> Runtime for poller module '%s': %.4f seconds with %s bytes", $module, $module_time, $module_mem));
app('Datastore')->put($this->deviceArray, 'poller-perf', [
'module' => $module,
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
'rrd_name' => ['poller-perf', $module],
], [
'poller' => $module_time,
]);
$this->os->enableGraph('poller_modules_perf');
}
private function isModuleManuallyEnabled(string $module): ?bool
{
if (empty($this->module_override)) {
return null;
}
foreach ($this->module_override as $override) {
[$override_module] = explode('/', $override);
if ($module == $override_module) {
return true;
}
}
return false;
}
private function initDevice(int $device_id): void
{
\DeviceCache::setPrimary($device_id);
$this->device = \DeviceCache::getPrimary();
$this->device->ip = $this->device->overwrite_ip ?: Dns::lookupIp($this->device) ?: $this->device->ip;
$this->deviceArray = $this->device->toArray();
if ($os_group = Config::get("os.{$this->device->os}.group")) {
$this->deviceArray['os_group'] = $os_group;
}
$this->printDeviceInfo($os_group);
$this->initRrdDirectory();
}
private function initRrdDirectory(): void
{
$host_rrd = \Rrd::name($this->device->hostname, '', '');
if (Config::get('rrd.enable', true) && ! is_dir($host_rrd)) {
try {
mkdir($host_rrd);
$this->logger->info("Created directory : $host_rrd");
} catch (\ErrorException $e) {
Eventlog::log("Failed to create rrd directory: $host_rrd", $this->device);
$this->logger->error($e);
}
}
}
private function parseModules(): void
{
foreach ($this->module_override as $index => $module) {
// parse submodules (only supported by some modules)
if (Str::contains($module, '/')) {
[$module, $submodule] = explode('/', $module, 2);
$existing_submodules = Config::get("poller_submodules.$module", []);
$existing_submodules[] = $submodule;
Config::set("poller_submodules.$module", $existing_submodules);
}
if (! Module::exists($module) && ! Module::legacyPollingExists($module)) {
unset($this->module_override[$index]);
continue;
}
Config::set("poller_modules.$module", 1);
}
$this->printModules();
}
private function printDeviceInfo(?string $group): void
{
$this->logger->info(sprintf(<<<'EOH'
Hostname: %s %s
ID: %s
OS: %s
IP: %s
EOH, $this->device->hostname, $group ? " ($group)" : '', $this->device->device_id, $this->device->os, $this->device->ip));
}
private function printModules(): void
{
$modules = array_map(function ($module) {
$submodules = Config::get("poller_submodules.$module");
return $module . ($submodules ? '(' . implode(',', $submodules) . ')' : '');
}, array_keys(Config::get('poller_modules', [])));
$this->logger->debug('Override poller modules: ' . implode(', ', $modules));
}
private function printHeader(): void
{
if (Debug::isEnabled() || Debug::isVerbose()) {
$this->logger->info(Version::get()->header());
}
}
private function recordPerformance(Measurement $measurement): void
{
$measurement->manager()->record('device', $measurement);
$this->device->last_polled = Carbon::now();
$this->device->last_polled_timetaken = $measurement->getDuration();
app('Datastore')->put($this->deviceArray, 'poller-perf', [
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
'module' => 'ALL',
], [
'poller' => $this->device->last_polled_timetaken,
]);
$this->os->enableGraph('poller_perf');
}
}

25
LibreNMS/Util/Module.php
View File

@ -34,7 +34,11 @@ class Module
{
public static function exists(string $module_name): bool
{
return class_exists(StringHelpers::toClass($module_name, '\\LibreNMS\\Modules\\'));
if (class_exists(StringHelpers::toClass($module_name, '\\LibreNMS\\Modules\\'))) {
return true;
}
return Config::has('discovery_modules.' . $module_name) || Config::has('poller_modules.' . $module_name);
}
public static function fromName(string $module_name): \LibreNMS\Interfaces\Module
@ -57,10 +61,27 @@ class Module
public static function pollingStatus(string $module_name, Device $device, ?bool $manual = null): ModuleStatus
{
return new ModuleStatus(
Config::get("poller_modules.$module_name"),
Config::get("poller_modules.$module_name", false),
Config::get("os.{$device->os}.poller_modules.$module_name"),
$device->getAttrib("poll_$module_name"),
$manual,
);
}
public static function parseUserOverrides(array $overrides): array
{
$modules = [];
foreach ($overrides as $index => $module) {
// parse submodules (only supported by some modules)
if (str_contains($module, '/')) {
[$module, $submodule] = explode('/', $module, 2);
$modules[$module][] = $submodule;
} elseif (self::exists($module)) {
$modules[$module] = true;
}
}
return $modules;
}
}

27
LibreNMS/Util/ModuleTestHelper.php
View File

@ -26,6 +26,7 @@
namespace LibreNMS\Util;
use App\Actions\Device\ValidateDeviceAndCreate;
use App\Jobs\PollDevice;
use App\Models\Device;
use DeviceCache;
use Illuminate\Database\Eloquent\Model;
@ -35,7 +36,6 @@ use LibreNMS\Config;
use LibreNMS\Data\Source\SnmpResponse;
use LibreNMS\Exceptions\FileNotFoundException;
use LibreNMS\Exceptions\InvalidModuleException;
use LibreNMS\Poller;
class ModuleTestHelper
{
@ -178,8 +178,7 @@ class ModuleTestHelper
Debug::set();
Debug::setVerbose();
discover_device($device, $this->parseArgs('discovery'));
$poller = app(Poller::class, ['device_spec' => $device_id, 'module_override' => $this->modules]);
$poller->poll();
(new PollDevice($device_id, $this->modules))->handle();
Debug::set($save_debug);
Debug::setVerbose($save_vdebug);
$collection_output = ob_get_contents();
@ -312,7 +311,7 @@ class ModuleTestHelper
* Probably needs to be more robust
*
* @param array $modules
* @return array
* @return array<string, bool|string[]>
*
* @throws InvalidModuleException
*/
@ -320,17 +319,22 @@ class ModuleTestHelper
{
// generate a full list of modules
$full_list = [];
foreach ($modules as $module) {
foreach ($modules as $index => $module) {
$module = is_string($index) ? $index : $module;
// only allow valid modules
if (! (Config::has("poller_modules.$module") || Config::has("discovery_modules.$module"))) {
if (! Module::exists($module)) {
throw new InvalidModuleException("Invalid module name: $module");
}
$full_list = array_merge($full_list, Module::fromName($module)->dependencies());
$full_list[] = $module;
foreach (Module::fromName($module)->dependencies() as $dependency) {
$full_list[$dependency] = true;
}
$full_list[$module] = true;
}
return array_unique($full_list);
return $full_list;
}
private function parseArgs($type)
@ -339,7 +343,7 @@ class ModuleTestHelper
return false;
}
return parse_modules($type, ['m' => implode(',', $this->modules)]);
return parse_modules($type, ['m' => implode(',', array_keys($this->modules))]);
}
private function qPrint($var)
@ -627,8 +631,7 @@ class ModuleTestHelper
ob_start();
\Log::setDefaultDriver('console');
$poller = app(Poller::class, ['device_spec' => $device_id, 'module_override' => $this->modules]);
$poller->poll();
(new PollDevice($device_id, $this->modules))->handle();
$this->poller_output = ob_get_contents();
if ($this->quiet) {

73
app/Console/Commands/DevicePoll.php
View File

@ -3,16 +3,24 @@
namespace App\Console\Commands;
use App\Console\LnmsCommand;
use App\Events\DevicePolled;
use App\Jobs\PollDevice;
use App\Models\Device;
use App\Polling\Measure\MeasurementManager;
use Illuminate\Database\QueryException;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
use LibreNMS\Config;
use LibreNMS\Poller;
use LibreNMS\Polling\Result;
use LibreNMS\Util\Module;
use LibreNMS\Util\Version;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
class DevicePoll extends LnmsCommand
{
protected $name = 'device:poll';
private ?int $current_device_id = null;
/**
* Create a new command instance.
@ -25,10 +33,15 @@ class DevicePoll extends LnmsCommand
$this->addArgument('device spec', InputArgument::REQUIRED);
$this->addOption('modules', 'm', InputOption::VALUE_REQUIRED);
$this->addOption('no-data', 'x', InputOption::VALUE_NONE);
$this->addOption('dispatch', 'd', InputOption::VALUE_NONE);
}
public function handle(MeasurementManager $measurements): int
{
if ($this->option('dispatch')) {
return $this->dispatchWork();
}
$this->configureOutputOptions();
if ($this->option('no-data')) {
@ -40,9 +53,30 @@ class DevicePoll extends LnmsCommand
}
try {
/** @var \LibreNMS\Poller $poller */
$poller = app(Poller::class, ['device_spec' => $this->argument('device spec'), 'module_override' => explode(',', $this->option('modules') ?? '')]);
$result = $poller->poll();
if ($this->getOutput()->isVerbose()) {
Log::debug(Version::get()->header());
\LibreNMS\Util\OS::updateCache(true); // Force update of OS Cache
}
$module_overrides = Module::parseUserOverrides(explode(',', $this->option('modules') ?? ''));
$this->printModules($module_overrides);
$result = new Result;
$this->line("Starting polling run:\n");
// listen for the device polled events to mark the device completed
Event::listen(function (DevicePolled $event) use ($result) {
if ($event->device->device_id == $this->current_device_id) {
$result->markCompleted($event->device->status);
}
});
foreach (Device::whereDeviceSpec($this->argument('device spec'))->pluck('device_id') as $device_id) {
$this->current_device_id = $device_id;
$result->markAttempted();
PollDevice::dispatchSync($device_id, $module_overrides);
}
if ($result->hasAnyCompleted()) {
if (! $this->output->isQuiet()) {
@ -92,4 +126,35 @@ class DevicePoll extends LnmsCommand
return 1; // failed to poll
}
private function dispatchWork()
{
\Log::setDefaultDriver('stack');
$module_overrides = Module::parseUserOverrides(explode(',', $this->option('modules') ?? ''));
$devices = Device::whereDeviceSpec($this->argument('device spec'))->pluck('device_id');
if (\config('queue.default') == 'sync') {
$this->error('Queue driver is sync, work will run in process.');
sleep(1);
}
foreach ($devices as $device_id) {
PollDevice::dispatch($device_id, $module_overrides);
}
$this->line('Submitted work for ' . $devices->count() . ' devices');
return 0;
}
private function printModules(array $module_overrides): void
{
if (! empty($module_overrides)) {
$modules = array_map(function ($module, $status) {
return $module . (is_array($status) ? '(' . implode(',', $status) . ')' : '');
}, array_keys($module_overrides), array_values($module_overrides));
Log::debug('Override poller modules: ' . implode(', ', $modules));
}
}
}

247
app/Jobs/PollDevice.php Normal file
View File

@ -0,0 +1,247 @@
<?php
namespace App\Jobs;
use App\Events\DevicePolled;
use App\Events\PollingDevice;
use App\Models\Eventlog;
use App\Polling\Measure\Measurement;
use App\Polling\Measure\MeasurementManager;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use LibreNMS\Config;
use LibreNMS\Enum\Severity;
use LibreNMS\OS;
use LibreNMS\Polling\ConnectivityHelper;
use LibreNMS\RRD\RrdDefinition;
use LibreNMS\Util\Dns;
use LibreNMS\Util\Module;
use Throwable;
class PollDevice implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private ?\App\Models\Device $device = null;
private ?array $deviceArray = null;
/**
* @var \LibreNMS\OS|\LibreNMS\OS\Generic
*/
private $os;
/**
* @param int $device_id
* @param array<string, bool|string[]> $module_overrides
*/
public function __construct(
public int $device_id,
public array $module_overrides = [],
) {
}
/**
* Execute the job.
*/
public function handle()
{
$this->initDevice();
PollingDevice::dispatch($this->device);
$this->os = OS::make($this->deviceArray);
$measurement = Measurement::start('poll');
$measurement->manager()->checkpoint(); // don't count previous stats
$helper = new ConnectivityHelper($this->device);
$helper->saveMetrics();
$helper->isUp(); // check and save status
$this->pollModules();
$measurement->end();
// if modules are not overridden, record performance
if (empty($this->modules)) {
if ($this->device->status) {
$this->recordPerformance($measurement);
}
if ($helper->canPing()) {
$this->os->enableGraph('ping_perf');
}
$this->os->persistGraphs($this->device->status); // save graphs but don't delete any if device is down
Log::info(sprintf("Enabled graphs (%s): %s\n\n",
$this->device->graphs->count(),
$this->device->graphs->pluck('graph')->implode(' ')
));
}
// finalize the device poll
$this->device->save();
Log::info(sprintf("\n>>> Polled %s (%s) in %0.3f seconds <<<",
$this->device->displayName(),
$this->device->device_id,
$measurement->getDuration()));
// add log file line, this is used by the simple python dispatcher watchdog
Log::channel('single')->alert(sprintf('INFO: device:poll %s (%s) polled in %0.3fs',
$this->device->hostname,
$this->device->device_id,
$measurement->getDuration()));
// check if the poll took too long and log an event
if ($measurement->getDuration() > Config::get('rrd.step')) {
Eventlog::log('Polling took longer than ' . round(Config::get('rrd.step') / 60, 2) .
' minutes! This will cause gaps in graphs.', $this->device, 'system', Severity::Error);
}
DevicePolled::dispatch($this->device);
}
private function pollModules(): void
{
// update $device array status
$this->deviceArray['status'] = $this->device->status;
$this->deviceArray['status_reason'] = $this->device->status_reason;
// import legacy garbage
include_once base_path('includes/functions.php');
include_once base_path('includes/common.php');
include_once base_path('includes/polling/functions.inc.php');
include_once base_path('includes/snmp.inc.php');
$datastore = app('Datastore');
foreach ($this->getModules() as $module => $status) {
$module_status = Module::pollingStatus($module, $this->device, $this->isModuleManuallyEnabled($module));
$should_poll = false;
$start_memory = memory_get_usage();
$module_start = microtime(true);
try {
$instance = Module::fromName($module);
$should_poll = $instance->shouldPoll($this->os, $module_status);
if ($should_poll) {
Log::info("#### Load poller module $module ####\n");
Log::debug($module_status);
if (is_array($status)) {
Config::set('poller_submodules.' . $module, $status);
}
$instance->poll($this->os, $datastore);
}
} catch (Throwable $e) {
// isolate module exceptions so they don't disrupt the polling process
Log::error("%rError polling $module module for {$this->device->hostname}.%n $e", ['color' => true]);
Eventlog::log("Error polling $module module. Check log file for more details.", $this->device, 'poller', Severity::Error);
report($e);
}
if ($should_poll) {
Log::info('');
app(MeasurementManager::class)->printChangedStats();
$this->saveModulePerformance($module, $module_start, $start_memory);
Log::info("#### Unload poller module $module ####\n");
}
}
}
private function saveModulePerformance(string $module, float $start_time, int $start_memory): void
{
$module_time = microtime(true) - $start_time;
$module_mem = (memory_get_usage() - $start_memory);
Log::info(sprintf(">> Runtime for poller module '%s': %.4f seconds with %s bytes", $module, $module_time, $module_mem));
app('Datastore')->put($this->deviceArray, 'poller-perf', [
'module' => $module,
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
'rrd_name' => ['poller-perf', $module],
], [
'poller' => $module_time,
]);
$this->os->enableGraph('poller_modules_perf');
}
private function initDevice(): void
{
\DeviceCache::setPrimary($this->device_id);
$this->device = \DeviceCache::getPrimary();
$this->device->ip = $this->device->overwrite_ip ?: Dns::lookupIp($this->device) ?: $this->device->ip;
$this->deviceArray = $this->device->toArray();
if ($os_group = Config::get("os.{$this->device->os}.group")) {
$this->deviceArray['os_group'] = $os_group;
}
$this->printDeviceInfo($os_group);
$this->initRrdDirectory();
}
private function initRrdDirectory(): void
{
$host_rrd = \Rrd::name($this->device->hostname, '', '');
if (Config::get('rrd.enable', true) && ! is_dir($host_rrd)) {
try {
mkdir($host_rrd);
Log::info("Created directory : $host_rrd");
} catch (\ErrorException $e) {
Eventlog::log("Failed to create rrd directory: $host_rrd", $this->device);
Log::error($e);
}
}
}
private function printDeviceInfo(?string $group): void
{
Log::info(sprintf(<<<'EOH'
Hostname: %s %s
ID: %s
OS: %s
IP: %s
EOH, $this->device->hostname, $group ? " ($group)" : '', $this->device->device_id, $this->device->os, $this->device->ip));
}
private function recordPerformance(Measurement $measurement): void
{
$measurement->manager()->record('device', $measurement);
$this->device->last_polled = Carbon::now();
$this->device->last_polled_timetaken = $measurement->getDuration();
app('Datastore')->put($this->deviceArray, 'poller-perf', [
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
'module' => 'ALL',
], [
'poller' => $this->device->last_polled_timetaken,
]);
$this->os->enableGraph('poller_perf');
}
private function getModules(): array
{
if (! empty($this->module_overrides)) {
return $this->module_overrides;
}
return \LibreNMS\Config::get('poller_modules', []);
}
private function isModuleManuallyEnabled(string $module): ?bool
{
if (empty($this->module_overrides)) {
return null;
}
return isset($this->module_overrides[$module]);
}
}