mirror of
https://github.com/librenms/librenms.git
synced 2024-10-07 16:52:45 +00:00
Poll device job (#16306)
* Split out single device polling to a Laravel Job Probably totally broken :) * Add dispatch option to device:poll * Fix up submodules * Add missing logger parameter * Apply fixes from StyleCI * Update module format in test helper * Apply fixes from StyleCI * Use Log facade to match other code --------- Co-authored-by: Tony Murray <murrant@users.noreply.github.com>
This commit is contained in:
@@ -1,322 +0,0 @@
|
|||||||
<?php
|
|
||||||
/**
|
|
||||||
* Poller.php
|
|
||||||
*
|
|
||||||
* -Description-
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
* @link https://www.librenms.org
|
|
||||||
*
|
|
||||||
* @copyright 2021 Tony Murray
|
|
||||||
* @author Tony Murray <murraytony@gmail.com>
|
|
||||||
*/
|
|
||||||
|
|
||||||
namespace LibreNMS;
|
|
||||||
|
|
||||||
use App\Events\DevicePolled;
|
|
||||||
use App\Events\PollingDevice;
|
|
||||||
use App\Models\Device;
|
|
||||||
use App\Models\Eventlog;
|
|
||||||
use App\Polling\Measure\Measurement;
|
|
||||||
use App\Polling\Measure\MeasurementManager;
|
|
||||||
use Carbon\Carbon;
|
|
||||||
use Illuminate\Support\Str;
|
|
||||||
use LibreNMS\Enum\Severity;
|
|
||||||
use LibreNMS\Polling\ConnectivityHelper;
|
|
||||||
use LibreNMS\Polling\Result;
|
|
||||||
use LibreNMS\RRD\RrdDefinition;
|
|
||||||
use LibreNMS\Util\Debug;
|
|
||||||
use LibreNMS\Util\Dns;
|
|
||||||
use LibreNMS\Util\Module;
|
|
||||||
use LibreNMS\Util\Version;
|
|
||||||
use Psr\Log\LoggerInterface;
|
|
||||||
use Throwable;
|
|
||||||
|
|
||||||
class Poller
|
|
||||||
{
|
|
||||||
/** @var string */
|
|
||||||
private $device_spec;
|
|
||||||
/** @var array */
|
|
||||||
private $module_override;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var Device
|
|
||||||
*/
|
|
||||||
private $device;
|
|
||||||
/**
|
|
||||||
* @var array
|
|
||||||
*/
|
|
||||||
private $deviceArray;
|
|
||||||
/**
|
|
||||||
* @var \LibreNMS\OS|\LibreNMS\OS\Generic
|
|
||||||
*/
|
|
||||||
private $os;
|
|
||||||
/**
|
|
||||||
* @var LoggerInterface
|
|
||||||
*/
|
|
||||||
private $logger;
|
|
||||||
|
|
||||||
public function __construct(string $device_spec, array $module_override, LoggerInterface $logger)
|
|
||||||
{
|
|
||||||
$this->device_spec = $device_spec;
|
|
||||||
$this->module_override = $module_override;
|
|
||||||
$this->logger = $logger;
|
|
||||||
$this->parseModules();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function poll(): Result
|
|
||||||
{
|
|
||||||
$results = new Result;
|
|
||||||
$this->printHeader();
|
|
||||||
|
|
||||||
if (Debug::isEnabled() && ! defined('PHPUNIT_RUNNING')) {
|
|
||||||
\LibreNMS\Util\OS::updateCache(true); // Force update of OS Cache
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->logger->info("Starting polling run:\n");
|
|
||||||
|
|
||||||
foreach (Device::whereDeviceSpec($this->device_spec)->pluck('device_id') as $device_id) {
|
|
||||||
$results->markAttempted();
|
|
||||||
$this->initDevice($device_id);
|
|
||||||
PollingDevice::dispatch($this->device);
|
|
||||||
$this->os = OS::make($this->deviceArray);
|
|
||||||
|
|
||||||
$measurement = Measurement::start('poll');
|
|
||||||
$measurement->manager()->checkpoint(); // don't count previous stats
|
|
||||||
|
|
||||||
$helper = new ConnectivityHelper($this->device);
|
|
||||||
$helper->saveMetrics();
|
|
||||||
$helper->isUp(); // check and save status
|
|
||||||
|
|
||||||
$this->pollModules();
|
|
||||||
|
|
||||||
$measurement->end();
|
|
||||||
|
|
||||||
// if modules are not overridden, record performance
|
|
||||||
if (empty($this->module_override)) {
|
|
||||||
if ($this->device->status) {
|
|
||||||
$this->recordPerformance($measurement);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($helper->canPing()) {
|
|
||||||
$this->os->enableGraph('ping_perf');
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->os->persistGraphs($this->device->status); // save graphs but don't delete any if device is down
|
|
||||||
$this->logger->info(sprintf("Enabled graphs (%s): %s\n\n",
|
|
||||||
$this->device->graphs->count(),
|
|
||||||
$this->device->graphs->pluck('graph')->implode(' ')
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
// finalize the device poll
|
|
||||||
$this->device->save();
|
|
||||||
$results->markCompleted($this->device->status);
|
|
||||||
DevicePolled::dispatch($this->device);
|
|
||||||
|
|
||||||
$this->logger->info(sprintf("\n>>> Polled %s (%s) in %0.3f seconds <<<",
|
|
||||||
$this->device->displayName(),
|
|
||||||
$this->device->device_id,
|
|
||||||
$measurement->getDuration()));
|
|
||||||
\Log::channel('single')->alert(sprintf('INFO: device:poll %s (%s) polled in %0.3fs',
|
|
||||||
$this->device->hostname,
|
|
||||||
$this->device->device_id,
|
|
||||||
$measurement->getDuration()));
|
|
||||||
|
|
||||||
// check if the poll took too long and log an event
|
|
||||||
if ($measurement->getDuration() > Config::get('rrd.step')) {
|
|
||||||
Eventlog::log('Polling took longer than ' . round(Config::get('rrd.step') / 60, 2) .
|
|
||||||
' minutes! This will cause gaps in graphs.', $this->device, 'system', Severity::Error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $results;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function pollModules(): void
|
|
||||||
{
|
|
||||||
// update $device array status
|
|
||||||
$this->deviceArray['status'] = $this->device->status;
|
|
||||||
$this->deviceArray['status_reason'] = $this->device->status_reason;
|
|
||||||
|
|
||||||
// import legacy garbage
|
|
||||||
include_once base_path('includes/functions.php');
|
|
||||||
include_once base_path('includes/common.php');
|
|
||||||
include_once base_path('includes/polling/functions.inc.php');
|
|
||||||
include_once base_path('includes/snmp.inc.php');
|
|
||||||
|
|
||||||
$datastore = app('Datastore');
|
|
||||||
|
|
||||||
foreach (array_keys(Config::get('poller_modules')) as $module) {
|
|
||||||
$module_status = Module::pollingStatus($module, $this->device, $this->isModuleManuallyEnabled($module));
|
|
||||||
$should_poll = false;
|
|
||||||
$start_memory = memory_get_usage();
|
|
||||||
$module_start = microtime(true);
|
|
||||||
|
|
||||||
try {
|
|
||||||
$instance = Module::fromName($module);
|
|
||||||
$should_poll = $instance->shouldPoll($this->os, $module_status);
|
|
||||||
|
|
||||||
if ($should_poll) {
|
|
||||||
$this->logger->info("#### Load poller module $module ####\n");
|
|
||||||
$this->logger->debug($module_status);
|
|
||||||
|
|
||||||
$instance->poll($this->os, $datastore);
|
|
||||||
}
|
|
||||||
} catch (Throwable $e) {
|
|
||||||
// isolate module exceptions so they don't disrupt the polling process
|
|
||||||
$this->logger->error("%rError polling $module module for {$this->device->hostname}.%n $e", ['color' => true]);
|
|
||||||
Eventlog::log("Error polling $module module. Check log file for more details.", $this->device, 'poller', Severity::Error);
|
|
||||||
report($e);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($should_poll) {
|
|
||||||
$this->logger->info('');
|
|
||||||
app(MeasurementManager::class)->printChangedStats();
|
|
||||||
$this->saveModulePerformance($module, $module_start, $start_memory);
|
|
||||||
$this->logger->info("#### Unload poller module $module ####\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function saveModulePerformance(string $module, float $start_time, int $start_memory): void
|
|
||||||
{
|
|
||||||
$module_time = microtime(true) - $start_time;
|
|
||||||
$module_mem = (memory_get_usage() - $start_memory);
|
|
||||||
|
|
||||||
$this->logger->info(sprintf(">> Runtime for poller module '%s': %.4f seconds with %s bytes", $module, $module_time, $module_mem));
|
|
||||||
|
|
||||||
app('Datastore')->put($this->deviceArray, 'poller-perf', [
|
|
||||||
'module' => $module,
|
|
||||||
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
|
|
||||||
'rrd_name' => ['poller-perf', $module],
|
|
||||||
], [
|
|
||||||
'poller' => $module_time,
|
|
||||||
]);
|
|
||||||
$this->os->enableGraph('poller_modules_perf');
|
|
||||||
}
|
|
||||||
|
|
||||||
private function isModuleManuallyEnabled(string $module): ?bool
|
|
||||||
{
|
|
||||||
if (empty($this->module_override)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($this->module_override as $override) {
|
|
||||||
[$override_module] = explode('/', $override);
|
|
||||||
if ($module == $override_module) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function initDevice(int $device_id): void
|
|
||||||
{
|
|
||||||
\DeviceCache::setPrimary($device_id);
|
|
||||||
$this->device = \DeviceCache::getPrimary();
|
|
||||||
$this->device->ip = $this->device->overwrite_ip ?: Dns::lookupIp($this->device) ?: $this->device->ip;
|
|
||||||
|
|
||||||
$this->deviceArray = $this->device->toArray();
|
|
||||||
if ($os_group = Config::get("os.{$this->device->os}.group")) {
|
|
||||||
$this->deviceArray['os_group'] = $os_group;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->printDeviceInfo($os_group);
|
|
||||||
$this->initRrdDirectory();
|
|
||||||
}
|
|
||||||
|
|
||||||
private function initRrdDirectory(): void
|
|
||||||
{
|
|
||||||
$host_rrd = \Rrd::name($this->device->hostname, '', '');
|
|
||||||
if (Config::get('rrd.enable', true) && ! is_dir($host_rrd)) {
|
|
||||||
try {
|
|
||||||
mkdir($host_rrd);
|
|
||||||
$this->logger->info("Created directory : $host_rrd");
|
|
||||||
} catch (\ErrorException $e) {
|
|
||||||
Eventlog::log("Failed to create rrd directory: $host_rrd", $this->device);
|
|
||||||
$this->logger->error($e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function parseModules(): void
|
|
||||||
{
|
|
||||||
foreach ($this->module_override as $index => $module) {
|
|
||||||
// parse submodules (only supported by some modules)
|
|
||||||
if (Str::contains($module, '/')) {
|
|
||||||
[$module, $submodule] = explode('/', $module, 2);
|
|
||||||
$existing_submodules = Config::get("poller_submodules.$module", []);
|
|
||||||
$existing_submodules[] = $submodule;
|
|
||||||
Config::set("poller_submodules.$module", $existing_submodules);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (! Module::exists($module) && ! Module::legacyPollingExists($module)) {
|
|
||||||
unset($this->module_override[$index]);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
Config::set("poller_modules.$module", 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->printModules();
|
|
||||||
}
|
|
||||||
|
|
||||||
private function printDeviceInfo(?string $group): void
|
|
||||||
{
|
|
||||||
$this->logger->info(sprintf(<<<'EOH'
|
|
||||||
Hostname: %s %s
|
|
||||||
ID: %s
|
|
||||||
OS: %s
|
|
||||||
IP: %s
|
|
||||||
|
|
||||||
EOH, $this->device->hostname, $group ? " ($group)" : '', $this->device->device_id, $this->device->os, $this->device->ip));
|
|
||||||
}
|
|
||||||
|
|
||||||
private function printModules(): void
|
|
||||||
{
|
|
||||||
$modules = array_map(function ($module) {
|
|
||||||
$submodules = Config::get("poller_submodules.$module");
|
|
||||||
|
|
||||||
return $module . ($submodules ? '(' . implode(',', $submodules) . ')' : '');
|
|
||||||
}, array_keys(Config::get('poller_modules', [])));
|
|
||||||
|
|
||||||
$this->logger->debug('Override poller modules: ' . implode(', ', $modules));
|
|
||||||
}
|
|
||||||
|
|
||||||
private function printHeader(): void
|
|
||||||
{
|
|
||||||
if (Debug::isEnabled() || Debug::isVerbose()) {
|
|
||||||
$this->logger->info(Version::get()->header());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function recordPerformance(Measurement $measurement): void
|
|
||||||
{
|
|
||||||
$measurement->manager()->record('device', $measurement);
|
|
||||||
$this->device->last_polled = Carbon::now();
|
|
||||||
$this->device->last_polled_timetaken = $measurement->getDuration();
|
|
||||||
|
|
||||||
app('Datastore')->put($this->deviceArray, 'poller-perf', [
|
|
||||||
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
|
|
||||||
'module' => 'ALL',
|
|
||||||
], [
|
|
||||||
'poller' => $this->device->last_polled_timetaken,
|
|
||||||
]);
|
|
||||||
|
|
||||||
$this->os->enableGraph('poller_perf');
|
|
||||||
}
|
|
||||||
}
|
|
@@ -34,7 +34,11 @@ class Module
|
|||||||
{
|
{
|
||||||
public static function exists(string $module_name): bool
|
public static function exists(string $module_name): bool
|
||||||
{
|
{
|
||||||
return class_exists(StringHelpers::toClass($module_name, '\\LibreNMS\\Modules\\'));
|
if (class_exists(StringHelpers::toClass($module_name, '\\LibreNMS\\Modules\\'))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Config::has('discovery_modules.' . $module_name) || Config::has('poller_modules.' . $module_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function fromName(string $module_name): \LibreNMS\Interfaces\Module
|
public static function fromName(string $module_name): \LibreNMS\Interfaces\Module
|
||||||
@@ -57,10 +61,27 @@ class Module
|
|||||||
public static function pollingStatus(string $module_name, Device $device, ?bool $manual = null): ModuleStatus
|
public static function pollingStatus(string $module_name, Device $device, ?bool $manual = null): ModuleStatus
|
||||||
{
|
{
|
||||||
return new ModuleStatus(
|
return new ModuleStatus(
|
||||||
Config::get("poller_modules.$module_name"),
|
Config::get("poller_modules.$module_name", false),
|
||||||
Config::get("os.{$device->os}.poller_modules.$module_name"),
|
Config::get("os.{$device->os}.poller_modules.$module_name"),
|
||||||
$device->getAttrib("poll_$module_name"),
|
$device->getAttrib("poll_$module_name"),
|
||||||
$manual,
|
$manual,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static function parseUserOverrides(array $overrides): array
|
||||||
|
{
|
||||||
|
$modules = [];
|
||||||
|
|
||||||
|
foreach ($overrides as $index => $module) {
|
||||||
|
// parse submodules (only supported by some modules)
|
||||||
|
if (str_contains($module, '/')) {
|
||||||
|
[$module, $submodule] = explode('/', $module, 2);
|
||||||
|
$modules[$module][] = $submodule;
|
||||||
|
} elseif (self::exists($module)) {
|
||||||
|
$modules[$module] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $modules;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@@ -26,6 +26,7 @@
|
|||||||
namespace LibreNMS\Util;
|
namespace LibreNMS\Util;
|
||||||
|
|
||||||
use App\Actions\Device\ValidateDeviceAndCreate;
|
use App\Actions\Device\ValidateDeviceAndCreate;
|
||||||
|
use App\Jobs\PollDevice;
|
||||||
use App\Models\Device;
|
use App\Models\Device;
|
||||||
use DeviceCache;
|
use DeviceCache;
|
||||||
use Illuminate\Database\Eloquent\Model;
|
use Illuminate\Database\Eloquent\Model;
|
||||||
@@ -35,7 +36,6 @@ use LibreNMS\Config;
|
|||||||
use LibreNMS\Data\Source\SnmpResponse;
|
use LibreNMS\Data\Source\SnmpResponse;
|
||||||
use LibreNMS\Exceptions\FileNotFoundException;
|
use LibreNMS\Exceptions\FileNotFoundException;
|
||||||
use LibreNMS\Exceptions\InvalidModuleException;
|
use LibreNMS\Exceptions\InvalidModuleException;
|
||||||
use LibreNMS\Poller;
|
|
||||||
|
|
||||||
class ModuleTestHelper
|
class ModuleTestHelper
|
||||||
{
|
{
|
||||||
@@ -178,8 +178,7 @@ class ModuleTestHelper
|
|||||||
Debug::set();
|
Debug::set();
|
||||||
Debug::setVerbose();
|
Debug::setVerbose();
|
||||||
discover_device($device, $this->parseArgs('discovery'));
|
discover_device($device, $this->parseArgs('discovery'));
|
||||||
$poller = app(Poller::class, ['device_spec' => $device_id, 'module_override' => $this->modules]);
|
(new PollDevice($device_id, $this->modules))->handle();
|
||||||
$poller->poll();
|
|
||||||
Debug::set($save_debug);
|
Debug::set($save_debug);
|
||||||
Debug::setVerbose($save_vdebug);
|
Debug::setVerbose($save_vdebug);
|
||||||
$collection_output = ob_get_contents();
|
$collection_output = ob_get_contents();
|
||||||
@@ -312,7 +311,7 @@ class ModuleTestHelper
|
|||||||
* Probably needs to be more robust
|
* Probably needs to be more robust
|
||||||
*
|
*
|
||||||
* @param array $modules
|
* @param array $modules
|
||||||
* @return array
|
* @return array<string, bool|string[]>
|
||||||
*
|
*
|
||||||
* @throws InvalidModuleException
|
* @throws InvalidModuleException
|
||||||
*/
|
*/
|
||||||
@@ -320,17 +319,22 @@ class ModuleTestHelper
|
|||||||
{
|
{
|
||||||
// generate a full list of modules
|
// generate a full list of modules
|
||||||
$full_list = [];
|
$full_list = [];
|
||||||
foreach ($modules as $module) {
|
foreach ($modules as $index => $module) {
|
||||||
|
$module = is_string($index) ? $index : $module;
|
||||||
|
|
||||||
// only allow valid modules
|
// only allow valid modules
|
||||||
if (! (Config::has("poller_modules.$module") || Config::has("discovery_modules.$module"))) {
|
if (! Module::exists($module)) {
|
||||||
throw new InvalidModuleException("Invalid module name: $module");
|
throw new InvalidModuleException("Invalid module name: $module");
|
||||||
}
|
}
|
||||||
|
|
||||||
$full_list = array_merge($full_list, Module::fromName($module)->dependencies());
|
foreach (Module::fromName($module)->dependencies() as $dependency) {
|
||||||
$full_list[] = $module;
|
$full_list[$dependency] = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
$full_list[$module] = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return array_unique($full_list);
|
return $full_list;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function parseArgs($type)
|
private function parseArgs($type)
|
||||||
@@ -339,7 +343,7 @@ class ModuleTestHelper
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
return parse_modules($type, ['m' => implode(',', $this->modules)]);
|
return parse_modules($type, ['m' => implode(',', array_keys($this->modules))]);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function qPrint($var)
|
private function qPrint($var)
|
||||||
@@ -627,8 +631,7 @@ class ModuleTestHelper
|
|||||||
ob_start();
|
ob_start();
|
||||||
|
|
||||||
\Log::setDefaultDriver('console');
|
\Log::setDefaultDriver('console');
|
||||||
$poller = app(Poller::class, ['device_spec' => $device_id, 'module_override' => $this->modules]);
|
(new PollDevice($device_id, $this->modules))->handle();
|
||||||
$poller->poll();
|
|
||||||
|
|
||||||
$this->poller_output = ob_get_contents();
|
$this->poller_output = ob_get_contents();
|
||||||
if ($this->quiet) {
|
if ($this->quiet) {
|
||||||
|
@@ -3,16 +3,24 @@
|
|||||||
namespace App\Console\Commands;
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
use App\Console\LnmsCommand;
|
use App\Console\LnmsCommand;
|
||||||
|
use App\Events\DevicePolled;
|
||||||
|
use App\Jobs\PollDevice;
|
||||||
|
use App\Models\Device;
|
||||||
use App\Polling\Measure\MeasurementManager;
|
use App\Polling\Measure\MeasurementManager;
|
||||||
use Illuminate\Database\QueryException;
|
use Illuminate\Database\QueryException;
|
||||||
|
use Illuminate\Support\Facades\Event;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
use LibreNMS\Config;
|
use LibreNMS\Config;
|
||||||
use LibreNMS\Poller;
|
use LibreNMS\Polling\Result;
|
||||||
|
use LibreNMS\Util\Module;
|
||||||
|
use LibreNMS\Util\Version;
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
|
||||||
class DevicePoll extends LnmsCommand
|
class DevicePoll extends LnmsCommand
|
||||||
{
|
{
|
||||||
protected $name = 'device:poll';
|
protected $name = 'device:poll';
|
||||||
|
private ?int $current_device_id = null;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a new command instance.
|
* Create a new command instance.
|
||||||
@@ -25,10 +33,15 @@ class DevicePoll extends LnmsCommand
|
|||||||
$this->addArgument('device spec', InputArgument::REQUIRED);
|
$this->addArgument('device spec', InputArgument::REQUIRED);
|
||||||
$this->addOption('modules', 'm', InputOption::VALUE_REQUIRED);
|
$this->addOption('modules', 'm', InputOption::VALUE_REQUIRED);
|
||||||
$this->addOption('no-data', 'x', InputOption::VALUE_NONE);
|
$this->addOption('no-data', 'x', InputOption::VALUE_NONE);
|
||||||
|
$this->addOption('dispatch', 'd', InputOption::VALUE_NONE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function handle(MeasurementManager $measurements): int
|
public function handle(MeasurementManager $measurements): int
|
||||||
{
|
{
|
||||||
|
if ($this->option('dispatch')) {
|
||||||
|
return $this->dispatchWork();
|
||||||
|
}
|
||||||
|
|
||||||
$this->configureOutputOptions();
|
$this->configureOutputOptions();
|
||||||
|
|
||||||
if ($this->option('no-data')) {
|
if ($this->option('no-data')) {
|
||||||
@@ -40,9 +53,30 @@ class DevicePoll extends LnmsCommand
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
/** @var \LibreNMS\Poller $poller */
|
if ($this->getOutput()->isVerbose()) {
|
||||||
$poller = app(Poller::class, ['device_spec' => $this->argument('device spec'), 'module_override' => explode(',', $this->option('modules') ?? '')]);
|
Log::debug(Version::get()->header());
|
||||||
$result = $poller->poll();
|
\LibreNMS\Util\OS::updateCache(true); // Force update of OS Cache
|
||||||
|
}
|
||||||
|
|
||||||
|
$module_overrides = Module::parseUserOverrides(explode(',', $this->option('modules') ?? ''));
|
||||||
|
$this->printModules($module_overrides);
|
||||||
|
|
||||||
|
$result = new Result;
|
||||||
|
|
||||||
|
$this->line("Starting polling run:\n");
|
||||||
|
|
||||||
|
// listen for the device polled events to mark the device completed
|
||||||
|
Event::listen(function (DevicePolled $event) use ($result) {
|
||||||
|
if ($event->device->device_id == $this->current_device_id) {
|
||||||
|
$result->markCompleted($event->device->status);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
foreach (Device::whereDeviceSpec($this->argument('device spec'))->pluck('device_id') as $device_id) {
|
||||||
|
$this->current_device_id = $device_id;
|
||||||
|
$result->markAttempted();
|
||||||
|
PollDevice::dispatchSync($device_id, $module_overrides);
|
||||||
|
}
|
||||||
|
|
||||||
if ($result->hasAnyCompleted()) {
|
if ($result->hasAnyCompleted()) {
|
||||||
if (! $this->output->isQuiet()) {
|
if (! $this->output->isQuiet()) {
|
||||||
@@ -92,4 +126,35 @@ class DevicePoll extends LnmsCommand
|
|||||||
|
|
||||||
return 1; // failed to poll
|
return 1; // failed to poll
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function dispatchWork()
|
||||||
|
{
|
||||||
|
\Log::setDefaultDriver('stack');
|
||||||
|
$module_overrides = Module::parseUserOverrides(explode(',', $this->option('modules') ?? ''));
|
||||||
|
$devices = Device::whereDeviceSpec($this->argument('device spec'))->pluck('device_id');
|
||||||
|
|
||||||
|
if (\config('queue.default') == 'sync') {
|
||||||
|
$this->error('Queue driver is sync, work will run in process.');
|
||||||
|
sleep(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($devices as $device_id) {
|
||||||
|
PollDevice::dispatch($device_id, $module_overrides);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->line('Submitted work for ' . $devices->count() . ' devices');
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function printModules(array $module_overrides): void
|
||||||
|
{
|
||||||
|
if (! empty($module_overrides)) {
|
||||||
|
$modules = array_map(function ($module, $status) {
|
||||||
|
return $module . (is_array($status) ? '(' . implode(',', $status) . ')' : '');
|
||||||
|
}, array_keys($module_overrides), array_values($module_overrides));
|
||||||
|
|
||||||
|
Log::debug('Override poller modules: ' . implode(', ', $modules));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
247
app/Jobs/PollDevice.php
Normal file
247
app/Jobs/PollDevice.php
Normal file
@@ -0,0 +1,247 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Jobs;
|
||||||
|
|
||||||
|
use App\Events\DevicePolled;
|
||||||
|
use App\Events\PollingDevice;
|
||||||
|
use App\Models\Eventlog;
|
||||||
|
use App\Polling\Measure\Measurement;
|
||||||
|
use App\Polling\Measure\MeasurementManager;
|
||||||
|
use Carbon\Carbon;
|
||||||
|
use Illuminate\Bus\Queueable;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
|
use Illuminate\Foundation\Bus\Dispatchable;
|
||||||
|
use Illuminate\Queue\InteractsWithQueue;
|
||||||
|
use Illuminate\Queue\SerializesModels;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
|
use LibreNMS\Config;
|
||||||
|
use LibreNMS\Enum\Severity;
|
||||||
|
use LibreNMS\OS;
|
||||||
|
use LibreNMS\Polling\ConnectivityHelper;
|
||||||
|
use LibreNMS\RRD\RrdDefinition;
|
||||||
|
use LibreNMS\Util\Dns;
|
||||||
|
use LibreNMS\Util\Module;
|
||||||
|
use Throwable;
|
||||||
|
|
||||||
|
class PollDevice implements ShouldQueue
|
||||||
|
{
|
||||||
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||||
|
|
||||||
|
private ?\App\Models\Device $device = null;
|
||||||
|
private ?array $deviceArray = null;
|
||||||
|
/**
|
||||||
|
* @var \LibreNMS\OS|\LibreNMS\OS\Generic
|
||||||
|
*/
|
||||||
|
private $os;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int $device_id
|
||||||
|
* @param array<string, bool|string[]> $module_overrides
|
||||||
|
*/
|
||||||
|
public function __construct(
|
||||||
|
public int $device_id,
|
||||||
|
public array $module_overrides = [],
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute the job.
|
||||||
|
*/
|
||||||
|
public function handle()
|
||||||
|
{
|
||||||
|
$this->initDevice();
|
||||||
|
PollingDevice::dispatch($this->device);
|
||||||
|
$this->os = OS::make($this->deviceArray);
|
||||||
|
|
||||||
|
$measurement = Measurement::start('poll');
|
||||||
|
$measurement->manager()->checkpoint(); // don't count previous stats
|
||||||
|
|
||||||
|
$helper = new ConnectivityHelper($this->device);
|
||||||
|
$helper->saveMetrics();
|
||||||
|
$helper->isUp(); // check and save status
|
||||||
|
|
||||||
|
$this->pollModules();
|
||||||
|
|
||||||
|
$measurement->end();
|
||||||
|
|
||||||
|
// if modules are not overridden, record performance
|
||||||
|
if (empty($this->modules)) {
|
||||||
|
if ($this->device->status) {
|
||||||
|
$this->recordPerformance($measurement);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($helper->canPing()) {
|
||||||
|
$this->os->enableGraph('ping_perf');
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->os->persistGraphs($this->device->status); // save graphs but don't delete any if device is down
|
||||||
|
Log::info(sprintf("Enabled graphs (%s): %s\n\n",
|
||||||
|
$this->device->graphs->count(),
|
||||||
|
$this->device->graphs->pluck('graph')->implode(' ')
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
// finalize the device poll
|
||||||
|
$this->device->save();
|
||||||
|
|
||||||
|
Log::info(sprintf("\n>>> Polled %s (%s) in %0.3f seconds <<<",
|
||||||
|
$this->device->displayName(),
|
||||||
|
$this->device->device_id,
|
||||||
|
$measurement->getDuration()));
|
||||||
|
|
||||||
|
// add log file line, this is used by the simple python dispatcher watchdog
|
||||||
|
Log::channel('single')->alert(sprintf('INFO: device:poll %s (%s) polled in %0.3fs',
|
||||||
|
$this->device->hostname,
|
||||||
|
$this->device->device_id,
|
||||||
|
$measurement->getDuration()));
|
||||||
|
|
||||||
|
// check if the poll took too long and log an event
|
||||||
|
if ($measurement->getDuration() > Config::get('rrd.step')) {
|
||||||
|
Eventlog::log('Polling took longer than ' . round(Config::get('rrd.step') / 60, 2) .
|
||||||
|
' minutes! This will cause gaps in graphs.', $this->device, 'system', Severity::Error);
|
||||||
|
}
|
||||||
|
|
||||||
|
DevicePolled::dispatch($this->device);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function pollModules(): void
|
||||||
|
{
|
||||||
|
// update $device array status
|
||||||
|
$this->deviceArray['status'] = $this->device->status;
|
||||||
|
$this->deviceArray['status_reason'] = $this->device->status_reason;
|
||||||
|
|
||||||
|
// import legacy garbage
|
||||||
|
include_once base_path('includes/functions.php');
|
||||||
|
include_once base_path('includes/common.php');
|
||||||
|
include_once base_path('includes/polling/functions.inc.php');
|
||||||
|
include_once base_path('includes/snmp.inc.php');
|
||||||
|
|
||||||
|
$datastore = app('Datastore');
|
||||||
|
|
||||||
|
foreach ($this->getModules() as $module => $status) {
|
||||||
|
$module_status = Module::pollingStatus($module, $this->device, $this->isModuleManuallyEnabled($module));
|
||||||
|
$should_poll = false;
|
||||||
|
$start_memory = memory_get_usage();
|
||||||
|
$module_start = microtime(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
$instance = Module::fromName($module);
|
||||||
|
$should_poll = $instance->shouldPoll($this->os, $module_status);
|
||||||
|
|
||||||
|
if ($should_poll) {
|
||||||
|
Log::info("#### Load poller module $module ####\n");
|
||||||
|
Log::debug($module_status);
|
||||||
|
|
||||||
|
if (is_array($status)) {
|
||||||
|
Config::set('poller_submodules.' . $module, $status);
|
||||||
|
}
|
||||||
|
|
||||||
|
$instance->poll($this->os, $datastore);
|
||||||
|
}
|
||||||
|
} catch (Throwable $e) {
|
||||||
|
// isolate module exceptions so they don't disrupt the polling process
|
||||||
|
Log::error("%rError polling $module module for {$this->device->hostname}.%n $e", ['color' => true]);
|
||||||
|
Eventlog::log("Error polling $module module. Check log file for more details.", $this->device, 'poller', Severity::Error);
|
||||||
|
report($e);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($should_poll) {
|
||||||
|
Log::info('');
|
||||||
|
app(MeasurementManager::class)->printChangedStats();
|
||||||
|
$this->saveModulePerformance($module, $module_start, $start_memory);
|
||||||
|
Log::info("#### Unload poller module $module ####\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function saveModulePerformance(string $module, float $start_time, int $start_memory): void
|
||||||
|
{
|
||||||
|
$module_time = microtime(true) - $start_time;
|
||||||
|
$module_mem = (memory_get_usage() - $start_memory);
|
||||||
|
|
||||||
|
Log::info(sprintf(">> Runtime for poller module '%s': %.4f seconds with %s bytes", $module, $module_time, $module_mem));
|
||||||
|
|
||||||
|
app('Datastore')->put($this->deviceArray, 'poller-perf', [
|
||||||
|
'module' => $module,
|
||||||
|
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
|
||||||
|
'rrd_name' => ['poller-perf', $module],
|
||||||
|
], [
|
||||||
|
'poller' => $module_time,
|
||||||
|
]);
|
||||||
|
$this->os->enableGraph('poller_modules_perf');
|
||||||
|
}
|
||||||
|
|
||||||
|
private function initDevice(): void
|
||||||
|
{
|
||||||
|
\DeviceCache::setPrimary($this->device_id);
|
||||||
|
$this->device = \DeviceCache::getPrimary();
|
||||||
|
$this->device->ip = $this->device->overwrite_ip ?: Dns::lookupIp($this->device) ?: $this->device->ip;
|
||||||
|
|
||||||
|
$this->deviceArray = $this->device->toArray();
|
||||||
|
if ($os_group = Config::get("os.{$this->device->os}.group")) {
|
||||||
|
$this->deviceArray['os_group'] = $os_group;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->printDeviceInfo($os_group);
|
||||||
|
$this->initRrdDirectory();
|
||||||
|
}
|
||||||
|
|
||||||
|
private function initRrdDirectory(): void
|
||||||
|
{
|
||||||
|
$host_rrd = \Rrd::name($this->device->hostname, '', '');
|
||||||
|
if (Config::get('rrd.enable', true) && ! is_dir($host_rrd)) {
|
||||||
|
try {
|
||||||
|
mkdir($host_rrd);
|
||||||
|
Log::info("Created directory : $host_rrd");
|
||||||
|
} catch (\ErrorException $e) {
|
||||||
|
Eventlog::log("Failed to create rrd directory: $host_rrd", $this->device);
|
||||||
|
Log::error($e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function printDeviceInfo(?string $group): void
|
||||||
|
{
|
||||||
|
Log::info(sprintf(<<<'EOH'
|
||||||
|
Hostname: %s %s
|
||||||
|
ID: %s
|
||||||
|
OS: %s
|
||||||
|
IP: %s
|
||||||
|
|
||||||
|
EOH, $this->device->hostname, $group ? " ($group)" : '', $this->device->device_id, $this->device->os, $this->device->ip));
|
||||||
|
}
|
||||||
|
|
||||||
|
private function recordPerformance(Measurement $measurement): void
|
||||||
|
{
|
||||||
|
$measurement->manager()->record('device', $measurement);
|
||||||
|
$this->device->last_polled = Carbon::now();
|
||||||
|
$this->device->last_polled_timetaken = $measurement->getDuration();
|
||||||
|
|
||||||
|
app('Datastore')->put($this->deviceArray, 'poller-perf', [
|
||||||
|
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
|
||||||
|
'module' => 'ALL',
|
||||||
|
], [
|
||||||
|
'poller' => $this->device->last_polled_timetaken,
|
||||||
|
]);
|
||||||
|
|
||||||
|
$this->os->enableGraph('poller_perf');
|
||||||
|
}
|
||||||
|
|
||||||
|
private function getModules(): array
|
||||||
|
{
|
||||||
|
if (! empty($this->module_overrides)) {
|
||||||
|
return $this->module_overrides;
|
||||||
|
}
|
||||||
|
|
||||||
|
return \LibreNMS\Config::get('poller_modules', []);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function isModuleManuallyEnabled(string $module): ?bool
|
||||||
|
{
|
||||||
|
if (empty($this->module_overrides)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return isset($this->module_overrides[$module]);
|
||||||
|
}
|
||||||
|
}
|
Reference in New Issue
Block a user