mirror/librenms-librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-10-07 16:52:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Connectivity Helper to check and record device reachability (#13315)

Browse Source 
* Fping WIP

* Update availability, move ping rrd update in the same place as db update.

* move classes around

* make device:ping command work

* use new code, remove legacy code

* save metrics boolean prevents all saves
style fixes

* update device array

* style fixes

* Update unit test

* fix whitespace

* Fix Fping stub

* fix backwards if

* fix phpstan complaining

* Fix return type

* add fillable to DeviceOutage model.

* device_outage migration to add id...

* missed line in db_schema.yaml

* 1 billion more comments on the brain damage up/down code

* tests for status and status_reason fields

* fix style again :D

* Duplicate legacy isSNMPable() functionality
but with only one snmp call ever 😎

* Remove unused variable

* fix migrations for sqlite
This commit is contained in:
Tony Murray
2021-10-03 22:45:10 -05:00
committed by GitHub
parent d443d2b4b1
commit aa35d2f0c0
18 changed files with 722 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
LibreNMS/Fping.php → LibreNMS/Data/Source/Fping.php
View File

@@ -1,5 +1,5 @@
<?php
/**
/*
* Fping.php
*
* -Description-
@@ -15,16 +15,17 @@
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @link https://www.librenms.org
*
* @copyright 2020 Tony Murray
* @package LibreNMS
* @link http://librenms.org
* @copyright 2021 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS;
namespace LibreNMS\Data\Source;
use LibreNMS\Config;
use Log;
use Symfony\Component\Process\Process;
@@ -38,9 +39,9 @@ class Fping
* @param int $interval (min 20)
* @param int $timeout (not more than $interval)
* @param string $address_family ipv4 or ipv6
* @return array
* @return \LibreNMS\Data\Source\FpingResponse
*/
public function ping($host, $count = 3, $interval = 1000, $timeout = 500, $address_family = 'ipv4')
public function ping($host, $count = 3, $interval = 1000, $timeout = 500, $address_family = 'ipv4'): FpingResponse
{
$interval = max($interval, 20);
@@ -67,28 +68,10 @@ class Fping
$process = app()->make(Process::class, ['command' => $cmd]);
Log::debug('[FPING] ' . $process->getCommandLine() . PHP_EOL);
$process->run();
$output = $process->getErrorOutput();
preg_match('#= (\d+)/(\d+)/(\d+)%(, min/avg/max = ([\d.]+)/([\d.]+)/([\d.]+))?$#', $output, $parsed);
[, $xmt, $rcv, $loss, , $min, $avg, $max] = array_pad($parsed, 8, 0);
$response = FpingResponse::parseOutput($process->getErrorOutput(), $process->getExitCode());
if ($loss < 0) {
$xmt = 1;
$rcv = 1;
$loss = 100;
}
$response = [
'xmt' => (int) $xmt,
'rcv' => (int) $rcv,
'loss' => (int) $loss,
'min' => (float) $min,
'max' => (float) $max,
'avg' => (float) $avg,
'dup' => substr_count($output, 'duplicate'),
'exitcode' => $process->getExitCode(),
];
Log::debug('response: ', $response);
Log::debug("response: $response");
return $response;
}

156
LibreNMS/Data/Source/FpingResponse.php Normal file
View File

@@ -0,0 +1,156 @@
<?php
/*
* FpingResponse.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2021 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Source;
use App\Models\DevicePerf;
class FpingResponse
{
/**
* @var int
*/
public $transmitted;
/**
* @var int
*/
public $received;
/**
* @var int
*/
public $loss;
/**
* @var float
*/
public $min_latency;
/**
* @var float
*/
public $max_latency;
/**
* @var float
*/
public $avg_latency;
/**
* @var int
*/
public $duplicates;
/**
* @var int
*/
public $exit_code;
/**
* @var bool
*/
private $skipped;
/**
* @param int $transmitted ICMP packets transmitted
* @param int $received ICMP packets received
* @param int $loss Percentage of packets lost
* @param float $min_latency Minimum latency (ms)
* @param float $max_latency Maximum latency (ms)
* @param float $avg_latency Average latency (ms)
* @param int $duplicates Number of duplicate responses (Indicates network issue)
* @param int $exit_code Return code from fping
*/
public function __construct(int $transmitted, int $received, int $loss, float $min_latency, float $max_latency, float $avg_latency, int $duplicates, int $exit_code, bool $skipped = false)
{
$this->transmitted = $transmitted;
$this->received = $received;
$this->loss = $loss;
$this->min_latency = $min_latency;
$this->max_latency = $max_latency;
$this->avg_latency = $avg_latency;
$this->duplicates = $duplicates;
$this->exit_code = $exit_code;
$this->skipped = $skipped;
}
public static function artificialUp(): FpingResponse
{
return new FpingResponse(1, 1, 0, 0, 0, 0, 0, 0, true);
}
public function wasSkipped(): bool
{
return $this->skipped;
}
public static function parseOutput(string $output, int $code): FpingResponse
{
preg_match('#= (\d+)/(\d+)/(\d+)%(, min/avg/max = ([\d.]+)/([\d.]+)/([\d.]+))?$#', $output, $parsed);
[, $xmt, $rcv, $loss, , $min, $avg, $max] = array_pad($parsed, 8, 0);
if ($loss < 0) {
$xmt = 1;
$rcv = 0;
$loss = 100;
}
return new FpingResponse(
(int) $xmt,
(int) $rcv,
(int) $loss,
(float) $min,
(float) $max,
(float) $avg,
substr_count($output, 'duplicate'),
$code
);
}
/**
* Ping result was successful.
* fping didn't have an error and we got at least one ICMP packet back.
*/
public function success(): bool
{
return $this->exit_code == 0 && $this->loss < 100;
}
public function toModel(): ?DevicePerf
{
return new DevicePerf([
'xmt' => $this->transmitted,
'rcv' => $this->received,
'loss' => $this->loss,
'min' => $this->min_latency,
'max' => $this->max_latency,
'avg' => $this->avg_latency,
]);
}
public function __toString()
{
$str = "xmt/rcv/%loss = $this->transmitted/$this->received/$this->loss%";
if ($this->max_latency) {
$str .= ", min/avg/max = $this->min_latency/$this->avg_latency/$this->max_latency";
}
return $str;
}
}

216
LibreNMS/Polling/ConnectivityHelper.php Normal file
View File

@@ -0,0 +1,216 @@
<?php
/*
* ConnectivityHelper.php
*
* Helper to check the connectivity to a device and optionally save metrics about that connectivity
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2021 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Polling;
use App\Models\Device;
use App\Models\DeviceOutage;
use LibreNMS\Config;
use LibreNMS\Data\Source\Fping;
use LibreNMS\Data\Source\FpingResponse;
use LibreNMS\RRD\RrdDefinition;
use Log;
use NetSnmp;
use Symfony\Component\Process\Process;
class ConnectivityHelper
{
/**
* @var \App\Models\Device
*/
private $device;
/**
* @var bool
*/
private $saveMetrics = false;
/**
* @var string
*/
private $family;
/**
* @var string
*/
private $target;
public function __construct(Device $device)
{
$this->device = $device;
$this->target = $device->overwrite_ip ?: $device->hostname;
}
/**
* After pinging the device, save metrics about the ping response
*/
public function saveMetrics(): void
{
$this->saveMetrics = true;
}
/**
* Check if the device is up.
* Save availability and ping data if enabled with savePingPerf()
*/
public function isUp(): bool
{
$previous = $this->device->status;
$ping_response = $this->isPingable();
// calculate device status
if ($ping_response->success()) {
if (! $this->canSnmp() || $this->isSNMPable()) {
// up
$this->device->status = true;
$this->device->status_reason = '';
} else {
// snmp down
$this->device->status = false;
$this->device->status_reason = 'snmp';
}
} else {
// icmp down
$this->device->status = false;
$this->device->status_reason = 'icmp';
}
if ($this->saveMetrics) {
if ($this->canPing()) {
$this->savePingStats($ping_response);
}
$this->updateAvailability($previous, $this->device->status);
$this->device->save(); // confirm device is saved
}
return $this->device->status;
}
/**
* Check if the device responds to ICMP echo requests ("pings").
*/
public function isPingable(): FpingResponse
{
if (! $this->canPing()) {
return FpingResponse::artificialUp();
}
$status = app()->make(Fping::class)->ping(
$this->target,
Config::get('fping_options.count', 3),
Config::get('fping_options.interval', 500),
Config::get('fping_options.timeout', 500),
$this->ipFamily()
);
if ($status->duplicates > 0) {
Log::event('Duplicate ICMP response detected! This could indicate a network issue.', $this->device, 'icmp', 4);
$status->exit_code = 0; // when duplicate is detected fping returns 1. The device is up, but there is another issue. Clue admins in with above event.
}
return $status;
}
public function isSNMPable(): bool
{
$response = NetSnmp::device($this->device)->get('SNMPv2-MIB::sysObjectID.0');
return $response->getExitCode() === 0 || $response->isValid();
}
public function traceroute(): array
{
$command = [Config::get('traceroute', 'traceroute'), '-q', '1', '-w', '1', $this->target];
if ($this->ipFamily() == 'ipv6') {
$command[] = '-6';
}
$process = new Process($command);
$process->run();
return [
'traceroute' => $process->getOutput(),
'output' => $process->getErrorOutput(),
];
}
public function canSnmp(): bool
{
return ! $this->device->snmp_disable;
}
public function canPing(): bool
{
return Config::get('icmp_check') && ! ($this->device->exists && $this->device->getAttrib('override_icmp_disable') === 'true');
}
public function ipFamily(): string
{
if ($this->family === null) {
$this->family = preg_match('/6$/', $this->device->transport) ? 'ipv6' : 'ipv4';
}
return $this->family;
}
private function updateAvailability(bool $previous, bool $status): void
{
if (Config::get('graphing.availability_consider_maintenance') && $this->device->isUnderMaintenance()) {
return;
}
// check for open outage
$open_outage = $this->device->outages()->whereNull('up_again')->orderBy('going_down', 'desc')->first();
if ($status) {
if ($open_outage) {
$open_outage->up_again = time();
$open_outage->save();
}
} elseif ($previous || $open_outage === null) {
// status changed from up to down or there is no open outage
// open new outage
$this->device->outages()->save(new DeviceOutage(['going_down' => time()]));
}
}
/**
* Save the ping stats to db and rrd, also updates last_ping_timetaken and saves the device model.
*/
private function savePingStats(FpingResponse $ping_response): void
{
$perf = $ping_response->toModel();
if (! $ping_response->success() && Config::get('debug.run_trace', false)) {
$perf->debug = $this->traceroute();
}
$this->device->perf()->save($perf);
$this->device->last_ping_timetaken = $ping_response->avg_latency ?: $this->device->last_ping_timetaken;
$this->device->save();
app('Datastore')->put($this->device->toArray(), 'ping-perf', [
'rrd_def' => RrdDefinition::make()->addDataset('ping', 'GAUGE', 0, 65535),
], [
'ping' => $ping_response->avg_latency,
]);
}
}

58
app/Console/Commands/DevicePing.php Normal file
View File

@@ -0,0 +1,58 @@
<?php
namespace App\Console\Commands;
use App\Console\LnmsCommand;
use App\Models\Device;
use Illuminate\Database\Eloquent\Builder;
use LibreNMS\Config;
use LibreNMS\Polling\ConnectivityHelper;
use Symfony\Component\Console\Input\InputArgument;
class DevicePing extends LnmsCommand
{
protected $name = 'device:ping';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
$this->addArgument('device spec', InputArgument::REQUIRED);
}
/**
* Execute the console command.
*
* @return int
*/
public function handle(): int
{
$spec = $this->argument('device spec');
$devices = Device::query()->when($spec !== 'all', function (Builder $query) use ($spec) {
/** @phpstan-var Builder<Device> $query */
return $query->where('device_id', $spec)
->orWhere('hostname', $spec)
->limit(1);
})->get();
if ($devices->isEmpty()) {
$devices = [new Device(['hostname' => $spec])];
}
Config::set('icmp_check', true); // ignore icmp disabled, this is an explicit user action
/** @var Device $device */
foreach ($devices as $device) {
$helper = new ConnectivityHelper($device);
$response = $helper->isPingable();
$this->line($device->displayName() . ' : ' . ($response->wasSkipped() ? 'skipped' : $response));
}
return 0;
}
}

5
app/Models/Device.php
View File

@@ -827,6 +827,11 @@ class Device extends BaseModel
return $this->hasMany(\App\Models\MplsTunnelCHop::class, 'device_id');
}
public function outages(): HasMany
{
return $this->hasMany(DeviceOutage::class, 'device_id');
}
public function printerSupplies(): HasMany
{
return $this->hasMany(PrinterSupply::class, 'device_id');

2
app/Models/DeviceOutage.php
View File

@@ -28,5 +28,5 @@ namespace App\Models;
class DeviceOutage extends DeviceRelatedModel
{
public $timestamps = false;
protected $primaryKey = null;
protected $fillable = ['going_down', 'up_again'];
}

1
app/Models/DevicePerf.php
View File

@@ -36,6 +36,7 @@ class DevicePerf extends DeviceRelatedModel
'min' => 'float',
'max' => 'float',
'avg' => 'float',
'debug' => 'array',
];
public $timestamps = false;
const CREATED_AT = 'timestamp';

7
app/Observers/DeviceObserver.php
View File

@@ -31,6 +31,13 @@ class DeviceObserver
$device->children->each->updateMaxDepth();
}
// log up/down status changes
if ($device->isDirty(['status', 'status_reason'])) {
$type = $device->status ? 'up' : 'down';
$reason = $device->status ? $device->getOriginal('status_reason') : $device->status_reason;
Log::event('Device status changed to ' . ucfirst($type) . " from $reason check.", $device, $type);
}
// key attribute changes
foreach (['os', 'sysName', 'version', 'hardware', 'features', 'serial', 'icon', 'type'] as $attribute) {
if ($device->isDirty($attribute)) {

1
database/migrations/2020_06_06_222222_create_device_outages_table.php
View File

@@ -13,6 +13,7 @@ class CreateDeviceOutagesTable extends Migration
public function up()
{
Schema::create('device_outages', function (Blueprint $table) {
$table->bigIncrements('id');
$table->unsignedInteger('device_id')->index();
$table->bigInteger('going_down');
$table->bigInteger('up_again')->nullable();

31
database/migrations/2021_10_02_190310_add_device_outages_index.php Normal file
View File

@@ -0,0 +1,31 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class AddDeviceOutagesIndex extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
if (! Schema::hasColumn('device_outages', 'id')) {
Schema::table('device_outages', function (Blueprint $table) {
$table->bigIncrements('id')->first();
});
}
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
}
}

4
includes/discovery/functions.inc.php
View File

@@ -137,9 +137,9 @@ function discover_device(&$device, $force_module = false)
// Start counting device poll time
echo $device['hostname'] . ' ' . $device['device_id'] . ' ' . $device['os'] . ' ';
$response = device_is_up($device, true);
$helper = new \LibreNMS\Polling\ConnectivityHelper(DeviceCache::getPrimary());
if ($response['status'] !== '1') {
if (! $helper->isUp()) {
return false;
}

181
includes/functions.php
View File

@@ -17,13 +17,11 @@ use LibreNMS\Exceptions\HostUnreachableException;
use LibreNMS\Exceptions\HostUnreachablePingException;
use LibreNMS\Exceptions\InvalidPortAssocModeException;
use LibreNMS\Exceptions\SnmpVersionUnsupportedException;
use LibreNMS\Fping;
use LibreNMS\Modules\Core;
use LibreNMS\Util\Debug;
use LibreNMS\Util\IPv4;
use LibreNMS\Util\IPv6;
use PHPMailer\PHPMailer\PHPMailer;
use Symfony\Component\Process\Process;
function array_sort_by_column($array, $on, $order = SORT_ASC)
{
@@ -413,9 +411,7 @@ function addHost($host, $snmp_version = '', $port = 161, $transport = 'udp', $po
// Test reachability
if (! $force_add) {
$address_family = snmpTransportToAddressFamily($transport);
$ping_result = isPingable($ip, $address_family);
if (! $ping_result['result']) {
if (! ((new \LibreNMS\Polling\ConnectivityHelper(new Device(['hostname' => $ip])))->isPingable()->success())) {
throw new HostUnreachablePingException("Could not ping $host");
}
}
@@ -512,44 +508,6 @@ function isSNMPable($device)
}
}
/**
* Check if the given host responds to ICMP echo requests ("pings").
*
* @param string $hostname The hostname or IP address to send ping requests to.
* @param string $address_family The address family ('ipv4' or 'ipv6') to use. Defaults to IPv4.
* Will *not* be autodetected for IP addresses, so it has to be set to 'ipv6' when pinging an IPv6 address or an IPv6-only host.
* @param array $attribs The device attributes
* @return array 'result' => bool pingable, 'last_ping_timetaken' => int time for last ping, 'db' => fping results
*/
function isPingable($hostname, $address_family = 'ipv4', $attribs = [])
{
if (can_ping_device($attribs) !== true) {
return [
'result' => true,
'last_ping_timetaken' => 0,
];
}
$status = app()->make(Fping::class)->ping(
$hostname,
Config::get('fping_options.count', 3),
Config::get('fping_options.interval', 500),
Config::get('fping_options.timeout', 500),
$address_family
);
if ($status['dup'] > 0) {
Log::event('Duplicate ICMP response detected! This could indicate a network issue.', getidbyname($hostname), 'icmp', 4);
$status['exitcode'] = 0; // when duplicate is detected fping returns 1. The device is up, but there is another issue. Clue admins in with above event.
}
return [
'result' => ($status['exitcode'] == 0 && $status['loss'] < 100),
'last_ping_timetaken' => $status['avg'],
'db' => array_intersect_key($status, array_flip(['xmt', 'rcv', 'loss', 'min', 'max', 'avg'])),
];
}
function getpollergroup($poller_group = '0')
{
//Is poller group an integer
@@ -1193,28 +1151,6 @@ function device_has_ip($ip)
return false; // not an ipv4 or ipv6 address...
}
/**
* Try to determine the address family (IPv4 or IPv6) associated with an SNMP
* transport specifier (like "udp", "udp6", etc.).
*
* @param string $transport The SNMP transport specifier, for example "udp",
* "udp6", "tcp", or "tcp6". See `man snmpcmd`,
* section "Agent Specification" for a full list.
* @return string The address family associated with the given transport
* specifier: 'ipv4' (or local connections not associated
* with an IP stack) or 'ipv6'.
*/
function snmpTransportToAddressFamily($transport)
{
$ipv6_snmp_transport_specifiers = ['udp6', 'udpv6', 'udpipv6', 'tcp6', 'tcpv6', 'tcpipv6'];
if (in_array($transport, $ipv6_snmp_transport_specifiers)) {
return 'ipv6';
}
return 'ipv4';
}
/**
* Checks if the $hostname provided exists in the DB already
*
@@ -1733,121 +1669,6 @@ function recordSnmpStatistic($stat, $start_time)
return $runtime;
}
function runTraceroute($device)
{
$address_family = snmpTransportToAddressFamily($device['transport']);
$trace_name = $address_family == 'ipv6' ? 'traceroute6' : 'traceroute';
$trace_path = Config::get($trace_name, $trace_name);
$process = new Process([$trace_path, '-q', '1', '-w', '1', $device['hostname']]);
$process->run();
if ($process->isSuccessful()) {
return ['traceroute' => $process->getOutput()];
}
return ['output' => $process->getErrorOutput()];
}
/**
* @param $device
* @param bool $record_perf
* @return array
*/
function device_is_up($device, $record_perf = false)
{
$address_family = snmpTransportToAddressFamily($device['transport']);
$poller_target = Device::pollerTarget($device['hostname']);
$ping_response = isPingable($poller_target, $address_family, $device['attribs']);
$device_perf = $ping_response['db'];
$device_perf['device_id'] = $device['device_id'];
$device_perf['timestamp'] = ['NOW()'];
$maintenance = DeviceCache::get($device['device_id'])->isUnderMaintenance();
$consider_maintenance = Config::get('graphing.availability_consider_maintenance');
$state_update_again = false;
if ($record_perf === true && can_ping_device($device['attribs'])) {
$trace_debug = [];
if ($ping_response['result'] === false && Config::get('debug.run_trace', false)) {
$trace_debug = runTraceroute($device);
}
$device_perf['debug'] = json_encode($trace_debug);
dbInsert($device_perf, 'device_perf');
// if device_perf is inserted and the ping was successful then update device last_ping timestamp
if (! empty($ping_response['last_ping_timetaken']) && $ping_response['last_ping_timetaken'] != '0') {
dbUpdate(
['last_ping' => NOW(), 'last_ping_timetaken' => $ping_response['last_ping_timetaken']],
'devices',
'device_id=?',
[$device['device_id']]
);
}
}
$response = [];
$response['ping_time'] = $ping_response['last_ping_timetaken'];
if ($ping_response['result']) {
if ($device['snmp_disable'] || isSNMPable($device)) {
$response['status'] = '1';
$response['status_reason'] = '';
} else {
echo 'SNMP Unreachable';
$response['status'] = '0';
$response['status_reason'] = 'snmp';
}
} else {
echo 'Unpingable';
$response['status'] = '0';
$response['status_reason'] = 'icmp';
}
// Special case where the device is still down, optional mode is on, device not in maintenance mode and has no ongoing outages
if (($consider_maintenance && ! $maintenance) && ($device['status'] == '0' && $response['status'] == '0')) {
$state_update_again = empty(dbFetchCell('SELECT going_down FROM device_outages WHERE device_id=? AND up_again IS NULL ORDER BY going_down DESC', [$device['device_id']]));
}
if ($device['status'] != $response['status'] || $device['status_reason'] != $response['status_reason'] || $state_update_again) {
if (! $state_update_again) {
dbUpdate(
['status' => $response['status'], 'status_reason' => $response['status_reason']],
'devices',
'device_id=?',
[$device['device_id']]
);
}
if ($response['status']) {
$type = 'up';
$reason = $device['status_reason'];
$going_down = dbFetchCell('SELECT going_down FROM device_outages WHERE device_id=? AND up_again IS NULL ORDER BY going_down DESC', [$device['device_id']]);
if (! empty($going_down)) {
$up_again = time();
dbUpdate(
['device_id' => $device['device_id'], 'up_again' => $up_again],
'device_outages',
'device_id=? and going_down=? and up_again is NULL',
[$device['device_id'], $going_down]
);
}
} else {
$type = 'down';
$reason = $response['status_reason'];
if ($device['status'] != $response['status']) {
if (! $consider_maintenance || (! $maintenance && $consider_maintenance)) {
// use current time as a starting point when an outage starts
$data = ['device_id' => $device['device_id'],
'going_down' => time(), ];
dbInsert($data, 'device_outages');
}
}
}
log_event('Device status changed to ' . ucfirst($type) . " from $reason check.", $device, $type);
}
return $response;
}
function update_device_logo(&$device)
{
$icon = getImageName($device, false);

42
includes/polling/functions.inc.php
View File

@@ -236,7 +236,8 @@ function poll_device($device, $force_module = false)
$device_start = microtime(true);
$attribs = DeviceCache::getPrimary()->getAttribs();
$deviceModel = DeviceCache::getPrimary();
$attribs = $deviceModel->getAttribs();
$device['attribs'] = $attribs;
load_os($device);
@@ -289,9 +290,10 @@ function poll_device($device, $force_module = false)
echo "Created directory : $host_rrd\n";
}
$response = device_is_up($device, true);
$helper = new \LibreNMS\Polling\ConnectivityHelper($deviceModel);
$helper->saveMetrics();
if ($response['status'] == '1') {
if ($helper->isUp()) {
if ($device['snmp_disable']) {
Config::set('poller_modules', ['availability' => true]);
} else {
@@ -299,6 +301,10 @@ function poll_device($device, $force_module = false)
Config::set('poller_modules', ['core' => true, 'availability' => true] + Config::get('poller_modules'));
}
// update $device array status
$device['status'] = $deviceModel->status;
$device['status_reason'] = $deviceModel->status_reason;
printChangedStats(true); // don't count previous stats
foreach (Config::get('poller_modules') as $module => $module_status) {
$os_module_status = Config::get("os.{$device['os']}.poller_modules.$module");
@@ -356,36 +362,10 @@ function poll_device($device, $force_module = false)
}
// Ping response
if (can_ping_device($attribs) === true && ! empty($response['ping_time'])) {
$tags = [
'rrd_def' => RrdDefinition::make()->addDataset('ping', 'GAUGE', 0, 65535),
];
$fields = [
'ping' => $response['ping_time'],
];
$update_array['last_ping'] = ['NOW()'];
$update_array['last_ping_timetaken'] = $response['ping_time'];
data_update($device, 'ping-perf', $tags, $fields);
if ($helper->canPing()) {
$os->enableGraph('ping_perf');
}
$device_time = round(microtime(true) - $device_start, 3);
// Poller performance
if (! empty($device_time)) {
$tags = [
'rrd_def' => RrdDefinition::make()->addDataset('poller', 'GAUGE', 0),
'module' => 'ALL',
];
$fields = [
'poller' => $device_time,
];
data_update($device, 'poller-perf', $tags, $fields);
$os->enableGraph('poller_modules_perf');
}
$os->enableGraph('poller_modules_perf');
if (! $force_module) {
// don't update last_polled time if we are forcing a specific module to be polled

2
misc/db_schema.yaml
View File

@@ -608,10 +608,12 @@ device_group_device:
device_group_device_device_id_foreign: { name: device_group_device_device_id_foreign, foreign_key: device_id, table: devices, key: device_id, extra: 'ON DELETE CASCADE' }
device_outages:
Columns:
- { Field: id, Type: 'bigint unsigned', 'Null': false, Extra: auto_increment }
- { Field: device_id, Type: 'int unsigned', 'Null': false, Extra: '' }
- { Field: going_down, Type: bigint, 'Null': false, Extra: '' }
- { Field: up_again, Type: bigint, 'Null': true, Extra: '' }
Indexes:
PRIMARY: { Name: PRIMARY, Columns: [ id ], Unique: true, Type: BTREE }
device_outages_device_id_going_down_unique: { Name: device_outages_device_id_going_down_unique, Columns: [device_id, going_down], Unique: true, Type: BTREE }
device_outages_device_id_index: { Name: device_outages_device_id_index, Columns: [device_id], Unique: false, Type: BTREE }
device_perf:

6
resources/lang/en/commands.php
View File

@@ -63,6 +63,12 @@ return [
'removed' => 'Device :id removed',
'updated' => 'Device :hostname (:id) updated',
],
'device:ping' => [
'description' => 'Ping device and record data for response',
'arguments' => [
'device spec' => 'Device to ping one of: <Device ID>, <Hostname/IP>, all',
],
],
'key:rotate' => [
'description' => 'Rotate APP_KEY, this decrypts all encrypted data with the given old key and stores it with the new key in APP_KEY.',
'arguments' => [

75
tests/FpingTest.php
View File

@@ -25,7 +25,7 @@
namespace LibreNMS\Tests;
use LibreNMS\Fping;
use LibreNMS\Data\Source\Fping;
use Symfony\Component\Process\Process;
class FpingTest extends TestCase
@@ -48,7 +48,15 @@ class FpingTest extends TestCase
$actual = app()->make(Fping::class)->ping('192.168.1.3');
$this->assertSame($expected, $actual);
$this->assertTrue($actual->success());
$this->assertEquals(3, $actual->transmitted);
$this->assertEquals(3, $actual->received);
$this->assertEquals(0, $actual->loss);
$this->assertEquals(0.62, $actual->min_latency);
$this->assertEquals(0.93, $actual->max_latency);
$this->assertEquals(0.71, $actual->avg_latency);
$this->assertEquals(0, $actual->duplicates);
$this->assertEquals(0, $actual->exit_code);
}
public function testPartialDownPing()
@@ -56,20 +64,17 @@ class FpingTest extends TestCase
$output = "192.168.1.7 : xmt/rcv/%loss = 5/3/40%, min/avg/max = 0.13/0.23/0.32\n";
$this->mockFpingProcess($output, 0);
$expected = [
'xmt' => 5,
'rcv' => 3,
'loss' => 40,
'min' => 0.13,
'max' => 0.32,
'avg' => 0.23,
'dup' => 0,
'exitcode' => 0,
];
$actual = app()->make(Fping::class)->ping('192.168.1.7');
$this->assertSame($expected, $actual);
$this->assertTrue($actual->success());
$this->assertEquals(5, $actual->transmitted);
$this->assertEquals(3, $actual->received);
$this->assertEquals(40, $actual->loss);
$this->assertEquals(0.13, $actual->min_latency);
$this->assertEquals(0.32, $actual->max_latency);
$this->assertEquals(0.23, $actual->avg_latency);
$this->assertEquals(0, $actual->duplicates);
$this->assertEquals(0, $actual->exit_code);
}
public function testDownPing()
@@ -77,20 +82,17 @@ class FpingTest extends TestCase
$output = "192.168.53.1 : xmt/rcv/%loss = 3/0/100%\n";
$this->mockFpingProcess($output, 1);
$expected = [
'xmt' => 3,
'rcv' => 0,
'loss' => 100,
'min' => 0.0,
'max' => 0.0,
'avg' => 0.0,
'dup' => 0,
'exitcode' => 1,
];
$actual = app()->make(Fping::class)->ping('192.168.53.1');
$this->assertSame($expected, $actual);
$this->assertFalse($actual->success());
$this->assertEquals(3, $actual->transmitted);
$this->assertEquals(0, $actual->received);
$this->assertEquals(100, $actual->loss);
$this->assertEquals(0.0, $actual->min_latency);
$this->assertEquals(0.0, $actual->max_latency);
$this->assertEquals(0.0, $actual->avg_latency);
$this->assertEquals(0, $actual->duplicates);
$this->assertEquals(1, $actual->exit_code);
}
public function testDuplicatePing()
@@ -103,20 +105,17 @@ OUT;
$this->mockFpingProcess($output, 1);
$expected = [
'xmt' => 3,
'rcv' => 3,
'loss' => 0,
'min' => 0.68,
'max' => 0.91,
'avg' => 0.79,
'dup' => 2,
'exitcode' => 1,
];
$actual = app()->make(Fping::class)->ping('192.168.1.2');
$this->assertSame($expected, $actual);
$this->assertFalse($actual->success());
$this->assertEquals(3, $actual->transmitted);
$this->assertEquals(3, $actual->received);
$this->assertEquals(0, $actual->loss);
$this->assertEquals(0.68, $actual->min_latency);
$this->assertEquals(0.91, $actual->max_latency);
$this->assertEquals(0.79, $actual->avg_latency);
$this->assertEquals(2, $actual->duplicates);
$this->assertEquals(1, $actual->exit_code);
}
private function mockFpingProcess($output, $exitCode)

16
tests/OSModulesTest.php
View File

@@ -28,9 +28,10 @@ namespace LibreNMS\Tests;
use DeviceCache;
use Illuminate\Foundation\Testing\DatabaseTransactions;
use LibreNMS\Config;
use LibreNMS\Data\Source\Fping;
use LibreNMS\Data\Source\FpingResponse;
use LibreNMS\Exceptions\FileNotFoundException;
use LibreNMS\Exceptions\InvalidModuleException;
use LibreNMS\Fping;
use LibreNMS\Util\Debug;
use LibreNMS\Util\ModuleTestHelper;
@@ -172,17 +173,8 @@ class OSModulesTest extends DBTestCase
});
$this->app->bind(Fping::class, function ($app) {
$mock = \Mockery::mock('\LibreNMS\Fping');
$mock->shouldReceive('ping')->andReturn([
'xmt' => 3,
'rcv' => 3,
'loss' => 0,
'min' => 0.62,
'max' => 0.93,
'avg' => 0.71,
'dup' => 0,
'exitcode' => 0,
]);
$mock = \Mockery::mock('\LibreNMS\Data\Source\Fping');
$mock->shouldReceive('ping')->andReturn(FpingResponse::artificialUp());
return $mock;
});

172
tests/Unit/ConnectivityHelperTest.php Normal file
View File

@@ -0,0 +1,172 @@
<?php
namespace LibreNMS\Tests\Unit;
use App\Models\Device;
use LibreNMS\Config;
use LibreNMS\Data\Source\Fping;
use LibreNMS\Data\Source\FpingResponse;
use LibreNMS\Data\Source\SnmpResponse;
use LibreNMS\Polling\ConnectivityHelper;
use LibreNMS\Tests\TestCase;
use Mockery;
use NetSnmp;
class ConnectivityHelperTest extends TestCase
{
public function testDeviceStatus(): void
{
// not called when ping is disabled
$this->app->singleton(Fping::class, function () {
$mock = Mockery::mock(Fping::class);
$up = FpingResponse::artificialUp();
$down = new FpingResponse(1, 0, 100, 0, 0, 0, 0, 0);
$mock->shouldReceive('ping')
->times(8)
->andReturn(
$up,
$down,
$up,
$down,
$up,
$down,
$up,
$down
);
return $mock;
});
// not called when snmp is disabled or ping up
$up = new SnmpResponse('SNMPv2-MIB::sysObjectID.0 = .1');
$down = new SnmpResponse('', '', 1);
NetSnmp::partialMock()->shouldReceive('get')
->times(6)
->andReturn(
$up,
$down,
$up,
$up,
$down,
$down
);
$device = new Device();
/** ping and snmp enabled */
Config::set('icmp_check', true);
$device->snmp_disable = false;
// ping up, snmp up
$ch = new ConnectivityHelper($device);
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping down, snmp up
$this->assertFalse($ch->isUp());
$this->assertEquals(false, $device->status);
$this->assertEquals('icmp', $device->status_reason);
// ping up, snmp down
$this->assertFalse($ch->isUp());
$this->assertEquals(false, $device->status);
$this->assertEquals('snmp', $device->status_reason);
// ping down, snmp down
$this->assertFalse($ch->isUp());
$this->assertEquals(false, $device->status);
$this->assertEquals('icmp', $device->status_reason);
/** ping disabled and snmp enabled */
Config::set('icmp_check', false);
$device->snmp_disable = false;
// ping up, snmp up
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping down, snmp up
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping up, snmp down
$this->assertFalse($ch->isUp());
$this->assertEquals(false, $device->status);
$this->assertEquals('snmp', $device->status_reason);
// ping down, snmp down
$this->assertFalse($ch->isUp());
$this->assertEquals(false, $device->status);
$this->assertEquals('snmp', $device->status_reason);
/** ping enabled and snmp disabled */
Config::set('icmp_check', true);
$device->snmp_disable = true;
// ping up, snmp up
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping down, snmp up
$this->assertFalse($ch->isUp());
$this->assertEquals(false, $device->status);
$this->assertEquals('icmp', $device->status_reason);
// ping up, snmp down
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping down, snmp down
$this->assertFalse($ch->isUp());
$this->assertEquals(false, $device->status);
$this->assertEquals('icmp', $device->status_reason);
/** ping and snmp disabled */
Config::set('icmp_check', false);
$device->snmp_disable = true;
// ping up, snmp up
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping down, snmp up
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping up, snmp down
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
// ping down, snmp down
$this->assertTrue($ch->isUp());
$this->assertEquals(true, $device->status);
$this->assertEquals('', $device->status_reason);
}
public function testIsSNMPable(): void
{
NetSnmp::partialMock()->shouldReceive('get')
->times(4)
->andReturn(
new SnmpResponse('SNMPv2-MIB::sysObjectID.0 = .1', '', 0),
new SnmpResponse('SNMPv2-MIB::sysObjectID.0 = .1', '', 1),
new SnmpResponse('', '', 0),
new SnmpResponse('', '', 1)
);
$ch = new ConnectivityHelper(new Device());
$this->assertTrue($ch->isSNMPable());
$this->assertTrue($ch->isSNMPable());
$this->assertTrue($ch->isSNMPable());
$this->assertFalse($ch->isSNMPable());
}
}