mirror/librenms-librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-10-07 16:52:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor datastores to classes (#9179)

Browse Source 
* Datastores to object oriented code, using the Laravel IoC container

Change instantiation
better DI
move OpenTSDB

Small re-orgs

remove unused stuff

Fix graphs and other scripts

Use DI for all except rrd

fix up connection error handling

Add tests, fix up a "few" things
Add Config::forget()

Style fixes

Don't reference legacy code

remove accidental code paste

Add datastores phpunit groups

some tests

* rebase fixes

* some test fixes

* shorter tests

* shorter tests

* Don't except when rrdtool can't be started.

* restore tests

* fix rrd tests

* fix iterable change upstream

* fix isValidDataset

* fix invalid data bug

* fix mysql incorrect ds

* fix issue with data that is too long

* use regular data_update()

* Use log facade

* OpenTSDB mis-ordered arguments fix

* Making a singleton with different options makes different singletons.  Just use the global config settings to disable datastores.

* only filter tags for datastores that won't it don't modify the tags permanently

* Update copyrights to include original authors.

* Stats for all datastores

* Fix mysql sends different rrd / other ds names

* fix snmp last stats not initialized
remove unused function

* remove unused function and move single use function closer to its use

* InfluxDB does not need to update null or U values.
Skip write if all fields are empty

* Fix smart value checks

* fix style issues
This commit is contained in:
Tony Murray
2020-03-10 19:08:00 -05:00
committed by GitHub
parent cc53801e22
commit e5dad7a64e
51 changed files with 2523 additions and 1007 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
includes/common.php
View File

@@ -436,7 +436,7 @@ function getidbyname($hostname)
function safename($name)
{
return preg_replace('/[^a-zA-Z0-9,._\-]/', '_', $name);
return \LibreNMS\Data\Store\Rrd::safeName($name);
}
/**
@@ -446,7 +446,7 @@ function safename($name)
*/
function safedescr($descr)
{
return preg_replace('/[^a-zA-Z0-9,._\-\/\ ]/', ' ', $descr);
return \LibreNMS\Data\Store\Rrd::safeDescr($descr);
}
function zeropad($num, $length = 2)

39
includes/datastore.inc.php
View File

@@ -18,25 +18,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Filter all elements with keys that start with 'rrd_'
*
* @param array $arr input array
* @return array Copy of $arr with all keys beginning with 'rrd_' removed.
*/
function rrd_array_filter($arr)
{
$result = array();
foreach ($arr as $k => $v) {
if (strpos($k, 'rrd_') === 0) {
continue;
}
$result[$k] = $v;
}
return $result;
} // rrd_array_filter
/**
* Datastore-independent function which should be used for all polled metrics.
*
@@ -54,20 +35,6 @@ function rrd_array_filter($arr)
*/
function data_update($device, $measurement, $tags, $fields)
{
// convenience conversion to allow calling with a single value, so, e.g., these are equivalent:
// data_update($device, 'mymeasurement', $tags, 1234);
// AND
// data_update($device, 'mymeasurement', $tags, array('mymeasurement' => 1234));
if (!is_array($fields)) {
$fields = array($measurement => $fields);
}
// rrdtool_data_update() will only use the tags it deems relevant, so we pass all of them.
// However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
rrdtool_data_update($device, $measurement, $tags, $fields);
prometheus_push($device, $measurement, rrd_array_filter($tags), $fields);
influx_update($device, $measurement, rrd_array_filter($tags), $fields);
opentsdb_update($device, $measurement, rrd_array_filter($tags), $fields);
graphite_update($device, $measurement, $tags, $fields);
} // data_update
$datastore = app('Datastore');
$datastore->put($device, $measurement, $tags, $fields);
}

179
includes/functions.php
View File

@@ -344,26 +344,6 @@ function percent_colour($perc)
return sprintf('#%02x%02x%02x', $r, $b, $b);
}
// Returns the last in/out errors value in RRD
function interface_errors($rrd_file, $period = '-1d')
{
$errors = array();
$cmd = Config::get('rrdtool') . " fetch -s $period -e -300s $rrd_file AVERAGE | grep : | cut -d\" \" -f 4,5";
$data = trim(shell_exec($cmd));
$in_errors = 0;
$out_errors = 0;
foreach (explode("\n", $data) as $entry) {
list($in, $out) = explode(" ", $entry);
$in_errors += ($in * 300);
$out_errors += ($out * 300);
}
$errors['in'] = round($in_errors);
$errors['out'] = round($out_errors);
return $errors;
}
/**
* @param $device
* @return string the logo image path for this device. Images are often wide, not square.
@@ -1542,30 +1522,6 @@ function function_check($function)
return function_exists($function);
}
function force_influx_data($data)
{
/*
* It is not trivial to detect if something is a float or an integer, and
* therefore may cause breakages on inserts.
* Just setting every number to a float gets around this, but may introduce
* inefficiencies.
* I've left the detection statement in there for a possible change in future,
* but currently everything just gets set to a float.
*/
if (is_numeric($data)) {
// If it is an Integer
if (ctype_digit($data)) {
return floatval($data);
// Else it is a float
} else {
return floatval($data);
}
} else {
return $data;
}
}// end force_influx_data
/**
* Try to determine the address family (IPv4 or IPv6) associated with an SNMP
* transport specifier (like "udp", "udp6", etc.).
@@ -1662,50 +1618,6 @@ function dnslookup($device, $type = false, $return = false)
return $record[0][$return];
}//end dnslookup
/**
* Run rrdtool info on a file path
*
* @param string $path Path to pass to rrdtool info
* @param string $stdOutput Variable to recieve the output of STDOUT
* @param string $stdError Variable to recieve the output of STDERR
*
* @return int exit code
*
**/
function rrdtest($path, &$stdOutput, &$stdError)
{
//rrdtool info <escaped rrd path>
$command = Config::get('rrdtool') . ' info ' . escapeshellarg($path);
$process = proc_open(
$command,
array (
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('pipe', 'w'),
),
$pipes
);
if (!is_resource($process)) {
throw new \RuntimeException('Could not create a valid process');
}
$status = proc_get_status($process);
while ($status['running']) {
usleep(2000); // Sleep 2000 microseconds or 2 milliseconds
$status = proc_get_status($process);
}
$stdOutput = stream_get_contents($pipes[1]);
$stdError = stream_get_contents($pipes[2]);
proc_close($process);
return $status['exitcode'];
}
/**
* Create a new state index. Update translations if $states is given.
*
@@ -2062,10 +1974,9 @@ function get_toner_levels($device, $raw_value, $capacity)
*/
function initStats()
{
global $snmp_stats, $rrd_stats;
global $snmp_stats_last, $rrd_stats_last;
global $snmp_stats, $snmp_stats_last;
if (!isset($snmp_stats, $rrd_stats)) {
if (!isset($snmp_stats)) {
$snmp_stats = array(
'ops' => array(
'snmpget' => 0,
@@ -2079,20 +1990,6 @@ function initStats()
)
);
$snmp_stats_last = $snmp_stats;
$rrd_stats = array(
'ops' => array(
'update' => 0,
'create' => 0,
'other' => 0,
),
'time' => array(
'update' => 0.0,
'create' => 0.0,
'other' => 0.0,
),
);
$rrd_stats_last = $rrd_stats;
}
}
@@ -2103,25 +2000,29 @@ function initStats()
*/
function printChangedStats($update_only = false)
{
global $snmp_stats, $db_stats, $rrd_stats;
global $snmp_stats_last, $db_stats_last, $rrd_stats_last;
global $snmp_stats, $db_stats;
global $snmp_stats_last, $db_stats_last;
$output = sprintf(
">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs]",
array_sum($snmp_stats['ops']) - array_sum($snmp_stats_last['ops']),
array_sum($snmp_stats['time']) - array_sum($snmp_stats_last['time']),
array_sum($db_stats['ops']) - array_sum($db_stats_last['ops']),
array_sum($db_stats['time']) - array_sum($db_stats_last['time'])
);
foreach (app('Datastore')->getStats() as $datastore => $stats) {
/** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
$output .= sprintf(" %s: [%d/%.2fs]", $datastore, $stats->getCountDiff(), $stats->getDurationDiff());
$stats->checkpoint();
}
if (!$update_only) {
printf(
">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs] RRD: [%d/%.2fs]\n",
array_sum($snmp_stats['ops']) - array_sum($snmp_stats_last['ops']),
array_sum($snmp_stats['time']) - array_sum($snmp_stats_last['time']),
array_sum($db_stats['ops']) - array_sum($db_stats_last['ops']),
array_sum($db_stats['time']) - array_sum($db_stats_last['time']),
array_sum($rrd_stats['ops']) - array_sum($rrd_stats_last['ops']),
array_sum($rrd_stats['time']) - array_sum($rrd_stats_last['time'])
);
echo $output . PHP_EOL;
}
// make a new checkpoint
$snmp_stats_last = $snmp_stats;
$db_stats_last = $db_stats;
$rrd_stats_last = $rrd_stats;
}
/**
@@ -2129,7 +2030,7 @@ function printChangedStats($update_only = false)
*/
function printStats()
{
global $snmp_stats, $db_stats, $rrd_stats;
global $snmp_stats, $db_stats;
if ($snmp_stats) {
printf(
@@ -2167,42 +2068,18 @@ function printStats()
);
}
if ($rrd_stats) {
printf(
"RRD [%d/%.2fs]: Update[%d/%.2fs] Create [%d/%.2fs] Other[%d/%.2fs]\n",
array_sum($rrd_stats['ops']),
array_sum($rrd_stats['time']),
$rrd_stats['ops']['update'],
$rrd_stats['time']['update'],
$rrd_stats['ops']['create'],
$rrd_stats['time']['create'],
$rrd_stats['ops']['other'],
$rrd_stats['time']['other']
);
foreach (app('Datastore')->getStats() as $datastore => $stats) {
/** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
printf("%s [%d/%.2fs]:", $datastore, $stats->getTotalCount(), $stats->getTotalDuration());
foreach ($stats as $stat) {
/** @var \LibreNMS\Data\Measure\MeasurementSummary $stat */
printf(" %s[%d/%.2fs]", ucfirst($stat->getType()), $stat->getCount(), $stat->getDuration());
}
echo PHP_EOL;
}
}
/**
* Update statistics for rrd operations
*
* @param string $stat create, update, and other
* @param float $start_time The time the operation started with 'microtime(true)'
* @return float The calculated run time
*/
function recordRrdStatistic($stat, $start_time)
{
global $rrd_stats;
initStats();
$stat = ($stat == 'update' || $stat == 'create') ? $stat : 'other';
$runtime = microtime(true) - $start_time;
$rrd_stats['ops'][$stat]++;
$rrd_stats['time'][$stat] += $runtime;
return $runtime;
}
/**
* @param string $stat snmpget, snmpwalk
* @param float $start_time The time the operation started with 'microtime(true)'

50
includes/graphite.inc.php
View File

@@ -1,50 +0,0 @@
<?php
/*
* LibreNMS
*
* Copyright (c) 2017 Falk Stern <https://github.com/fstern/ >
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function graphite_update($device, $measurement, $tags, $fields)
{
global $graphite;
if ($graphite != false) {
$timestamp = time();
$graphite_prefix = \LibreNMS\Config::get('graphite.prefix');
// metrics will be built as prefix.hostname.measurement.field value timestamp
// metric fields can not contain . as this is used by graphite as a field separator
$hostname = preg_replace('/\./', '_', $device['hostname']);
$measurement = preg_replace(array('/\./', '/\//'), '_', $measurement);
$measurement = preg_replace('/\|/', '.', $measurement);
$measurement_name = preg_replace('/\./', '_', $tags['rrd_name']);
if (is_array($measurement_name)) {
$ms_name = implode(".", $measurement_name);
} else {
$ms_name = $measurement_name;
}
// remove the port-id tags from the metric
if (preg_match('/^port-id\d+/', $ms_name)) {
$ms_name = "";
}
foreach ($fields as $k => $v) {
// Send zero for fields without values
if (empty($v)) {
$v = 0;
}
$metric = implode(".", array_filter(array($graphite_prefix, $hostname, $measurement, $ms_name, $k)));
// Further sanitize the full metric before sending, whitespace isn't allowed
$metric = preg_replace('/\s+/', '_', $metric);
$line = implode(" ", array($metric, $v, $timestamp));
d_echo("Sending $line\n");
fwrite($graphite, $line . "\n");
}
}
}

4
includes/html/api_functions.inc.php
View File

@@ -21,6 +21,7 @@ use Illuminate\Routing\Router;
use Illuminate\Support\Facades\Validator;
use LibreNMS\Alerting\QueryBuilderParser;
use LibreNMS\Config;
use LibreNMS\Data\Store\Datastore;
use LibreNMS\Exceptions\InvalidIpException;
use LibreNMS\Util\IPv4;
@@ -83,9 +84,8 @@ function api_get_graph(array $vars)
ob_start();
rrdtool_initialize(false);
include 'includes/html/graphs/graph.inc.php';
rrdtool_close();
Datastore::terminate();
$image = ob_get_contents();
ob_end_clean();

2
includes/html/graphs/application/proxmox_traffic.inc.php
View File

@@ -18,7 +18,7 @@
require 'includes/html/graphs/common.inc.php';
$proxmox_rrd = proxmox_rrd_name($vars['cluster'], $vars['vmid'], $vars['port']);
$proxmox_rrd = Rrd::proxmoxName($vars['cluster'], $vars['vmid'], $vars['port']);
if (rrdtool_check_rrd_exists($proxmox_rrd)) {
$rrd_filename = $proxmox_rrd;

89
includes/influxdb.inc.php
View File

@@ -1,89 +0,0 @@
<?php
use LibreNMS\Config;
/*
* LibreNMS
*
* Copyright (c) 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function influxdb_connect()
{
$influxdb_cred = '';
if (!empty(Config::get('influxdb.username')) && !empty(Config::get('influxdb.password'))) {
$influxdb_cred = Config::get('influxdb.username') . ':' . Config::get('influxdb.password') . '@';
d_echo('Using authentication for InfluxDB');
}
$influxdb_url = $influxdb_cred . Config::get('influxdb.host') . ':' . Config::get('influxdb.port') . '/' . Config::get('influxdb.db');
d_echo(Config::get('influxdb.transport') . " transport being used");
if (Config::get('influxdb.transport') == 'http') {
$influxdb_conn = 'influxdb';
} elseif (Config::get('influxdb.transport') == 'https') {
$influxdb_conn = 'https+influxdb';
} elseif (Config::get('influxdb.transport') == 'udp') {
$influxdb_conn = 'udp+influxdb';
} else {
echo 'InfluxDB support enabled but no valid transport details provided';
return false;
}
$db = \InfluxDB\Client::fromDSN($influxdb_conn . '://' . $influxdb_url, Config::get('influxdb.timeout'), Config::get('influxdb.verifySSL'));
return($db);
}// end influxdb_connect
function influx_update($device, $measurement, $tags, $fields)
{
global $influxdb;
if ($influxdb !== false) {
$tmp_fields = array();
$tmp_tags['hostname'] = $device['hostname'];
foreach ($tags as $k => $v) {
$v = preg_replace(array('/ /','/,/','/=/'), array('\ ','\,','\='), $v);
if (empty($v)) {
$v = '_blank_';
}
$tmp_tags[$k] = $v;
}
foreach ($fields as $k => $v) {
if ($k == 'time') {
$k = 'rtime';
}
$tmp_fields[$k] = force_influx_data($v);
if ($tmp_fields[$k] === null) {
unset($tmp_fields[$k]);
}
}
d_echo("\nInfluxDB data:\n");
d_echo($measurement);
d_echo($tmp_tags);
d_echo($tmp_fields);
d_echo("\nEND\n");
if (Config::get('noinfluxdb') !== true) {
$points = array(
new InfluxDB\Point(
$measurement,
null, // the measurement value
$tmp_tags,
$tmp_fields // optional additional fields
)
);
try {
$result = $influxdb->writePoints($points);
} catch (Exception $e) {
d_echo("Caught exception: " . $e->getMessage() . PHP_EOL);
d_echo($e->getTrace());
}
} else {
c_echo("[%gInfluxDB Disabled%n]\n");
}//end if
}//end if
}// end influx_update

4
includes/init.php
View File

@@ -57,10 +57,6 @@ if (!function_exists('module_selected')) {
require_once $install_dir . '/includes/common.php';
require_once $install_dir . '/includes/dbFacile.php';
require_once $install_dir . '/includes/rrdtool.inc.php';
require_once $install_dir . '/includes/influxdb.inc.php';
require_once $install_dir . '/includes/prometheus.inc.php';
require_once $install_dir . '/includes/opentsdb.inc.php';
require_once $install_dir . '/includes/graphite.inc.php';
require_once $install_dir . '/includes/datastore.inc.php';
require_once $install_dir . '/includes/billing.php';
require_once $install_dir . '/includes/syslog.php';

62
includes/opentsdb.inc.php
View File

@@ -1,62 +0,0 @@
<?php
/*
* LibreNMS
*
* Copyright (c) 2017 Yacine Benamsili <https://github.com/yac01/librenms.git >
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function opentsdb_update($device, $measurement, $tags, $fields)
{
global $opentsdb;
if (\LibreNMS\Config::get('opentsdb.enable') == true) {
if ($opentsdb != true) {
$opentsdb = fsockopen(\LibreNMS\Config::get('opentsdb.host'), \LibreNMS\Config::get('opentsdb.port'));
}
if ($opentsdb == true) {
d_echo("Connection to OpenTSDB is done\n");
} else {
d_echo("Connection to OpenTSDB has failed\n");
}
$flag = \LibreNMS\Config::get('opentsdb.co');
$timestamp = time();
$tmp_tags = "hostname=".$device['hostname'];
foreach ($tags as $k => $v) {
$v = str_replace(array(' ',',','='), '_', $v);
if (!empty($v)) {
$tmp_tags = $tmp_tags ." ". $k ."=".$v;
}
}
if ($measurement == 'port') {
foreach ($fields as $k => $v) {
$measurement = $k;
if ($flag == true) {
$measurement = $measurement.".".$device['co'];
}
$line = sprintf('put net.port.%s %d %f %s', strtolower($measurement), $timestamp, $v, $tmp_tags);
d_echo("Sending to OPenTSDB: $line\n");
fwrite($opentsdb, $line . "\n"); // send $line into OpenTSDB
}
} else {
if ($flag == true) {
$measurement = $measurement.'.'.$device['co'];
}
foreach ($fields as $k => $v) {
$tmp_tags_key = $tmp_tags ." ". "key" ."=".$k;
$line = sprintf('put net.%s %d %f %s', strtolower($measurement), $timestamp, $v, $tmp_tags_key);
d_echo("Sending to OPenTSDB: $line\n");
fwrite($opentsdb, $line . "\n"); // send $line into OpenTSDB
}
}
}
}

2
includes/polling/applications/mysql.inc.php
View File

@@ -219,6 +219,8 @@ $mapping_status = array(
$rrd_name = array('app', $name, $app_id, 'status');
$rrd_def = new RrdDefinition();
// because this sends different names for rrd and compared to other datastores, disable $fields name checks
$rrd_def->disableNameChecking();
$fields = array();
foreach ($mapping_status as $desc => $id) {

103
includes/polling/applications/smart.inc.php
View File

@@ -47,83 +47,34 @@ while (isset($lines[$int])) {
$id196, $id197, $id198, $id199, $id231, $id233, $completed, $interrupted, $read_failure,
$unknown_failure, $extended, $short, $conveyance, $selective)=explode(",", $lines[$int]);
if (is_int($id5)) {
$id=null;
}
if (is_int($id10)) {
$id10=null;
}
if (is_int($id173)) {
$id173=null;
}
if (is_int($id177)) {
$id177=null;
}
if (is_int($id183)) {
$id183=null;
}
if (is_int($id184)) {
$id184=null;
}
if (is_int($id187)) {
$id187=null;
}
if (is_int($id188)) {
$id188=null;
}
if (is_int($id190)) {
$id190=null;
}
if (is_int($id194)) {
$id194=null;
}
if (is_int($id196)) {
$id196=null;
}
if (is_int($id197)) {
$id197=null;
}
if (is_int($id198)) {
$id198=null;
}
if (is_int($id199)) {
$id199=null;
}
if (is_int($id231)) {
$id231=null;
}
if (is_int($id233)) {
$id233=null;
}
$rrd_name = array('app', $name, $app_id, $disk);
$fields = array(
'id5'=>$id5,
'id10'=>$id10,
'id173'=>$id173,
'id177'=>$id177,
'id183'=>$id183,
'id184'=>$id184,
'id187'=>$id187,
'id188'=>$id188,
'id190'=>$id190,
'id194'=>$id194,
'id196'=>$id196,
'id197'=>$id197,
'id198'=>$id198,
'id199'=>$id199,
'id231'=>$id231,
'id233'=>$id233,
'completed'=>$completed,
'interrupted'=>$interrupted,
'readfailure'=>$read_failure,
'unknownfail'=>$unknown_failure,
'extended'=>$extended,
'short'=>$short,
'conveyance'=>$conveyance,
'selective'=>$selective
);
$fields = [
'id5' => is_numeric($id5) ? $id5 : null,
'id10' => is_numeric($id10) ? $id10 : null,
'id173' => is_numeric($id173) ? $id173 : null,
'id177' => is_numeric($id177) ? $id177 : null,
'id183' => is_numeric($id183) ? $id183 : null,
'id184' => is_numeric($id184) ? $id184 : null,
'id187' => is_numeric($id187) ? $id187 : null,
'id188' => is_numeric($id188) ? $id188 : null,
'id190' => is_numeric($id190) ? $id190 : null,
'id194' => is_numeric($id194) ? $id194 : null,
'id196' => is_numeric($id196) ? $id196 : null,
'id197' => is_numeric($id197) ? $id197 : null,
'id198' => is_numeric($id198) ? $id198 : null,
'id199' => is_numeric($id199) ? $id199 : null,
'id231' => is_numeric($id231) ? $id231 : null,
'id233' => is_numeric($id233) ? $id233 : null,
'completed' => is_numeric($completed) ? $completed : null,
'interrupted' => is_numeric($interrupted) ? $interrupted : null,
'readfailure' => is_numeric($read_failure) ? $read_failure : null,
'unknownfail' => is_numeric($unknown_failure) ? $unknown_failure : null,
'extended' => is_numeric($extended) ? $extended : null,
'short' => is_numeric($short) ? $short : null,
'conveyance' => is_numeric($conveyance) ? $conveyance : null,
'selective' => is_numeric($selective) ? $selective : null
];
$metrics[$disk] = $fields;
$tags = array('name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name);
@@ -146,7 +97,7 @@ while (isset($lines[$int])) {
$fields = ['id9' => $id9];
$metrics[$disk]['id9'] = $id9;
$tags = array('name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name);
data_update($device, 'app', $tags, $fields);

2
includes/polling/ipSystemStats.inc.php
View File

@@ -105,7 +105,7 @@ if ($data) {
if (strstr($stats[$oid], 'No') || strstr($stats[$oid], 'd') || strstr($stats[$oid], 's')) {
$stats[$oid] = '0';
}
$fields[$oid] = $stats[$oid];
$fields[$oid_ds] = $stats[$oid];
}
$tags = compact('af', 'rrd_name', 'rrd_def');

2
includes/polling/netstats/netstats-icmp.inc.php
View File

@@ -43,7 +43,7 @@ if (!starts_with($device['os'], array('Snom', 'asa'))) {
$fields = array();
foreach ($oids as $oid) {
$rrd_def->addDataset($oid, 'COUNTER', null, 100000000000);
$fields[$oid] = isset($data[$oid]) ? $data[$oid] : 'U';
$fields[substr($oid, 0, 19)] = isset($data[$oid]) ? $data[$oid] : 'U';
}
$tags = compact('rrd_def');

2
includes/polling/netstats/netstats-snmp.inc.php
View File

@@ -45,7 +45,7 @@ if ($device['os'] != 'Snom') {
$fields = array();
foreach ($oids as $oid) {
$rrd_def->addDataset($oid, 'COUNTER', null, 100000000000);
$fields[$oid] = isset($data[0][$oid]) ? $data[0][$oid] : 'U';
$fields[substr($oid, 0, 19)] = isset($data[0][$oid]) ? $data[0][$oid] : 'U';
}
$tags = compact('rrd_def');

25
includes/polling/ports.inc.php
View File

@@ -823,16 +823,7 @@ foreach ($ports as $port) {
'OUTMULTICASTPKTS' => $this_port['ifOutMulticastPkts'],
);
if ($tune_port === true) {
rrdtool_tune('port', $rrdfile, $this_port['ifSpeed']);
}
$port_descr_type = $port['port_descr_type'];
$ifName = $port['ifName'];
$ifAlias = $port['ifAlias'];
$tags = compact('ifName', 'ifAlias', 'port_descr_type', 'rrd_name', 'rrd_def');
rrdtool_data_update($device, 'ports', $tags, $fields);
// non rrd stats (will be filtered)
$fields['ifInUcastPkts_rate'] = $port['ifInUcastPkts_rate'];
$fields['ifOutUcastPkts_rate'] = $port['ifOutUcastPkts_rate'];
$fields['ifInErrors_rate'] = $port['ifInErrors_rate'];
@@ -844,10 +835,16 @@ foreach ($ports as $port) {
$fields['ifInBits_rate'] = $port['stats']['ifInBits_rate'];
$fields['ifOutBits_rate'] = $port['stats']['ifOutBits_rate'];
prometheus_push($device, 'ports', rrd_array_filter($tags), $fields);
influx_update($device, 'ports', rrd_array_filter($tags), $fields);
graphite_update($device, 'ports|' . $ifName, $tags, $fields);
opentsdb_update($device, 'port', array('ifName' => $this_port['ifName'], 'ifIndex' => getPortRrdName($port_id)), $fields);
if ($tune_port === true) {
Rrd::tune('port', $rrdfile, $this_port['ifSpeed']);
}
$port_descr_type = $port['port_descr_type'];
$ifName = $port['ifName'];
$ifAlias = $port['ifAlias'];
$tags = compact('ifName', 'ifAlias', 'port_descr_type', 'rrd_name', 'rrd_def');
app('Datastore')->put($device, 'ports', $tags, $fields);
// End Update IF-MIB
// Update PAgP
if ($this_port['pagpOperationMode'] || $port['pagpOperationMode']) {

2
includes/polling/ports/cisco-if-extension.inc.php
View File

@@ -49,5 +49,5 @@ if (isset($this_port['cieIfInRuntsErrs'])) {
*/
$ifName = $port['ifName'];
$tags = compact('ifName', 'rrd_name', 'rrd_def');
rrdtool_data_update($device, 'drops', $tags, $rrd_data);
data_update($device, 'drops', $tags, $rrd_data);
}

75
includes/prometheus.inc.php
View File

@@ -1,75 +0,0 @@
<?php
use LibreNMS\Config;
/*
* LibreNMS
*
* Copyright (c) 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation, either version 3 of the License, or (at your
* option) any later version. Please see LICENSE.txt at the top level of
* the source code distribution for details.
*/
function prometheus_push($device, $measurement, $tags, $fields)
{
global $prometheus;
if (Config::get('prometheus.enable') === true) {
if ($prometheus !== false) {
try {
$ch = curl_init();
set_curl_proxy($ch);
$vals = "";
$promtags = "/measurement/".$measurement;
foreach ($fields as $k => $v) {
if ($v !== null) {
$vals = $vals . "$k $v\n";
}
}
foreach ($tags as $t => $v) {
if ($v !== null) {
$promtags = $promtags . "/$t/$v";
}
}
$promurl = Config::get('prometheus.url') . '/metrics/job/' . Config::get('prometheus.job') . '/instance/' . $device['hostname'] . $promtags;
$promurl = str_replace(" ", "-", $promurl); // Prometheus doesn't handle tags with spaces in url
d_echo("\nPrometheus data:\n");
d_echo($measurement);
d_echo($tags);
d_echo($fields);
d_echo($vals);
d_echo($promurl);
d_echo("\nEND\n");
curl_setopt($ch, CURLOPT_URL, $promurl);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $vals);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$headers = array();
$headers[] = "Content-Type: test/plain";
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_exec($ch);
if (curl_errno($ch)) {
d_echo('Error:' . curl_error($ch));
}
} catch (Exception $e) {
d_echo("Caught exception: " . $e->getMessage() . PHP_EOL);
d_echo($e->getTrace());
}
} else {
c_echo("[%gPrometheus Push Disabled%n]\n");
}//end if
}//end if
}// end prometheus_push

339
includes/rrdtool.inc.php
View File

@@ -24,71 +24,6 @@
* @author Tony Murray <murraytony@gmail.com>
*/
use LibreNMS\Config;
use LibreNMS\Exceptions\FileExistsException;
use LibreNMS\Proc;
/**
* Opens up a pipe to RRDTool using handles provided
*
* @param bool $dual_process start an additional process that's output should be read after every command
* @return bool the process(s) have been successfully started
*/
function rrdtool_initialize($dual_process = true)
{
global $rrd_sync_process, $rrd_async_process;
$command = Config::get('rrdtool') . ' -';
$descriptor_spec = array(
0 => array('pipe', 'r'), // stdin is a pipe that the child will read from
1 => array('pipe', 'w'), // stdout is a pipe that the child will write to
2 => array('pipe', 'w'), // stderr is a pipe that the child will write to
);
$cwd = Config::get('rrd_dir');
if (!rrdtool_running($rrd_sync_process)) {
$rrd_sync_process = new Proc($command, $descriptor_spec, $cwd);
}
if ($dual_process && !rrdtool_running($rrd_async_process)) {
$rrd_async_process = new Proc($command, $descriptor_spec, $cwd);
$rrd_async_process->setSynchronous(false);
}
return rrdtool_running($rrd_sync_process) && ($dual_process ? rrdtool_running($rrd_async_process) : true);
}
/**
* Checks if the variable is a running rrdtool process
*
* @param $process
* @return bool
*/
function rrdtool_running(&$process)
{
return isset($process) && $process instanceof Proc && $process->isRunning();
}
/**
* Close all open rrdtool processes.
* This should be done before exiting a script that has called rrdtool_initilize()
*/
function rrdtool_close()
{
global $rrd_sync_process, $rrd_async_process;
/** @var Proc $rrd_sync_process */
/** @var Proc $rrd_async_process */
if (rrdtool_running($rrd_sync_process)) {
$rrd_sync_process->close('quit');
}
if (rrdtool_running($rrd_async_process)) {
$rrd_async_process->close('quit');
}
}
/**
* Generates a graph file at $graph_file using $options
* Opens its own rrdtool pipe.
@@ -99,123 +34,10 @@ function rrdtool_close()
*/
function rrdtool_graph($graph_file, $options)
{
global $debug, $rrd_sync_process;
/** @var Proc $rrd_sync_process */
if (rrdtool_initialize(false)) {
$cmd = rrdtool_build_command('graph', $graph_file, $options);
$output = implode($rrd_sync_process->sendCommand($cmd));
if ($debug) {
echo "<p>$cmd</p>";
echo "<p>command returned ($output)</p>";
}
return $output;
} else {
return 0;
}
return Rrd::graph($graph_file, $options);
}
/**
* Generates and pipes a command to rrdtool
*
* @internal
* @param string $command create, update, updatev, graph, graphv, dump, restore, fetch, tune, first, last, lastupdate, info, resize, xport, flushcached
* @param string $filename The full patth to the rrd file
* @param string $options rrdtool command options
* @return array the output of stdout and stderr in an array
* @throws FileExistsException thrown when a create command is set to rrdtool < 1.4 and the rrd already exists
* @throws Exception thrown when the rrdtool process(s) cannot be started
*/
function rrdtool($command, $filename, $options)
{
global $debug, $vdebug, $rrd_async_process, $rrd_sync_process;
/** @var Proc $rrd_sync_process */
/** @var Proc $rrd_async_process */
$start_time = microtime(true);
try {
$cmd = rrdtool_build_command($command, $filename, $options);
} catch (FileExistsException $e) {
c_echo('RRD[%g' . $filename . " already exists%n]\n", $debug);
return array(null, null);
}
c_echo("RRD[%g$cmd%n]\n", $debug);
// do not write rrd files, but allow read-only commands
$ro_commands = array('graph', 'graphv', 'dump', 'fetch', 'first', 'last', 'lastupdate', 'info', 'xport');
if (!empty(Config::get('norrd')) && !in_array($command, $ro_commands)) {
c_echo('[%rRRD Disabled%n]', !Config::get('hide_rrd_disabled'));
return array(null, null);
}
// send the command!
if ($command == 'last' && rrdtool_initialize(false)) {
// send this to our synchronous process so output is guaranteed
$output = $rrd_sync_process->sendCommand($cmd);
} elseif (rrdtool_initialize()) {
// don't care about the return of other commands, so send them to the faster async process
$output = $rrd_async_process->sendCommand($cmd);
} else {
throw new Exception('rrdtool could not start');
}
if ($vdebug) {
echo 'RRDtool Output: ';
echo $output[0];
echo $output[1];
}
recordRrdStatistic($command, $start_time);
return $output;
}
/**
* Build a command for rrdtool
* Shortens the filename as needed
* Determines if --daemon and -O should be used
*
* @internal
* @param string $command The base rrdtool command. Usually create, update, last.
* @param string $filename The full path to the rrd file
* @param string $options Options for the command possibly including the rrd definition
* @return string returns a full command ready to be piped to rrdtool
* @throws FileExistsException if rrdtool <1.4.3 and the rrd file exists locally
*/
function rrdtool_build_command($command, $filename, $options)
{
if ($command == 'create') {
// <1.4.3 doesn't support -O, so make sure the file doesn't exist
if (version_compare(Config::get('rrdtool_version', '1.4'), '1.4.3', '<')) {
if (is_file($filename)) {
throw new FileExistsException();
}
} else {
$options .= ' -O';
}
}
// no remote for create < 1.5.5 and tune < 1.5
$rrdtool_version = Config::get('rrdtool_version', '1.4');
if (Config::get('rrdcached') &&
!($command == 'create' && version_compare($rrdtool_version, '1.5.5', '<')) &&
!($command == 'tune' && Config::get('rrdcached') && version_compare($rrdtool_version, '1.5', '<'))
) {
// only relative paths if using rrdcached
$filename = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $filename);
$options = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $options);
return "$command $filename $options --daemon " . Config::get('rrdcached');
}
return "$command $filename $options";
}
/**
* Checks if the rrd file exists on the server
* This will perform a remote check if using rrdcached and rrdtool >= 1.5
@@ -225,46 +47,9 @@ function rrdtool_build_command($command, $filename, $options)
*/
function rrdtool_check_rrd_exists($filename)
{
if (Config::get('rrdcached') && version_compare(Config::get('rrdtool_version', '1.4'), '1.5', '>=')) {
$chk = rrdtool('last', $filename, '');
$filename = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $filename);
return !str_contains(implode($chk), "$filename': No such file or directory");
} else {
return is_file($filename);
}
return Rrd::checkRrdExists($filename);
}
/**
* Updates an rrd database at $filename using $options
* Where $options is an array, each entry which is not a number is replaced with "U"
*
* @internal
* @param string $filename
* @param array $data
* @return array|string
*/
function rrdtool_update($filename, $data)
{
$values = array();
// Do some sanitation on the data if passed as an array.
if (is_array($data)) {
$values[] = 'N';
foreach ($data as $v) {
if (!is_numeric($v)) {
$v = 'U';
}
$values[] = $v;
}
$data = implode(':', $values);
return rrdtool('update', $filename, $data);
} else {
return 'Bad options passed to rrdtool_update';
}
} // rrdtool_update
/**
* Escapes strings for RRDtool
@@ -312,11 +97,9 @@ function rrdtool_escape($string, $length = null)
*/
function rrd_name($host, $extra, $extension = ".rrd")
{
$filename = safename(is_array($extra) ? implode("-", $extra) : $extra);
return implode("/", array(get_rrd_dir($host), $filename.$extension));
return Rrd::name($host, $extra, $extension);
} // rrd_name
/**
* Generates a path based on the hostname (or IP)
*
@@ -329,107 +112,6 @@ function get_rrd_dir($host)
return implode("/", [Config::get('rrd_dir'), $host]);
} // rrd_dir
/**
* Generates a filename for a proxmox cluster rrd
*
* @param $pmxcluster
* @param $vmid
* @param $vmport
* @return string full path to the rrd.
*/
function proxmox_rrd_name($pmxcluster, $vmid, $vmport)
{
$pmxcdir = join('/', [Config::get('rrd_dir'), 'proxmox', safename($pmxcluster)]);
// this is not needed for remote rrdcached
if (!is_dir($pmxcdir)) {
mkdir($pmxcdir, 0775, true);
}
return join('/', array($pmxcdir, safename($vmid.'_netif_'.$vmport.'.rrd')));
}
/**
* Modify an rrd file's max value and trim the peaks as defined by rrdtool
*
* @param string $type only 'port' is supported at this time
* @param string $filename the path to the rrd file
* @param integer $max the new max value
* @return bool
*/
function rrdtool_tune($type, $filename, $max)
{
$fields = array();
if ($type === 'port') {
if ($max < 10000000) {
return false;
}
$max = $max / 8;
$fields = array(
'INOCTETS',
'OUTOCTETS',
'INERRORS',
'OUTERRORS',
'INUCASTPKTS',
'OUTUCASTPKTS',
'INNUCASTPKTS',
'OUTNUCASTPKTS',
'INDISCARDS',
'OUTDISCARDS',
'INUNKNOWNPROTOS',
'INBROADCASTPKTS',
'OUTBROADCASTPKTS',
'INMULTICASTPKTS',
'OUTMULTICASTPKTS'
);
}
if (count($fields) > 0) {
$options = "--maximum " . implode(":$max --maximum ", $fields) . ":$max";
rrdtool('tune', $filename, $options);
}
return true;
} // rrdtool_tune
/**
* rrdtool backend implementation of data_update
*
* Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device device array
* @param string $measurement the name of this measurement (if no rrd_name tag is given, this will be used to name the file)
* @param array $tags tags to pass additional info to rrdtool
* @param array $fields data values to update
*/
function rrdtool_data_update($device, $measurement, $tags, $fields)
{
$rrd_name = $tags['rrd_name'] ?: $measurement;
$step = $tags['rrd_step'] ?: Config::get('rrd.step');
$oldname = $tags['rrd_oldname'];
if (!empty($oldname)) {
rrd_file_rename($device, $oldname, $rrd_name);
}
if (isset($tags['rrd_proxmox_name'])) {
$pmxvars = $tags['rrd_proxmox_name'];
$rrd = proxmox_rrd_name($pmxvars['pmxcluster'], $pmxvars['vmid'], $pmxvars['vmport']);
} else {
$rrd = rrd_name($device['hostname'], $rrd_name);
}
if (isset($tags['rrd_def']) && !rrdtool_check_rrd_exists($rrd)) {
$newdef = "--step $step " . $tags['rrd_def'] . Config::get('rrd_rra');
rrdtool('create', $rrd, $newdef);
}
rrdtool_update($rrd, $fields);
} // rrdtool_data_update
/**
* rename an rrdfile, can only be done on the LibreNMS server hosting the rrd files
*
@@ -440,18 +122,5 @@ function rrdtool_data_update($device, $measurement, $tags, $fields)
*/
function rrd_file_rename($device, $oldname, $newname)
{
$oldrrd = rrd_name($device['hostname'], $oldname);
$newrrd = rrd_name($device['hostname'], $newname);
if (is_file($oldrrd) && !is_file($newrrd)) {
if (rename($oldrrd, $newrrd)) {
log_event("Renamed $oldrrd to $newrrd", $device, "poller", 1);
return true;
} else {
log_event("Failed to rename $oldrrd to $newrrd", $device, "poller", 5);
return false;
}
} else {
// we don't need to rename the file
return true;
}
return Rrd::renameFile($device, $oldname, $newname);
} // rrd_file_rename