mirror/librenms-librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-10-07 16:52:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor Datastores to allow future improvements. OpenTSDB Tags. (#11283)

Browse Source 
* Datastores to object oriented code, using the Laravel IoC container

Change instantiation
better DI
move OpenTSDB

Small re-orgs

remove unused stuff

Fix graphs and other scripts

Use DI for all except rrd

fix up connection error handling

Add tests, fix up a "few" things
Add Config::forget()

Style fixes

Don't reference legacy code

remove accidental code paste

Add datastores phpunit groups

some tests

* rebase fixes

* some test fixes

* shorter tests

* shorter tests

* Don't except when rrdtool can't be started.

* restore tests

* fix rrd tests

* fix iterable change upstream

* fix isValidDataset

* fix invalid data bug

* fix mysql incorrect ds

* fix issue with data that is too long

* use regular data_update()

* Use log facade

* OpenTSDB mis-ordered arguments fix

* Making a singleton with different options makes different singletons.  Just use the global config settings to disable datastores.

* only filter tags for datastores that won't it don't modify the tags permanently

* Update copyrights to include original authors.

* Stats for all datastores

* Fix mysql sends different rrd / other ds names

* fix snmp last stats not initialized
remove unused function

* remove unused function and move single use function closer to its use

* InfluxDB does not need to update null or U values.
Skip write if all fields are empty

* Fix smart value checks

* fix style issues

* Make sure port data is stored the same way as before for Graphite and OpenTSDB
Add ifIndex tag to all to be compatible

* Missed rrdtool_tune() call

* Test update WIP

* OpenTSDB now includes tags

* fix style
This commit is contained in:
Tony Murray
2020-03-16 09:17:58 -05:00
committed by GitHub
parent 5deda8646a
commit df16de9d2f
52 changed files with 2531 additions and 1008 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
includes/functions.php
View File

@@ -344,26 +344,6 @@ function percent_colour($perc)
return sprintf('#%02x%02x%02x', $r, $b, $b);
}
// Returns the last in/out errors value in RRD
function interface_errors($rrd_file, $period = '-1d')
{
$errors = array();
$cmd = Config::get('rrdtool') . " fetch -s $period -e -300s $rrd_file AVERAGE | grep : | cut -d\" \" -f 4,5";
$data = trim(shell_exec($cmd));
$in_errors = 0;
$out_errors = 0;
foreach (explode("\n", $data) as $entry) {
list($in, $out) = explode(" ", $entry);
$in_errors += ($in * 300);
$out_errors += ($out * 300);
}
$errors['in'] = round($in_errors);
$errors['out'] = round($out_errors);
return $errors;
}
/**
* @param $device
* @return string the logo image path for this device. Images are often wide, not square.
@@ -1542,30 +1522,6 @@ function function_check($function)
return function_exists($function);
}
function force_influx_data($data)
{
/*
* It is not trivial to detect if something is a float or an integer, and
* therefore may cause breakages on inserts.
* Just setting every number to a float gets around this, but may introduce
* inefficiencies.
* I've left the detection statement in there for a possible change in future,
* but currently everything just gets set to a float.
*/
if (is_numeric($data)) {
// If it is an Integer
if (ctype_digit($data)) {
return floatval($data);
// Else it is a float
} else {
return floatval($data);
}
} else {
return $data;
}
}// end force_influx_data
/**
* Try to determine the address family (IPv4 or IPv6) associated with an SNMP
* transport specifier (like "udp", "udp6", etc.).
@@ -1662,50 +1618,6 @@ function dnslookup($device, $type = false, $return = false)
return $record[0][$return];
}//end dnslookup
/**
* Run rrdtool info on a file path
*
* @param string $path Path to pass to rrdtool info
* @param string $stdOutput Variable to recieve the output of STDOUT
* @param string $stdError Variable to recieve the output of STDERR
*
* @return int exit code
*
**/
function rrdtest($path, &$stdOutput, &$stdError)
{
//rrdtool info <escaped rrd path>
$command = Config::get('rrdtool') . ' info ' . escapeshellarg($path);
$process = proc_open(
$command,
array (
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('pipe', 'w'),
),
$pipes
);
if (!is_resource($process)) {
throw new \RuntimeException('Could not create a valid process');
}
$status = proc_get_status($process);
while ($status['running']) {
usleep(2000); // Sleep 2000 microseconds or 2 milliseconds
$status = proc_get_status($process);
}
$stdOutput = stream_get_contents($pipes[1]);
$stdError = stream_get_contents($pipes[2]);
proc_close($process);
return $status['exitcode'];
}
/**
* Create a new state index. Update translations if $states is given.
*
@@ -2062,10 +1974,9 @@ function get_toner_levels($device, $raw_value, $capacity)
*/
function initStats()
{
global $snmp_stats, $rrd_stats;
global $snmp_stats_last, $rrd_stats_last;
global $snmp_stats, $snmp_stats_last;
if (!isset($snmp_stats, $rrd_stats)) {
if (!isset($snmp_stats)) {
$snmp_stats = array(
'ops' => array(
'snmpget' => 0,
@@ -2079,20 +1990,6 @@ function initStats()
)
);
$snmp_stats_last = $snmp_stats;
$rrd_stats = array(
'ops' => array(
'update' => 0,
'create' => 0,
'other' => 0,
),
'time' => array(
'update' => 0.0,
'create' => 0.0,
'other' => 0.0,
),
);
$rrd_stats_last = $rrd_stats;
}
}
@@ -2103,25 +2000,29 @@ function initStats()
*/
function printChangedStats($update_only = false)
{
global $snmp_stats, $db_stats, $rrd_stats;
global $snmp_stats_last, $db_stats_last, $rrd_stats_last;
global $snmp_stats, $db_stats;
global $snmp_stats_last, $db_stats_last;
$output = sprintf(
">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs]",
array_sum($snmp_stats['ops']) - array_sum($snmp_stats_last['ops']),
array_sum($snmp_stats['time']) - array_sum($snmp_stats_last['time']),
array_sum($db_stats['ops']) - array_sum($db_stats_last['ops']),
array_sum($db_stats['time']) - array_sum($db_stats_last['time'])
);
foreach (app('Datastore')->getStats() as $datastore => $stats) {
/** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
$output .= sprintf(" %s: [%d/%.2fs]", $datastore, $stats->getCountDiff(), $stats->getDurationDiff());
$stats->checkpoint();
}
if (!$update_only) {
printf(
">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs] RRD: [%d/%.2fs]\n",
array_sum($snmp_stats['ops']) - array_sum($snmp_stats_last['ops']),
array_sum($snmp_stats['time']) - array_sum($snmp_stats_last['time']),
array_sum($db_stats['ops']) - array_sum($db_stats_last['ops']),
array_sum($db_stats['time']) - array_sum($db_stats_last['time']),
array_sum($rrd_stats['ops']) - array_sum($rrd_stats_last['ops']),
array_sum($rrd_stats['time']) - array_sum($rrd_stats_last['time'])
);
echo $output . PHP_EOL;
}
// make a new checkpoint
$snmp_stats_last = $snmp_stats;
$db_stats_last = $db_stats;
$rrd_stats_last = $rrd_stats;
}
/**
@@ -2129,7 +2030,7 @@ function printChangedStats($update_only = false)
*/
function printStats()
{
global $snmp_stats, $db_stats, $rrd_stats;
global $snmp_stats, $db_stats;
if ($snmp_stats) {
printf(
@@ -2167,42 +2068,18 @@ function printStats()
);
}
if ($rrd_stats) {
printf(
"RRD [%d/%.2fs]: Update[%d/%.2fs] Create [%d/%.2fs] Other[%d/%.2fs]\n",
array_sum($rrd_stats['ops']),
array_sum($rrd_stats['time']),
$rrd_stats['ops']['update'],
$rrd_stats['time']['update'],
$rrd_stats['ops']['create'],
$rrd_stats['time']['create'],
$rrd_stats['ops']['other'],
$rrd_stats['time']['other']
);
foreach (app('Datastore')->getStats() as $datastore => $stats) {
/** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
printf("%s [%d/%.2fs]:", $datastore, $stats->getTotalCount(), $stats->getTotalDuration());
foreach ($stats as $stat) {
/** @var \LibreNMS\Data\Measure\MeasurementSummary $stat */
printf(" %s[%d/%.2fs]", ucfirst($stat->getType()), $stat->getCount(), $stat->getDuration());
}
echo PHP_EOL;
}
}
/**
* Update statistics for rrd operations
*
* @param string $stat create, update, and other
* @param float $start_time The time the operation started with 'microtime(true)'
* @return float The calculated run time
*/
function recordRrdStatistic($stat, $start_time)
{
global $rrd_stats;
initStats();
$stat = ($stat == 'update' || $stat == 'create') ? $stat : 'other';
$runtime = microtime(true) - $start_time;
$rrd_stats['ops'][$stat]++;
$rrd_stats['time'][$stat] += $runtime;
return $runtime;
}
/**
* @param string $stat snmpget, snmpwalk
* @param float $start_time The time the operation started with 'microtime(true)'