mirror/librenms-librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-10-07 16:52:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor Datastores to allow future improvements. OpenTSDB Tags. (#11283)

Browse Source 
* Datastores to object oriented code, using the Laravel IoC container

Change instantiation
better DI
move OpenTSDB

Small re-orgs

remove unused stuff

Fix graphs and other scripts

Use DI for all except rrd

fix up connection error handling

Add tests, fix up a "few" things
Add Config::forget()

Style fixes

Don't reference legacy code

remove accidental code paste

Add datastores phpunit groups

some tests

* rebase fixes

* some test fixes

* shorter tests

* shorter tests

* Don't except when rrdtool can't be started.

* restore tests

* fix rrd tests

* fix iterable change upstream

* fix isValidDataset

* fix invalid data bug

* fix mysql incorrect ds

* fix issue with data that is too long

* use regular data_update()

* Use log facade

* OpenTSDB mis-ordered arguments fix

* Making a singleton with different options makes different singletons.  Just use the global config settings to disable datastores.

* only filter tags for datastores that won't it don't modify the tags permanently

* Update copyrights to include original authors.

* Stats for all datastores

* Fix mysql sends different rrd / other ds names

* fix snmp last stats not initialized
remove unused function

* remove unused function and move single use function closer to its use

* InfluxDB does not need to update null or U values.
Skip write if all fields are empty

* Fix smart value checks

* fix style issues

* Make sure port data is stored the same way as before for Graphite and OpenTSDB
Add ifIndex tag to all to be compatible

* Missed rrdtool_tune() call

* Test update WIP

* OpenTSDB now includes tags

* fix style
This commit is contained in:
Tony Murray
2020-03-16 09:17:58 -05:00
committed by GitHub
parent 5deda8646a
commit df16de9d2f
52 changed files with 2531 additions and 1008 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
LibreNMS/Data/Store/BaseDatastore.php Normal file
View File

@@ -0,0 +1,64 @@
<?php
/**
* BaseDatastore.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use LibreNMS\Data\Measure\Measurement;
use LibreNMS\Data\Measure\MeasurementCollection;
use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
abstract class BaseDatastore implements DatastoreContract
{
private $stats;
public function __construct()
{
$this->stats = new MeasurementCollection();
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return true;
}
/**
* Record statistics for operation
* @param Measurement $stat
*/
protected function recordStatistic(Measurement $stat)
{
$this->stats->record($stat);
}
public function getStats()
{
return $this->stats;
}
}

159
LibreNMS/Data/Store/Datastore.php Normal file
View File

@@ -0,0 +1,159 @@
<?php
/**
* Datastore.php
*
* Aggregates all enabled datastores and dispatches data to them
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use LibreNMS\Config;
use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
class Datastore
{
protected $stores;
/**
* Initialize and create the Datastore(s)
*
* @param array $options
* @return DatastoreContract
*/
public static function init($options = [])
{
$opts = [
'r' => 'rrd.enable',
'f' => 'influxdb.enable',
'p' => 'prometheus.enable',
'g' => 'graphite.enable',
];
foreach ($opts as $opt => $setting) {
if (isset($options[$opt])) {
Config::set($setting, false);
}
}
return app('Datastore');
}
public static function terminate()
{
\Rrd::close();
}
/**
* Datastore constructor.
* @param array $datastores Implement DatastoreInterface
*/
public function __construct($datastores)
{
$this->stores = $datastores;
}
/**
* Disable a datastore for the rest of this run
*
* @param string $name
*/
public function disable($name)
{
$store = app("LibreNMS\\Data\\Store\\$name");
$position = array_search($store, $this->stores);
if ($position !== false) {
c_echo("[%g$name Disabled%n]\n");
unset($this->stores[$position]);
} else {
c_echo("[%g$name is not a valid datastore name%n]\n");
}
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
// convenience conversion to allow calling with a single value, so, e.g., these are equivalent:
// data_update($device, 'mymeasurement', $tags, 1234);
// AND
// data_update($device, 'mymeasurement', $tags, array('mymeasurement' => 1234));
if (!is_array($fields)) {
$fields = [$measurement => $fields];
}
foreach ($this->stores as $store) {
/** @var DatastoreContract $store */
// rrdtool_data_update() will only use the tags it deems relevant, so we pass all of them.
// However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
$temp_tags = $store->wantsRrdTags() ? $tags : $this->rrdTagFilter($tags);
$store->put($device, $measurement, $temp_tags, $fields);
}
}
/**
* Filter all elements with keys that start with 'rrd_'
*
* @param array $arr input array
* @return array Copy of $arr with all keys beginning with 'rrd_' removed.
*/
private function rrdTagFilter($arr)
{
$result = [];
foreach ($arr as $k => $v) {
if (strpos($k, 'rrd_') === 0) {
continue;
}
$result[$k] = $v;
}
return $result;
}
/**
* Get all the active data stores
*
* @return array
*/
public function getStores()
{
return $this->stores;
}
public function getStats()
{
return array_reduce($this->stores, function ($result, DatastoreContract $store) {
$result[$store->getName()] = $store->getStats();
return $result;
}, []);
}
}

146
LibreNMS/Data/Store/Graphite.php Normal file
View File

@@ -0,0 +1,146 @@
<?php
/**
* Graphite.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2017 Falk Stern <https://github.com/fstern/>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use Carbon\Carbon;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class Graphite extends BaseDatastore
{
protected $connection;
protected $prefix;
public function __construct(\Socket\Raw\Factory $socketFactory)
{
parent::__construct();
$host = Config::get('graphite.host');
$port = Config::get('graphite.port', 2003);
try {
$this->connection = $socketFactory->createClient("$host:$port");
} catch (\Exception $e) {
d_echo($e->getMessage());
}
if ($this->connection) {
Log::notice("Graphite connection made to $host");
} else {
Log::error("Graphite connection to $host has failed!");
}
$this->prefix = Config::get('graphite.prefix', '');
}
public function getName()
{
return 'Graphite';
}
public static function isEnabled()
{
return Config::get('graphite.enable', false);
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
if (!$this->connection) {
d_echo("Graphite Error: not connected\n");
return;
}
$timestamp = Carbon::now()->timestamp;
if ($measurement == 'ports') {
$measurement = 'ports|' . $tags['ifName'];
}
// metrics will be built as prefix.hostname.measurement.field value timestamp
// metric fields can not contain . as this is used by graphite as a field separator
$hostname = preg_replace('/\./', '_', $device['hostname']);
$measurement = preg_replace(['/\./', '/\//'], '_', $measurement);
$measurement = preg_replace('/\|/', '.', $measurement);
$measurement_name = preg_replace('/\./', '_', $tags['rrd_name']);
if (is_array($measurement_name)) {
$ms_name = implode(".", $measurement_name);
} else {
$ms_name = $measurement_name;
}
// remove the port-id tags from the metric
if (preg_match('/^port-id\d+/', $ms_name)) {
$ms_name = "";
}
foreach ($fields as $k => $v) {
// Send zero for fields without values
if (empty($v)) {
$v = 0;
}
$metric = implode(".", array_filter([$this->prefix, $hostname, $measurement, $ms_name, $k]));
$this->writeData($metric, $v, $timestamp);
}
}
/**
* @param $metric
* @param $value
* @param $timestamp
*/
private function writeData($metric, $value, $timestamp)
{
try {
$stat = Measurement::start('write');
// Further sanitize the full metric before sending, whitespace isn't allowed
$metric = preg_replace('/\s+/', '_', $metric);
$line = implode(" ", [$metric, $value, $timestamp]);
Log::debug("Sending to Graphite: $line\n");
$this->connection->write("$line\n");
$this->recordStatistic($stat->end());
} catch (\Socket\Raw\Exception $e) {
Log::error('Graphite write error: ' . $e->getMessage());
}
}
}

189
LibreNMS/Data/Store/InfluxDB.php Normal file
View File

@@ -0,0 +1,189 @@
<?php
/**
* InfluxDB.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use InfluxDB\Client;
use InfluxDB\Driver\UDP;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class InfluxDB extends BaseDatastore
{
/** @var \InfluxDB\Database $connection */
private $connection;
public function __construct(\InfluxDB\Database $influx)
{
parent::__construct();
$this->connection = $influx;
// if the database doesn't exist, create it.
try {
if (!$influx->exists()) {
$influx->create();
}
} catch (\Exception $e) {
Log::warning('InfluxDB: Could not create database');
}
}
public function getName()
{
return 'InfluxDB';
}
public static function isEnabled()
{
return Config::get('influxdb.enable', false);
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
$stat = Measurement::start('write');
$tmp_fields = [];
$tmp_tags['hostname'] = $device['hostname'];
foreach ($tags as $k => $v) {
$v = preg_replace(['/ /', '/,/', '/=/'], ['\ ', '\,', '\='], $v);
if (empty($v)) {
$v = '_blank_';
}
$tmp_tags[$k] = $v;
}
foreach ($fields as $k => $v) {
if ($k == 'time') {
$k = 'rtime';
}
if (($value = $this->forceType($v)) !== null) {
$tmp_fields[$k] = $value;
}
}
if (empty($tmp_fields)) {
Log::warning('All fields empty, skipping update', ['orig_fields' => $fields]);
return;
}
Log::debug('InfluxDB data: ', [
'measurement' => $measurement,
'tags' => $tmp_tags,
'fields' => $tmp_fields,
]);
try {
$points = [
new \InfluxDB\Point(
$measurement,
null, // the measurement value
$tmp_tags,
$tmp_fields // optional additional fields
)
];
$this->connection->writePoints($points);
$this->recordStatistic($stat->end());
} catch (\Exception $e) {
Log::error('InfluxDB exception: ' . $e->getMessage());
Log::debug($e->getTraceAsString());
}
}
/**
* Create a new client and select the database
*
* @return \InfluxDB\Database
*/
public static function createFromConfig()
{
$host = Config::get('influxdb.host', 'localhost');
$transport = Config::get('influxdb.transport', 'http');
$port = Config::get('influxdb.port', 8086);
$db = Config::get('influxdb.db', 'librenms');
$username = Config::get('influxdb.username', '');
$password = Config::get('influxdb.password', '');
$timeout = Config::get('influxdb.timeout', 0);
$verify_ssl = Config::get('influxdb.verifySSL', false);
$client = new Client($host, $port, $username, $password, $transport == 'https', $verify_ssl, $timeout, $timeout);
if ($transport == 'udp') {
$client->setDriver(new UDP($host, $port));
}
return $client->selectDB($db);
}
private function forceType($data)
{
/*
* It is not trivial to detect if something is a float or an integer, and
* therefore may cause breakages on inserts.
* Just setting every number to a float gets around this, but may introduce
* inefficiencies.
* I've left the detection statement in there for a possible change in future,
* but currently everything just gets set to a float.
*/
if (is_numeric($data)) {
// If it is an Integer
if (ctype_digit($data)) {
return floatval($data);
// Else it is a float
} else {
return floatval($data);
}
}
return $data === 'U' ? null : $data;
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return false;
}
}

145
LibreNMS/Data/Store/OpenTSDB.php Normal file
View File

@@ -0,0 +1,145 @@
<?php
/**
* OpenTSDB.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2017 Yacine Benamsili <https://github.com/yac01/librenms.git>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use Carbon\Carbon;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class OpenTSDB extends BaseDatastore
{
/** @var \Socket\Raw\Socket $connection */
protected $connection;
public function __construct(\Socket\Raw\Factory $socketFactory)
{
parent::__construct();
$host = Config::get('opentsdb.host');
$port = Config::get('opentsdb.port', 2181);
try {
$this->connection = $socketFactory->createClient("$host:$port");
} catch (\Socket\Raw\Exception $e) {
Log::debug('OpenTSDB Error: ' . $e->getMessage());
}
if ($this->connection) {
Log::notice('Connected to OpenTSDB');
} else {
Log::error('Connection to OpenTSDB has failed!');
}
}
public function getName()
{
return 'OpenTSDB';
}
/**
* Datastore-independent function which should be used for all polled metrics.
*
* RRD Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device
* @param string $measurement Name of this measurement
* @param array $tags tags for the data (or to control rrdtool)
* @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
* single values are allowed and will be paired with $measurement
*/
public function put($device, $measurement, $tags, $fields)
{
if (!$this->connection) {
Log::error("OpenTSDB Error: not connected\n");
return;
}
$flag = Config::get('opentsdb.co');
$timestamp = Carbon::now()->timestamp;
$tmp_tags = "hostname=".$device['hostname'];
foreach ($tags as $k => $v) {
$v = str_replace(array(' ',',','='), '_', $v);
if (!empty($v)) {
$tmp_tags = $tmp_tags ." ". $k ."=".$v;
}
}
if ($measurement == 'ports') {
foreach ($fields as $k => $v) {
$measurement = $k;
if ($flag == true) {
$measurement = $measurement.".".$device['co'];
}
$this->putData('port.'.$measurement, $timestamp, $v, $tmp_tags);
}
} else {
if ($flag == true) {
$measurement = $measurement . '.' . $device['co'];
}
foreach ($fields as $k => $v) {
$tmp_tags_key = $tmp_tags . " " . "key" . "=" . $k;
$this->putData($measurement, $timestamp, $v, $tmp_tags_key);
}
}
}
private function putData($measurement, $timestamp, $value, $tags)
{
try {
$stat = Measurement::start('put');
$line = sprintf('put net.%s %d %f %s', strtolower($measurement), $timestamp, $value, $tags);
Log::debug("Sending to OpenTSDB: $line\n");
$this->connection->write("$line\n"); // send $line into OpenTSDB
$this->recordStatistic($stat->end());
} catch (\Socket\Raw\Exception $e) {
Log::error('OpenTSDB Error: ' . $e->getMessage());
}
}
public static function isEnabled()
{
return Config::get('opentsdb.enable', false);
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return false;
}
}

132
LibreNMS/Data/Store/Prometheus.php Normal file
View File

@@ -0,0 +1,132 @@
<?php
/**
* Prometheus.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2020 Tony Murray
* @copyright 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use GuzzleHttp\Exception\GuzzleException;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use Log;
class Prometheus extends BaseDatastore
{
private $client;
private $base_uri;
private $default_opts;
private $enabled;
public function __construct(\GuzzleHttp\Client $client)
{
parent::__construct();
$this->client = $client;
$url = Config::get('prometheus.url');
$job = Config::get('prometheus.job', 'librenms');
$this->base_uri = "$url/metrics/job/$job/instance/";
$this->default_opts = [
'headers' => ['Content-Type' => 'text/plain'],
];
if ($proxy = get_proxy()) {
$this->default_opts['proxy'] = $proxy;
}
$this->enabled = self::isEnabled();
}
public function getName()
{
return 'Prometheus';
}
public static function isEnabled()
{
return Config::get('prometheus.enable', false);
}
public function put($device, $measurement, $tags, $fields)
{
$stat = Measurement::start('put');
// skip if needed
if (!$this->enabled) {
return;
}
try {
$vals = "";
$promtags = "/measurement/" . $measurement;
foreach ($fields as $k => $v) {
if ($v !== null) {
$vals .= "$k $v\n";
}
}
foreach ($tags as $t => $v) {
if ($v !== null) {
$promtags .= "/$t/$v";
}
}
$options = $this->getDefaultOptions();
$options['body'] = $vals;
$promurl = $this->base_uri . $device['hostname'] . $promtags;
$promurl = str_replace(" ", "-", $promurl); // Prometheus doesn't handle tags with spaces in url
Log::debug("Prometheus put $promurl: ", [
'measurement' => $measurement,
'tags' => $tags,
'fields' => $fields,
'vals' => $vals,
]);
$result = $this->client->request('POST', $promurl, $options);
$this->recordStatistic($stat->end());
if ($result->getStatusCode() !== 200) {
Log::error('Prometheus Error: ' . $result->getReasonPhrase());
}
} catch (GuzzleException $e) {
Log::error("Prometheus Exception: " . $e->getMessage());
}
}
private function getDefaultOptions()
{
return $this->default_opts;
}
/**
* Checks if the datastore wants rrdtags to be sent when issuing put()
*
* @return boolean
*/
public function wantsRrdTags()
{
return false;
}
}

496
LibreNMS/Data/Store/Rrd.php Normal file
View File

@@ -0,0 +1,496 @@
<?php
/**
* Rrd.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use LibreNMS\Config;
use LibreNMS\Data\Measure\Measurement;
use LibreNMS\Exceptions\FileExistsException;
use LibreNMS\Proc;
use Log;
class Rrd extends BaseDatastore
{
private $disabled = false;
/** @var Proc $sync_process */
private $sync_process;
/** @var Proc $async_process */
private $async_process;
private $rrd_dir;
private $version;
private $rrdcached;
private $rra;
private $step;
public function __construct()
{
parent::__construct();
$this->rrdcached = Config::get('rrdcached', false);
$this->init();
$this->rrd_dir = Config::get('rrd_dir', Config::get('install_dir') . '/rrd');
$this->step = Config::get('rrd.step', 300);
$this->rra = Config::get(
'rrd_rra',
'RRA:AVERAGE:0.5:1:2016 RRA:AVERAGE:0.5:6:1440 RRA:AVERAGE:0.5:24:1440 RRA:AVERAGE:0.5:288:1440 ' .
' RRA:MIN:0.5:1:720 RRA:MIN:0.5:6:1440 RRA:MIN:0.5:24:775 RRA:MIN:0.5:288:797 ' .
' RRA:MAX:0.5:1:720 RRA:MAX:0.5:6:1440 RRA:MAX:0.5:24:775 RRA:MAX:0.5:288:797 ' .
' RRA:LAST:0.5:1:1440 '
);
$this->version = Config::get('rrdtool_version', '1.4');
}
public function getName()
{
return 'RRD';
}
public static function isEnabled()
{
return Config::get('rrd.enable', true);
}
/**
* Opens up a pipe to RRDTool using handles provided
*
* @param bool $dual_process start an additional process that's output should be read after every command
* @return bool the process(s) have been successfully started
*/
public function init($dual_process = true)
{
$command = Config::get('rrdtool', 'rrdtool') . ' -';
$descriptor_spec = [
0 => ['pipe', 'r'], // stdin is a pipe that the child will read from
1 => ['pipe', 'w'], // stdout is a pipe that the child will write to
2 => ['pipe', 'w'], // stderr is a pipe that the child will write to
];
$cwd = Config::get('rrd_dir');
if (!$this->isSyncRunning()) {
$this->sync_process = new Proc($command, $descriptor_spec, $cwd);
}
if ($dual_process && !$this->isAsyncRunning()) {
$this->async_process = new Proc($command, $descriptor_spec, $cwd);
$this->async_process->setSynchronous(false);
}
return $this->isSyncRunning() && ($dual_process ? $this->isAsyncRunning() : true);
}
public function isSyncRunning()
{
return isset($this->sync_process) && $this->sync_process->isRunning();
}
public function isAsyncRunning()
{
return isset($this->async_process) && $this->async_process->isRunning();
}
/**
* Close all open rrdtool processes.
* This should be done before exiting
*/
public function close()
{
if ($this->isSyncRunning()) {
$this->sync_process->close('quit');
}
if ($this->isAsyncRunning()) {
$this->async_process->close('quit');
}
}
/**
* rrdtool backend implementation of data_update
*
* Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device device array
* @param string $measurement the name of this measurement (if no rrd_name tag is given, this will be used to name the file)
* @param array $tags tags to pass additional info to rrdtool
* @param array $fields data values to update
*/
public function put($device, $measurement, $tags, $fields)
{
$rrd_name = isset($tags['rrd_name']) ? $tags['rrd_name'] : $measurement;
$step = isset($tags['rrd_step']) ? $tags['rrd_step'] : $this->step;
if (!empty($tags['rrd_oldname'])) {
self::renameFile($device, $tags['rrd_oldname'], $rrd_name);
}
if (isset($tags['rrd_proxmox_name'])) {
$pmxvars = $tags['rrd_proxmox_name'];
$rrd = self::proxmoxName($pmxvars['pmxcluster'], $pmxvars['vmid'], $pmxvars['vmport']);
} else {
$rrd = self::name($device['hostname'], $rrd_name);
}
if (isset($tags['rrd_def'])) {
$rrd_def = $tags['rrd_def'];
// filter out data not in the definition
$fields = array_filter($fields, function ($key) use ($rrd_def) {
$valid = $rrd_def->isValidDataset($key);
if (!$valid) {
Log::warning("RRD warning: unused data sent $key");
}
return $valid;
}, ARRAY_FILTER_USE_KEY);
if (!$this->checkRrdExists($rrd)) {
$newdef = "--step $step $rrd_def $this->rra";
$this->command('create', $rrd, $newdef);
}
}
$this->update($rrd, $fields);
}
/**
* Updates an rrd database at $filename using $options
* Where $options is an array, each entry which is not a number is replaced with "U"
*
* @internal
* @param string $filename
* @param array $data
* @return array|string
*/
public function update($filename, $data)
{
$values = [];
// Do some sanitation on the data if passed as an array.
if (is_array($data)) {
$values[] = 'N';
foreach ($data as $v) {
if (!is_numeric($v)) {
$v = 'U';
}
$values[] = $v;
}
$data = implode(':', $values);
return $this->command('update', $filename, $data);
} else {
return 'Bad options passed to rrdtool_update';
}
} // rrdtool_update
/**
* Modify an rrd file's max value and trim the peaks as defined by rrdtool
*
* @param string $type only 'port' is supported at this time
* @param string $filename the path to the rrd file
* @param integer $max the new max value
* @return bool
*/
public function tune($type, $filename, $max)
{
$fields = [];
if ($type === 'port') {
if ($max < 10000000) {
return false;
}
$max = $max / 8;
$fields = [
'INOCTETS',
'OUTOCTETS',
'INERRORS',
'OUTERRORS',
'INUCASTPKTS',
'OUTUCASTPKTS',
'INNUCASTPKTS',
'OUTNUCASTPKTS',
'INDISCARDS',
'OUTDISCARDS',
'INUNKNOWNPROTOS',
'INBROADCASTPKTS',
'OUTBROADCASTPKTS',
'INMULTICASTPKTS',
'OUTMULTICASTPKTS'
];
}
if (count($fields) > 0) {
$options = "--maximum " . implode(":$max --maximum ", $fields) . ":$max";
$this->command('tune', $filename, $options);
}
return true;
} // rrdtool_tune
/**
* Generates a filename for a proxmox cluster rrd
*
* @param $pmxcluster
* @param $vmid
* @param $vmport
* @return string full path to the rrd.
*/
public function proxmoxName($pmxcluster, $vmid, $vmport)
{
$pmxcdir = join('/', [$this->rrd_dir, 'proxmox', self::safeName($pmxcluster)]);
// this is not needed for remote rrdcached
if (!is_dir($pmxcdir)) {
mkdir($pmxcdir, 0775, true);
}
return join('/', [$pmxcdir, self::safeName($vmid . '_netif_' . $vmport . '.rrd')]);
}
/**
* rename an rrdfile, can only be done on the LibreNMS server hosting the rrd files
*
* @param array $device Device object
* @param string|array $oldname RRD name array as used with rrd_name()
* @param string|array $newname RRD name array as used with rrd_name()
* @return bool indicating rename success or failure
*/
public function renameFile($device, $oldname, $newname)
{
$oldrrd = self::name($device['hostname'], $oldname);
$newrrd = self::name($device['hostname'], $newname);
if (is_file($oldrrd) && !is_file($newrrd)) {
if (rename($oldrrd, $newrrd)) {
log_event("Renamed $oldrrd to $newrrd", $device, "poller", 1);
return true;
} else {
log_event("Failed to rename $oldrrd to $newrrd", $device, "poller", 5);
return false;
}
} else {
// we don't need to rename the file
return true;
}
}
/**
* Generates a filename based on the hostname (or IP) and some extra items
*
* @param string $host Host name
* @param array|string $extra Components of RRD filename - will be separated with "-", or a pre-formed rrdname
* @param string $extension File extension (default is .rrd)
* @return string the name of the rrd file for $host's $extra component
*/
public function name($host, $extra, $extension = ".rrd")
{
$filename = self::safeName(is_array($extra) ? implode("-", $extra) : $extra);
return implode("/", [$this->dirFromHost($host), $filename . $extension]);
}
/**
* Generates a path based on the hostname (or IP)
*
* @param string $host Host name
* @return string the name of the rrd directory for $host
*/
public function dirFromHost($host)
{
$host = str_replace(':', '_', trim($host, '[]'));
return implode("/", [$this->rrd_dir, $host]);
}
/**
* Generates and pipes a command to rrdtool
*
* @internal
* @param string $command create, update, updatev, graph, graphv, dump, restore, fetch, tune, first, last, lastupdate, info, resize, xport, flushcached
* @param string $filename The full patth to the rrd file
* @param string $options rrdtool command options
* @return array the output of stdout and stderr in an array
* @throws \Exception thrown when the rrdtool process(s) cannot be started
*/
private function command($command, $filename, $options)
{
global $vdebug;
$stat = Measurement::start($this->coalesceStatisticType($command));
try {
$cmd = self::buildCommand($command, $filename, $options);
} catch (FileExistsException $e) {
Log::debug("RRD[%g$filename already exists%n]", ['color' => true]);
return [null, null];
}
Log::debug("RRD[%g$cmd%n]", ['color' => true]);
// do not write rrd files, but allow read-only commands
$ro_commands = ['graph', 'graphv', 'dump', 'fetch', 'first', 'last', 'lastupdate', 'info', 'xport'];
if ($this->disabled && !in_array($command, $ro_commands)) {
if (!Config::get('hide_rrd_disabled')) {
Log::debug('[%rRRD Disabled%n]', ['color' => true]);
}
return [null, null];
}
// send the command!
if ($command == 'last' && $this->init(false)) {
// send this to our synchronous process so output is guaranteed
$output = $this->sync_process->sendCommand($cmd);
} elseif ($this->init()) {
// don't care about the return of other commands, so send them to the faster async process
$output = $this->async_process->sendCommand($cmd);
} else {
Log::error('rrdtool could not start');
}
if ($vdebug) {
echo 'RRDtool Output: ';
echo $output[0];
echo $output[1];
}
$this->recordStatistic($stat->end());
return $output;
}
/**
* Build a command for rrdtool
* Shortens the filename as needed
* Determines if --daemon and -O should be used
*
* @internal
* @param string $command The base rrdtool command. Usually create, update, last.
* @param string $filename The full path to the rrd file
* @param string $options Options for the command possibly including the rrd definition
* @return string returns a full command ready to be piped to rrdtool
* @throws FileExistsException if rrdtool <1.4.3 and the rrd file exists locally
*/
private function buildCommand($command, $filename, $options)
{
if ($command == 'create') {
// <1.4.3 doesn't support -O, so make sure the file doesn't exist
if (version_compare($this->version, '1.4.3', '<')) {
if (is_file($filename)) {
throw new FileExistsException();
}
} else {
$options .= ' -O';
}
}
// no remote for create < 1.5.5 and tune < 1.5
if ($this->rrdcached &&
!($command == 'create' && version_compare($this->version, '1.5.5', '<')) &&
!($command == 'tune' && $this->rrdcached && version_compare($this->version, '1.5', '<'))
) {
// only relative paths if using rrdcached
$filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
$options = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $options);
return "$command $filename $options --daemon " . $this->rrdcached;
}
return "$command $filename $options";
}
/**
* Checks if the rrd file exists on the server
* This will perform a remote check if using rrdcached and rrdtool >= 1.5
*
* @param string $filename full path to the rrd file
* @return bool whether or not the passed rrd file exists
*/
public function checkRrdExists($filename)
{
if ($this->rrdcached && version_compare($this->version, '1.5', '>=')) {
$chk = $this->command('last', $filename, '');
$filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
return !str_contains(implode($chk), "$filename': No such file or directory");
} else {
return is_file($filename);
}
}
/**
* Generates a graph file at $graph_file using $options
* Opens its own rrdtool pipe.
*
* @param string $graph_file
* @param string $options
* @return integer
*/
public function graph($graph_file, $options)
{
if ($this->init(false)) {
$cmd = $this->buildCommand('graph', $graph_file, $options);
$output = implode($this->sync_process->sendCommand($cmd));
d_echo("<p>$cmd</p>\n<p>command returned ($output)</p>");
return $output;
} else {
return 0;
}
}
public function __destruct()
{
$this->close();
}
/**
* Remove invalid characters from the rrd file name
*
* @param string $name
* @return string
*/
public static function safeName($name)
{
return (string)preg_replace('/[^a-zA-Z0-9,._\-]/', '_', $name);
}
/**
* Remove invalid characters from the rrd description
*
* @param string $descr
* @return string
*/
public static function safeDescr($descr)
{
return (string)preg_replace('/[^a-zA-Z0-9,._\-\/\ ]/', ' ', $descr);
}
/**
* Only track update and create primarily, just put all others in an "other" bin
*
* @param $type
* @return string
*/
private function coalesceStatisticType($type)
{
return ($type == 'update' || $type == 'create') ? $type : 'other';
}
}