mirror/librenms-librenms
1
0
Fork 0
mirror of https://github.com/librenms/librenms.git synced 2024-10-07 16:52:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #2740 from paulgear/unified-data-storage-rrd-influx

Browse Source 
First cut at unified data storage for rrdtool & influxdb
This commit is contained in:
Neil Lathwood
2016-01-21 23:31:09 +00:00
parent aec0453c76 72b9f52c59
commit abf2fc0ba5
12 changed files with 171 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
includes/datastore.inc.php Normal file
View File

@@ -0,0 +1,60 @@
<?php
/*
* LibreNMS abstract data storage interface to both rrdtool & influxdb
*
* Copyright (c) 2016 Paul D. Gear <paul@librenms.org>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
require_once $config['install_dir'] . "/includes/rrdtool.inc.php";
require_once $config['install_dir'] . "/includes/influxdb.inc.php";
/*
* @return Copy of $arr with all keys beginning with 'rrd_' removed.
*/
function rrd_array_filter($arr)
{
$result = array();
foreach ($arr as $k => $v) {
if (strpos($k, 'rrd_') === 0) {
continue;
}
$result[$k] = $v;
}
return $result;
} // rrd_array_filter
/*
* Datastore-independent function which should be used for all polled metrics.
*/
function data_update($device, $measurement, $tags, $fields)
{
// convenience conversion to allow calling with a single value, so, e.g., these are equivalent:
// data_update($device, 'mymeasurement', $tags, 1234);
// AND
// data_update($device, 'mymeasurement', $tags, array('mymeasurement' => 1234));
if (!is_array($fields)) {
$fields = array($measurement => $fields);
}
// rrdtool_data_update() will only use the tags it deems relevant, so we pass all of them.
// However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
rrdtool_data_update($device, $measurement, $tags, $fields);
influx_update($device, $measurement, rrd_array_filter($tags), $fields);
} // data_update

3
includes/functions.php
View File

@@ -21,8 +21,7 @@ include_once("Net/IPv6.php");
include_once($config['install_dir'] . "/includes/dbFacile.php");
include_once($config['install_dir'] . "/includes/common.php");
include_once($config['install_dir'] . "/includes/rrdtool.inc.php");
include_once($config['install_dir'] . "/includes/influxdb.inc.php");
include_once($config['install_dir'] . "/includes/datastore.inc.php");
include_once($config['install_dir'] . "/includes/billing.php");
include_once($config['install_dir'] . "/includes/cisco-entities.php");
include_once($config['install_dir'] . "/includes/syslog.php");

37
includes/polling/altiga-ssl.inc.php
View File

@@ -3,6 +3,8 @@
if ($device['os'] == 'asa' || $device['os'] == 'pix') {
echo "ALTIGA-MIB SSL VPN Statistics \n";
$tags = array();
$oids = array(
'alSslStatsTotalSessions',
'alSslStatsActiveSessions',
@@ -13,26 +15,27 @@ if ($device['os'] == 'asa' || $device['os'] == 'pix') {
'alSslStatsPostEncryptOctets',
);
unset($snmpstring, $fields, $snmpdata, $snmpdata_cmd, $rrd_create);
$rrdfile = $config['rrd_dir'].'/'.$device['hostname'].'/'.safename('altiga-ssl.rrd');
$rrd_create .= ' DS:TotalSessions:COUNTER:600:U:100000 DS:ActiveSessions:GAUGE:600:0:U DS:MaxSessions:GAUGE:600:0:U';
$rrd_create .= ' DS:PreDecryptOctets:COUNTER:600:U:100000000000 DS:PostDecryptOctets:COUNTER:600:U:100000000000 DS:PreEncryptOctets:COUNTER:600:U:100000000000';
$rrd_create .= ' DS:PostEncryptOctets:COUNTER:600:U:100000000000';
$rrd_create .= $config['rrd_rra'];
if (!file_exists($rrdfile)) {
rrdtool_create($rrdfile, $rrd_create);
}
$tags['rrd_def'] = array(
'DS:TotalSessions:COUNTER:600:U:100000',
'DS:ActiveSessions:GAUGE:600:0:U',
'DS:MaxSessions:GAUGE:600:0:U',
'DS:PreDecryptOctets:COUNTER:600:U:100000000000',
'DS:PostDecryptOctets:COUNTER:600:U:100000000000',
'DS:PreEncryptOctets:COUNTER:600:U:100000000000',
'DS:PostEncryptOctets:COUNTER:600:U:100000000000',
);
$data_array = snmpwalk_cache_oid($device, $proto, array(), 'ALTIGA-SSL-STATS-MIB');
$fields = array();
$got_value = false;
foreach ($oids as $oid) {
if (is_numeric($data_array[0][$oid])) {
$value = $data_array[0][$oid];
if ($value > 0) {
$got_value = true;
}
}
else {
$value = '0';
@@ -40,13 +43,9 @@ if ($device['os'] == 'asa' || $device['os'] == 'pix') {
$fields[$oid] = $value;
}
if ($data_array[0]['alSslStatsTotalSessions'] || is_file($rrdfile)) {
rrdtool_update($rrdfile, $fields);
$tags = array();
influx_update($device,'altiga-ssl',$tags,$fields);
if ($got_value) {
data_update($device, 'altiga-ssl', $tags, $fields);
}
unset($rrdfile, $fields, $data_array);
unset($tags, $fields, $oids, $data_array);
}//end if

18
includes/polling/core.inc.php
View File

@@ -67,22 +67,10 @@ if (is_numeric($uptime)) {
log_event('Device rebooted after '.formatUptime($device['uptime']), $device, 'reboot', $device['uptime']);
}
$uptime_rrd = $config['rrd_dir'].'/'.$device['hostname'].'/uptime.rrd';
if (!is_file($uptime_rrd)) {
rrdtool_create($uptime_rrd, 'DS:uptime:GAUGE:600:0:U '.$config['rrd_rra']);
}
$fields = array(
'uptime' => $uptime,
$tags = array(
'rrd_def' => 'DS:uptime:GAUGE:600:0:U',
);
rrdtool_update($uptime_rrd, $fields);
$measurement = 'uptime';
$tags = array();
influx_update($device,$measurement,$tags,$fields);
data_update($device, 'uptime', $tags, $uptime);
$graphs['uptime'] = true;

45
includes/polling/functions.inc.php
View File

@@ -268,47 +268,30 @@ function poll_device($device, $options) {
$device_run = ($device_end - $device_start);
$device_time = substr($device_run, 0, 5);
// TODO: These should be easy converts to rrd_create_update()
// Poller performance rrd
$poller_rrd = $config['rrd_dir'].'/'.$device['hostname'].'/poller-perf.rrd';
if (!is_file($poller_rrd)) {
rrdtool_create($poller_rrd, 'DS:poller:GAUGE:600:0:U '.$config['rrd_rra']);
}
// Poller performance
if (!empty($device_time)) {
$tags = array(
'rrd_def' => 'DS:poller:GAUGE:600:0:U',
);
$fields = array(
'poller' => $device_time,
);
rrdtool_update($poller_rrd, $fields);
$tags = array();
influx_update($device,'poller-perf',$tags,$fields);
data_update($device, 'poller-perf', $tags, $fields);
}
// Ping response rrd
if (can_ping_device($attribs) === true) {
$ping_rrd = $config['rrd_dir'].'/'.$device['hostname'].'/ping-perf.rrd';
if (!is_file($ping_rrd)) {
rrdtool_create($ping_rrd, 'DS:ping:GAUGE:600:0:65535 '.$config['rrd_rra']);
}
if (!empty($ping_time)) {
$fields = array(
'ping' => $ping_time,
);
rrdtool_update($ping_rrd, $fields);
}
// Ping response
if (can_ping_device($attribs) === true && !empty($ping_time)) {
$tags = array(
'rrd_def' => 'DS:ping:GAUGE:600:0:65535',
);
$fields = array(
'ping' => $ping_time,
);
$update_array['last_ping'] = array('NOW()');
$update_array['last_ping_timetaken'] = $ping_time;
rrdtool_update($ping_rrd, $fields);
$tags = array();
influx_update($device,'ping-perf',$tags,$fields);
data_update($device, 'ping-perf', $tags, $fields);
}
$update_array['last_polled'] = array('NOW()');

1
includes/polling/system.inc.php
View File

@@ -1,2 +1 @@
<?php

29
includes/polling/toner.inc.php
View File

@@ -12,32 +12,15 @@ if ($config['enable_printers']) {
}
$tonerperc = round((snmp_get($device, $toner['toner_oid'], '-OUqnv') / $toner['toner_capacity'] * 100));
$old_tonerrrd = $config['rrd_dir'].'/'.$device['hostname'].'/'.safename('toner-'.$toner['toner_descr'].'.rrd');
$tonerrrd = $config['rrd_dir'].'/'.$device['hostname'].'/'.safename('toner-'.$toner['toner_index'].'.rrd');
if (!is_file($tonerrrd) && is_file($old_tonerrrd)) {
rename($old_tonerrrd, $tonerrrd);
}
if (!is_file($tonerrrd)) {
rrdtool_create(
$tonerrrd,
'--step 300
DS:toner:GAUGE:600:0:20000 '.$config['rrd_rra']
);
}
echo $tonerperc." %\n";
$fields = array(
'toner' => $tonerperc,
$tags = array(
'rrd_def' => 'DS:toner:GAUGE:600:0:20000',
'rrd_name' => array('toner', $toner['toner_index']),
'rrd_oldname' => array('toner', $toner['toner_descr']),
'index' => $toner['toner_index'],
);
rrdtool_update($tonerrrd, $fields);
$tags = array('index' => $toner['toner_index']);
influx_update($device,'toner',$tags,$fields);
data_update($device, 'toner', $tags, $tonerperc);
// FIXME should report for toner out... :)
// Log toner swap

29
includes/polling/ucd-diskio.inc.php
View File

@@ -17,20 +17,16 @@ if (count($diskio_data)) {
d_echo($entry);
$rrd = $config['rrd_dir'].'/'.$device['hostname'].'/'.safename('ucd_diskio-'.$diskio['diskio_descr'].'.rrd');
d_echo("$rrd ");
if (!is_file($rrd)) {
rrdtool_create(
$rrd,
'--step 300
DS:read:DERIVE:600:0:125000000000
DS:written:DERIVE:600:0:125000000000
DS:reads:DERIVE:600:0:125000000000
DS:writes:DERIVE:600:0:125000000000 '.$config['rrd_rra']
);
}
$tags = array(
'rrd_name' => array('ucd_diskio', $diskio['diskio_descr']),
'rrd_def' => array(
'DS:read:DERIVE:600:0:125000000000',
'DS:written:DERIVE:600:0:125000000000',
'DS:reads:DERIVE:600:0:125000000000',
'DS:writes:DERIVE:600:0:125000000000',
),
'descr' => $diskio['diskio_descr'],
);
$fields = array(
'read' => $entry['diskIONReadX'],
@@ -39,10 +35,7 @@ if (count($diskio_data)) {
'writes' => $entry['diskIOWrites'],
);
rrdtool_update($rrd, $fields);
$tags = array('descr' => $diskio['diskio_descr']);
influx_update($device,'ucd_diskio',$tags,$fields);
data_update($device, 'ucd_diskio', $tags, $fields);
}//end foreach

13
includes/polling/unix-agent.inc.php
View File

@@ -41,19 +41,14 @@ if ($device['os_group'] == 'unix') {
if (!empty($agent_raw)) {
echo 'execution time: '.$agent_time.'ms';
$agent_rrd = $config['rrd_dir'].'/'.$device['hostname'].'/agent.rrd';
if (!is_file($agent_rrd)) {
rrdtool_create($agent_rrd, 'DS:time:GAUGE:600:0:U '.$config['rrd_rra']);
}
$tags = array(
'rrd_def' => 'DS:time:GAUGE:600:0:U',
);
$fields = array(
'time' => $agent_time,
);
rrdtool_update($agent_rrd, $fields);
$tags = array();
influx_update($device,'agent',$tags,$fields);
data_update($device, 'agent', $tags, $fields);
$graphs['agent'] = true;

48
includes/polling/wifi.inc.php
View File

@@ -78,51 +78,23 @@ if ($device['type'] == 'network' || $device['type'] == 'firewall' || $device['ty
echo (($wificlients1 + 0).' clients on wireless connector, ');
}
// # RRD Filling Code
if (isset($wificlients1) && $wificlients1 != '') {
$wificlientsrrd = $config['rrd_dir'].'/'.$device['hostname'].'/'.safename('wificlients-radio1.rrd');
if (!is_file($wificlientsrrd)) {
rrdtool_create(
$wificlientsrrd,
'--step 300
DS:wificlients:GAUGE:600:-273:1000 '.$config['rrd_rra']
);
}
$fields = array(
'wificlients' => $wificlients1,
$tags = array(
'rrd_def' => 'DS:wificlients:GAUGE:600:-273:1000',
'rrd_name' => array('wificlients', 'radio1'),
'radio' => 1,
);
rrdtool_update($wificlientsrrd, $fields);
$tags = array('radio' => '1');
influx_update($device,'wificlients',$tags,$fields);
data_update($device, 'wificlients', $tags, $wificlients1);
$graphs['wifi_clients'] = true;
}
if (isset($wificlients2) && $wificlients2 != '') {
$wificlientsrrd = $config['rrd_dir'].'/'.$device['hostname'].'/'.safename('wificlients-radio2.rrd');
if (!is_file($wificlientsrrd)) {
rrdtool_create(
$wificlientsrrd,
'--step 300
DS:wificlients:GAUGE:600:-273:1000 '.$config['rrd_rra']
);
}
$fields = array(
'wificlients' => $wificlients2,
$tags = array(
'rrd_def' => 'DS:wificlients:GAUGE:600:-273:1000',
'rrd_name' => array('wificlients', 'radio2'),
'radio' => 2,
);
rrdtool_update($wificlientsrrd, $fields);
$tags = array('radio' => '2');
influx_update($device,'wificlients',$tags,$fields);
data_update($device, 'wificlients', $tags, $wificlients2);
$graphs['wifi_clients'] = true;
}

62
includes/rrdtool.inc.php
View File

@@ -331,49 +331,55 @@ function rrdtool_tune($type, $filename, $max) {
}
} // rrdtool_tune
/*
* Please use this instead of creating & updating RRD files manually.
* @param device Device object - only 'hostname' is used at present
* @param name Array of rrdname components
* @param def Array of data definitions
* @param val Array of value definitions
*
* rrdtool backend implementation of data_update
*/
function rrd_create_update($device, $name, $def, $val, $step=300)
function rrdtool_data_update($device, $measurement, $tags, $fields)
{
global $config;
$rrd = rrd_name($device['hostname'], $name);
if (!is_file($rrd) && $def != null) {
// add the --step and the rra definitions to the array
$newdef = "--step $step ".implode(' ', $def).$config['rrd_rra'];
$rrd_name = $tags['rrd_name'] ? $tags['rrd_name'] : $measurement;
$step = $tags['rrd_step'] ? $tags['rrd_step'] : 300;
$oldname = $tags['rrd_oldname'];
if (isset($oldname) && !empty($oldname)) {
rrd_file_rename($device, $oldname, $rrd_name);
}
$rrd = rrd_name($device['hostname'], $rrd_name);
if (!is_file($rrd) && $tags['rrd_def']) {
$rrd_def = is_array($tags['rrd_def']) ? $tags['rrd_def'] : array($tags['rrd_def']);
// add the --step and the rra definitions to the command
$newdef = "--step $step ".implode(' ', $rrd_def).$config['rrd_rra'];
rrdtool_create($rrd, $newdef);
}
rrdtool_update($rrd, $val);
} // rrd_create_update
/*
* @return bool indicating existence of RRD file
* @param device Device object as used with rrd_create_update()
* @param name RRD name array as used with rrd_create_update() and rrd_name()
*/
function rrd_file_exists($device, $name)
{
return is_file(rrd_name($device['hostname'], $name));
} // rrd_file_exists
rrdtool_update($rrd, $fields);
} // rrdtool_data_update
/*
* @return bool indicating rename success or failure
* @param device Device object as used with rrd_create_update()
* @param oldname RRD name array as used with rrd_create_update() and rrd_name()
* @param newname RRD name array as used with rrd_create_update() and rrd_name()
* @param device Device object
* @param oldname RRD name array as used with rrd_name()
* @param newname RRD name array as used with rrd_name()
*/
function rrd_file_rename($device, $oldname, $newname)
{
$oldrrd = rrd_name($device['hostname'], $oldname);
$newrrd = rrd_name($device['hostname'], $newname);
return rename($oldrrd, $newrrd);
if (is_file($oldrrd) && !is_file($newrrd)) {
if (rename($oldrrd, $newrrd)) {
log_event("Renamed $oldrrd to $newrrd", $device);
return true;
}
else {
log_event("Failed to rename $oldrrd to $newrrd", $device);
return false;
}
}
else {
// we don't need to rename the file
return true;
}
} // rrd_file_rename

28
includes/snmp.inc.php
View File

@@ -1139,25 +1139,17 @@ function save_mibs($device, $mibname, $oids, $mibdef, &$graphs)
$usedoids[$index][$obj] = $val;
// if there's a file from the previous version of MIB-based polling, rename it
if (rrd_file_exists($device, array($mibname, $mibdef[$obj]['object_type'], $index))
&& !rrd_file_exists($device, array($mibname, $mibdef[$obj]['shortname'], $index))) {
rrd_file_rename($device,
array($mibname, $mibdef[$obj]['object_type'], $index),
array($mibname, $mibdef[$obj]['shortname'], $index));
// Note: polling proceeds regardless of rename result
}
rrd_create_update(
$device,
array(
$mibname,
$mibdef[$obj]['shortname'],
$index,
),
array("DS:mibval:$type"),
array("mibval" => $val)
$tags = array(
'rrd_def' => array("DS:mibval:$type"),
'rrd_name' => array($mibname, $mibdef[$obj]['shortname'], $index),
'rrd_oldname' => array($mibname, $mibdef[$obj]['object_type'], $index),
'index' => $index,
'oid' => $mibdef[$obj]['oid'],
'module' => $mibdef[$obj]['module'],
'mib' => $mibdef[$obj]['mib'],
'object_type' => $obj,
);
data_update($device, 'mibval', $tags, $val);
}
}