diff --git a/daily.php b/daily.php index 238400ead1..cfcf8d5398 100644 --- a/daily.php +++ b/daily.php @@ -93,6 +93,27 @@ if ($options['f'] === 'notifications') { include_once 'includes/notifications.php'; } +if ($options['f'] === 'bill_data') { + if (is_numeric($config['billing_data_purge']) && $config['billing_data_purge'] > 0) { + # Deletes data older than XX months before the start of the last complete billing period + $months = $config['billing_data_purge']; + echo "Deleting billing data more than $months month before the last completed billing cycle\n"; + $sql = "DELETE bill_data + FROM bill_data + INNER JOIN (SELECT bill_id, + SUBDATE( + SUBDATE( + ADDDATE( + subdate(curdate(), (day(curdate())-1)), # Start of this month + bill_day - 1), # Billing anniversary + INTERVAL IF(bill_day > DAY(curdate()), 1, 0) MONTH), # Deal with anniversary not yet happened this month + INTERVAL ? MONTH) AS threshold # Adjust based on config threshold + FROM bills) q + ON bill_data.bill_id = q.bill_id AND bill_data.timestamp < q.threshold;"; + dbQuery($sql, array($months)); + } +} + if ($options['f'] === 'purgeusers') { $purge = 0; if (is_numeric($config['radius']['users_purge']) && $config['auth_mechanism'] === 'radius') { diff --git a/daily.sh b/daily.sh index 49dcb075e0..0841160554 100755 --- a/daily.sh +++ b/daily.sh @@ -74,6 +74,7 @@ else php daily.php -f callback php daily.php -f device_perf php daily.php -f purgeusers + php daily.php -f bill_data ;; submodules) # Init+Update our submodules diff --git a/doc/Extensions/Billing-Module.md b/doc/Extensions/Billing-Module.md index e46b5195d3..68ace18ba0 100644 --- a/doc/Extensions/Billing-Module.md +++ b/doc/Extensions/Billing-Module.md @@ -16,3 +16,17 @@ Edit `/etc/cron.d/librenms` and add the following: ``` Create billing graphs as required. + +## Options + +Billing data is stored in the MySQL database, and you may wish to purge the detailed +stats for old data (per-month totals will always be kept). To enable this, add the +following to `config.php`: + +```php +$config['billing_data_purge'] = 12; // Number of months to retain +``` + +Data for the last complete billing cycle will always be retained - only data older than +this by the configured number of months will be removed. This task is performed in the +daily cleanup tasks. \ No newline at end of file diff --git a/includes/billing.php b/includes/billing.php index cfdc657255..6ca49dcc26 100644 --- a/includes/billing.php +++ b/includes/billing.php @@ -79,27 +79,26 @@ function getValue($host, $port, $id, $inout) { }//end getValue() - -function getLastPortCounter($port_id, $inout) { +function getLastPortCounter($port_id) { $return = array(); - $row = dbFetchRow('SELECT counter,delta FROM `port_'.mres($inout)."_measurements` WHERE `port_id`='".mres($port_id)."' ORDER BY timestamp DESC LIMIT 1"); + $row = dbFetchRow("SELECT in_counter, in_delta, out_counter, out_delta FROM bill_port_counters WHERE `port_id` = ?", array($port_id)); if (!is_null($row)) { - $return[counter] = $row['counter']; - $return[delta] = $row['delta']; - $return[state] = 'ok'; - } + $return[in_counter] = $row['in_counter']; + $return[in_delta] = $row['in_delta']; + $return[out_counter] = $row['out_counter']; + $return[out_delta] = $row['out_delta']; + $return[state] = 'ok'; + } else { - $return[state] = 'failed'; + $return[state] = 'failed'; } - - return ($return); - + return $return; }//end getLastPortCounter() function getLastMeasurement($bill_id) { $return = array(); - $row = dbFetchRow("SELECT timestamp,delta,in_delta,out_delta FROM bill_data WHERE bill_id='".mres($bill_id)."' ORDER BY timestamp DESC LIMIT 1"); + $row = dbFetchRow("SELECT timestamp,delta,in_delta,out_delta FROM bill_data WHERE bill_id = ? ORDER BY timestamp DESC LIMIT 1", array($bill_id)); if (!is_null($row)) { $return[delta] = $row['delta']; $return[delta_in] = $row['delta_in']; @@ -110,9 +109,7 @@ function getLastMeasurement($bill_id) { else { $return[state] = 'failed'; } - return ($return); - }//end getLastMeasurement() diff --git a/misc/notifications.rss b/misc/notifications.rss index 5d094945b1..fd92ea616d 100644 --- a/misc/notifications.rss +++ b/misc/notifications.rss @@ -4,8 +4,8 @@ LibreNMS Notifications RSS feed for notifications to LibreNMS users http://www.librenms.org/ - Tue, 19 Jan 2016 12:00:00 +0000 - Tue, 19 Jan 2016 12:00:00 +0000 + Mon, 07 Mar 2016 12:00:00 +0000 + Mon, 07 Mar 2016 12:00:00 +0000 1800 Hello World! @@ -17,5 +17,12 @@ We have overhauled the Cisco syslog parsing. Please monitor your syslog entries from Cisco IOS, IOSXR, and CatOS devices. If you notice any issues please open a new issue on GitHub and include the original syslog message. Thu, 03 Mar 2016 12:00:00 +0000 + + Billing Data Changes + We have updated the method for storing billing data, removing a lot of information stored in + the database which was not used. Please check that your billing data is updating as expected, and + if you notice any problems open an issue on GitHub. + Mon, 07 Mar 2016 12:00:00 +0000 + diff --git a/poll-billing.php b/poll-billing.php index 5f2c278ab1..a4d16c553a 100755 --- a/poll-billing.php +++ b/poll-billing.php @@ -24,8 +24,16 @@ $iter = '0'; rrdtool_pipe_open($rrd_process, $rrd_pipes); +$poller_start = microtime(true); echo "Starting Polling Session ... \n\n"; +// Wait for schema update, as running during update can break update +$dbVersion = dbFetchCell('SELECT version FROM dbSchema'); +if ($dbVersion < 107) { + logfile("BILLING: Cannot continue until dbSchema update to >= 107 is complete"); + exit(1); +} + foreach (dbFetchRows('SELECT * FROM `bills`') as $bill_data) { echo 'Bill : '.$bill_data['bill_name']."\n"; @@ -43,39 +51,35 @@ foreach (dbFetchRows('SELECT * FROM `bills`') as $bill_data) { function CollectData($bill_id) { $port_list = dbFetchRows('SELECT * FROM `bill_ports` as P, `ports` as I, `devices` as D WHERE P.bill_id=? AND I.port_id = P.port_id AND D.device_id = I.device_id', array($bill_id)); - print_r($port_list); + $now = dbFetchCell('SELECT NOW()'); + $delta = 0; + $in_delta = 0; + $out_delta = 0; foreach ($port_list as $port_data) { $port_id = $port_data['port_id']; $host = $port_data['hostname']; $port = $port_data['port']; - echo "\nPolling ".$port_data['ifDescr'].' on '.$port_data['hostname']."\n"; + echo " Polling ${port_data['ifName']} (${port_data['ifDescr']}) on ${port_data['hostname']}\n"; $port_data['in_measurement'] = getValue($port_data['hostname'], $port_data['port'], $port_data['ifIndex'], 'In'); $port_data['out_measurement'] = getValue($port_data['hostname'], $port_data['port'], $port_data['ifIndex'], 'Out'); - $last_data = getLastPortCounter($port_id, in); - if ($last_data['state'] == 'ok') { - $port_data['last_in_measurement'] = $last_data[counter]; - $port_data['last_in_delta'] = $last_data[delta]; + $last_counters = getLastPortCounter($port_id); + if ($last_counters['state'] == 'ok') { + $port_data['last_in_measurement'] = $last_counters[in_counter]; + $port_data['last_in_delta'] = $last_counters[in_delta]; + $port_data['last_out_measurement'] = $last_counters[out_counter]; + $port_data['last_out_delta'] = $last_counters[out_delta]; + if ($port_data['in_measurement'] >= $port_data['last_in_measurement']) { $port_data['in_delta'] = ($port_data['in_measurement'] - $port_data['last_in_measurement']); } else { $port_data['in_delta'] = $port_data['last_in_delta']; } - } - else { - $port_data['in_delta'] = '0'; - } - - dbInsert(array('port_id' => $port_id, 'timestamp' => $now, 'counter' => $port_data['in_measurement'], 'delta' => $port_data['in_delta']), 'port_in_measurements'); - - $last_data = getLastPortCounter($port_id, out); - if ($last_data[state] == 'ok') { - $port_data['last_out_measurement'] = $last_data[counter]; - $port_data['last_out_delta'] = $last_data[delta]; + if ($port_data['out_measurement'] >= $port_data['last_out_measurement']) { $port_data['out_delta'] = ($port_data['out_measurement'] - $port_data['last_out_measurement']); } @@ -84,10 +88,15 @@ function CollectData($bill_id) { } } else { + $port_data['in_delta'] = '0'; $port_data['out_delta'] = '0'; } - - dbInsert(array('port_id' => $port_id, 'timestamp' => $now, 'counter' => $port_data['out_measurement'], 'delta' => $port_data['out_delta']), 'port_out_measurements'); + + $fields = array('timestamp' => $now, 'in_counter' => $port_data['in_measurement'], 'out_counter' => $port_data['out_measurement'], 'in_delta' => $port_data['in_delta'], 'out_delta' => $port_data['out_delta']); + if (dbUpdate($fields, 'bill_port_counters', "`port_id`='" . mres($port_id) . "'") == 0) { + $fields['port_id'] = $port_id; + dbInsert($fields, 'bill_port_counters'); + } $delta = ($delta + $port_data['in_delta'] + $port_data['out_delta']); $in_delta = ($in_delta + $port_data['in_delta']); @@ -130,4 +139,14 @@ if ($argv[1]) { CollectData($argv[1]); } +$poller_end = microtime(true); +$poller_run = ($poller_end - $poller_start); +$poller_time = substr($poller_run, 0, 5); + +dbInsert(array('type' => 'pollbill', 'doing' => $doing, 'start' => $poller_start, 'duration' => $poller_time, 'devices' => 0, 'poller' => $config['distributed_poller_name'] ), 'perf_times'); +if ($poller_time > 300) { + logfile("BILLING: polling took longer than 5 minutes ($poller_time seconds)!"); +} +echo "\nCompleted in $poller_time sec\n"; + rrdtool_pipe_close($rrd_process, $rrd_pipes); diff --git a/sql-schema/107.sql b/sql-schema/107.sql new file mode 100644 index 0000000000..0fb3afc881 --- /dev/null +++ b/sql-schema/107.sql @@ -0,0 +1,11 @@ +UPDATE dbSchema SET version = 107; +CREATE TABLE bill_port_counters_tmp(port_id int NOT NULL PRIMARY KEY, `timestamp` timestamp NOT NULL DEFAULT current_timestamp, in_counter bigint, in_delta bigint NOT NULL DEFAULT 0, out_counter bigint, out_delta bigint NOT NULL DEFAULT 0); +INSERT INTO bill_port_counters_tmp(port_id, timestamp, in_counter, out_counter) SELECT q.port_id, q.max_timestamp, max(i.counter), max(o.counter) FROM (SELECT port_id, MAX(`timestamp`) AS max_timestamp FROM port_in_measurements GROUP BY port_id) q INNER JOIN port_in_measurements i ON q.port_id = i.port_id AND q.max_timestamp = i.timestamp INNER JOIN port_out_measurements o ON q.port_id = o.port_id AND q.max_timestamp = o.timestamp GROUP BY q.port_id, q.max_timestamp; +RENAME TABLE bill_port_counters_tmp TO bill_port_counters; +ALTER TABLE bill_data ADD id int NOT NULL PRIMARY KEY AUTO_INCREMENT FIRST; +DELETE bill_data FROM bill_data INNER JOIN (SELECT bill_id, timestamp, MIN(id) as first_id FROM bill_data GROUP BY bill_id, timestamp HAVING COUNT(id) > 1) q ON bill_data.bill_id = q.bill_id AND bill_data.timestamp = q.timestamp; +ALTER TABLE bill_data DROP id; +ALTER TABLE bill_data ADD PRIMARY KEY (bill_id, timestamp); +ALTER TABLE bill_data DROP INDEX bill_id_2; +DROP TABLE port_in_measurements; +DROP TABLE port_out_measurements;