Merge branch 'master' into issue-2193

daily.php

@@ -93,6 +93,27 @@ if ($options['f'] === 'notifications') {
     include_once 'includes/notifications.php';
 }
 
+if ($options['f'] === 'bill_data') {
+    if (is_numeric($config['billing_data_purge']) && $config['billing_data_purge'] > 0) {
+        # Deletes data older than XX months before the start of the last complete billing period
+        $months = $config['billing_data_purge'];
+        echo "Deleting billing data more than $months month before the last completed billing cycle\n";
+        $sql = "DELETE bill_data
+                FROM bill_data
+                    INNER JOIN (SELECT bill_id, 
+                        SUBDATE(
+                            SUBDATE(
+                                ADDDATE(
+                                    subdate(curdate(), (day(curdate())-1)),             # Start of this month
+                                    bill_day - 1),                                      # Billing anniversary
+                                INTERVAL IF(bill_day > DAY(curdate()), 1, 0) MONTH),    # Deal with anniversary not yet happened this month
+                            INTERVAL ? MONTH) AS threshold                              # Adjust based on config threshold
+                FROM bills) q
+                ON bill_data.bill_id = q.bill_id AND bill_data.timestamp < q.threshold;";
+        dbQuery($sql, array($months));
+    }
+}
+
 if ($options['f'] === 'purgeusers') {
     $purge = 0;
     if (is_numeric($config['radius']['users_purge']) && $config['auth_mechanism'] === 'radius') {

daily.sh

@@ -74,6 +74,7 @@ else
             php daily.php -f callback
             php daily.php -f device_perf
             php daily.php -f purgeusers
+            php daily.php -f bill_data
         ;;
         submodules)
             # Init+Update our submodules

doc/Extensions/Billing-Module.md

@@ -16,3 +16,17 @@ Edit `/etc/cron.d/librenms` and add the following:
 ```
 
 Create billing graphs as required.
+
+## Options
+
+Billing data is stored in the MySQL database, and you may wish to purge the detailed 
+stats for old data (per-month totals will always be kept).  To enable this, add the 
+following to `config.php`:
+
+```php
+$config['billing_data_purge'] = 12;     // Number of months to retain
+```
+
+Data for the last complete billing cycle will always be retained - only data older than
+this by the configured number of months will be removed.  This task is performed in the
+daily cleanup tasks.

includes/billing.php

@@ -79,27 +79,26 @@ function getValue($host, $port, $id, $inout) {
 
 }//end getValue()
 
-
-function getLastPortCounter($port_id, $inout) {
+function getLastPortCounter($port_id) {
     $return = array();
-    $row    = dbFetchRow('SELECT counter,delta FROM `port_'.mres($inout)."_measurements` WHERE `port_id`='".mres($port_id)."' ORDER BY timestamp DESC LIMIT 1");
+    $row    = dbFetchRow("SELECT in_counter, in_delta, out_counter, out_delta FROM bill_port_counters WHERE `port_id` = ?", array($port_id));
     if (!is_null($row)) {
-        $return[counter] = $row['counter'];
-        $return[delta]   = $row['delta'];
-        $return[state]   = 'ok';
-    }
+        $return[in_counter]  = $row['in_counter'];
+        $return[in_delta]    = $row['in_delta'];
+        $return[out_counter] = $row['out_counter'];
+        $return[out_delta]   = $row['out_delta'];
+        $return[state]       = 'ok';
+    } 
     else {
-        $return[state] = 'failed';
+        $return[state]       = 'failed';
     }
-
-    return ($return);
-
+    return $return;
 }//end getLastPortCounter()
 
 
 function getLastMeasurement($bill_id) {
     $return = array();
-    $row    = dbFetchRow("SELECT timestamp,delta,in_delta,out_delta FROM bill_data WHERE bill_id='".mres($bill_id)."' ORDER BY timestamp DESC LIMIT 1");
+    $row    = dbFetchRow("SELECT timestamp,delta,in_delta,out_delta FROM bill_data WHERE bill_id = ? ORDER BY timestamp DESC LIMIT 1", array($bill_id));
     if (!is_null($row)) {
         $return[delta]     = $row['delta'];
         $return[delta_in]  = $row['delta_in'];
@@ -110,9 +109,7 @@ function getLastMeasurement($bill_id) {
     else {
         $return[state] = 'failed';
     }
-
     return ($return);
-
 }//end getLastMeasurement()
 
 

misc/notifications.rss

@@ -4,8 +4,8 @@
  <title>LibreNMS Notifications</title>
  <description>RSS feed for notifications to LibreNMS users</description>
  <link>http://www.librenms.org/</link>
- <lastBuildDate>Tue, 19 Jan 2016 12:00:00 +0000</lastBuildDate>
- <pubDate>Tue, 19 Jan 2016 12:00:00 +0000</pubDate>
+ <lastBuildDate>Mon, 07 Mar 2016 12:00:00 +0000</lastBuildDate>
+ <pubDate>Mon, 07 Mar 2016 12:00:00 +0000</pubDate>
  <ttl>1800</ttl>
  <item>
   <title>Hello World!</title>
@@ -17,5 +17,12 @@
   <description>We have overhauled the Cisco syslog parsing.  Please monitor your syslog entries from Cisco IOS, IOSXR, and CatOS devices.  If you notice any issues please open a new issue on GitHub and include the original syslog message.</description>
   <pubDate>Thu, 03 Mar 2016 12:00:00 +0000</pubDate>
  </item>
+ <item>
+   <title>Billing Data Changes</title>
+   <description>We have updated the method for storing billing data, removing a lot of information stored in 
+   the database which was not used.  Please check that your billing data is updating as expected, and 
+   if you notice any problems open an issue on GitHub.</description>
+   <pubDate>Mon, 07 Mar 2016 12:00:00 +0000</pubDate>
+ </item>
 </channel>
 </rss>

poll-billing.php

@@ -24,8 +24,16 @@ $iter = '0';
 
 rrdtool_pipe_open($rrd_process, $rrd_pipes);
 
+$poller_start = microtime(true);
 echo "Starting Polling Session ... \n\n";
 
+// Wait for schema update, as running during update can break update
+$dbVersion = dbFetchCell('SELECT version FROM dbSchema');
+if ($dbVersion < 107) {
+    logfile("BILLING: Cannot continue until dbSchema update to >= 107 is complete");
+    exit(1);
+}
+
 foreach (dbFetchRows('SELECT * FROM `bills`') as $bill_data) {
     echo 'Bill : '.$bill_data['bill_name']."\n";
 
@@ -43,39 +51,35 @@ foreach (dbFetchRows('SELECT * FROM `bills`') as $bill_data) {
 
 function CollectData($bill_id) {
     $port_list = dbFetchRows('SELECT * FROM `bill_ports` as P, `ports` as I, `devices` as D WHERE P.bill_id=? AND I.port_id = P.port_id AND D.device_id = I.device_id', array($bill_id));
-    print_r($port_list);
+
     $now = dbFetchCell('SELECT NOW()');
+    $delta = 0;
+    $in_delta = 0;
+    $out_delta = 0;
     foreach ($port_list as $port_data) {
         $port_id = $port_data['port_id'];
         $host    = $port_data['hostname'];
         $port    = $port_data['port'];
 
-        echo "\nPolling ".$port_data['ifDescr'].' on '.$port_data['hostname']."\n";
+        echo "  Polling ${port_data['ifName']} (${port_data['ifDescr']}) on ${port_data['hostname']}\n";
 
         $port_data['in_measurement']  = getValue($port_data['hostname'], $port_data['port'], $port_data['ifIndex'], 'In');
         $port_data['out_measurement'] = getValue($port_data['hostname'], $port_data['port'], $port_data['ifIndex'], 'Out');
 
-        $last_data = getLastPortCounter($port_id, in);
-        if ($last_data['state'] == 'ok') {
-            $port_data['last_in_measurement'] = $last_data[counter];
-            $port_data['last_in_delta']       = $last_data[delta];
+        $last_counters = getLastPortCounter($port_id);
+        if ($last_counters['state'] == 'ok') {
+            $port_data['last_in_measurement']  = $last_counters[in_counter];
+            $port_data['last_in_delta']        = $last_counters[in_delta];
+            $port_data['last_out_measurement'] = $last_counters[out_counter];
+            $port_data['last_out_delta']       = $last_counters[out_delta];
+
             if ($port_data['in_measurement'] >= $port_data['last_in_measurement']) {
                 $port_data['in_delta'] = ($port_data['in_measurement'] - $port_data['last_in_measurement']);
             }
             else {
                 $port_data['in_delta'] = $port_data['last_in_delta'];
             }
-        }
-        else {
-            $port_data['in_delta'] = '0';
-        }
-
-        dbInsert(array('port_id' => $port_id, 'timestamp' => $now, 'counter' => $port_data['in_measurement'], 'delta' => $port_data['in_delta']), 'port_in_measurements');
-
-        $last_data = getLastPortCounter($port_id, out);
-        if ($last_data[state] == 'ok') {
-            $port_data['last_out_measurement'] = $last_data[counter];
-            $port_data['last_out_delta']       = $last_data[delta];
+            
             if ($port_data['out_measurement'] >= $port_data['last_out_measurement']) {
                 $port_data['out_delta'] = ($port_data['out_measurement'] - $port_data['last_out_measurement']);
             }
@@ -84,10 +88,15 @@ function CollectData($bill_id) {
             }
         }
         else {
+            $port_data['in_delta'] = '0';
             $port_data['out_delta'] = '0';
         }
-
-        dbInsert(array('port_id' => $port_id, 'timestamp' => $now, 'counter' => $port_data['out_measurement'], 'delta' => $port_data['out_delta']), 'port_out_measurements');
+        
+        $fields = array('timestamp' => $now, 'in_counter' => $port_data['in_measurement'], 'out_counter' => $port_data['out_measurement'], 'in_delta' => $port_data['in_delta'], 'out_delta' => $port_data['out_delta']);
+        if (dbUpdate($fields, 'bill_port_counters', "`port_id`='" . mres($port_id) . "'") == 0) {
+            $fields['port_id'] = $port_id;
+            dbInsert($fields, 'bill_port_counters');
+        }
 
         $delta     = ($delta + $port_data['in_delta'] + $port_data['out_delta']);
         $in_delta  = ($in_delta + $port_data['in_delta']);
@@ -130,4 +139,14 @@ if ($argv[1]) {
     CollectData($argv[1]);
 }
 
+$poller_end  = microtime(true);
+$poller_run  = ($poller_end - $poller_start);
+$poller_time = substr($poller_run, 0, 5);
+
+dbInsert(array('type' => 'pollbill', 'doing' => $doing, 'start' => $poller_start, 'duration' => $poller_time, 'devices' => 0, 'poller' => $config['distributed_poller_name'] ), 'perf_times');
+if ($poller_time > 300) {
+    logfile("BILLING: polling took longer than 5 minutes ($poller_time seconds)!");
+}
+echo "\nCompleted in $poller_time sec\n";
+
 rrdtool_pipe_close($rrd_process, $rrd_pipes);

sql-schema/107.sql

@@ -0,0 +1,11 @@
+UPDATE dbSchema SET version = 107;
+CREATE TABLE bill_port_counters_tmp(port_id int NOT NULL PRIMARY KEY, `timestamp` timestamp NOT NULL DEFAULT current_timestamp, in_counter bigint, in_delta bigint NOT NULL DEFAULT 0, out_counter bigint, out_delta bigint NOT NULL DEFAULT 0);
+INSERT INTO bill_port_counters_tmp(port_id, timestamp, in_counter, out_counter) SELECT q.port_id, q.max_timestamp, max(i.counter), max(o.counter) FROM (SELECT port_id, MAX(`timestamp`) AS max_timestamp FROM port_in_measurements GROUP BY port_id) q INNER JOIN port_in_measurements i ON q.port_id = i.port_id AND q.max_timestamp = i.timestamp INNER JOIN port_out_measurements o ON q.port_id = o.port_id AND q.max_timestamp = o.timestamp GROUP BY q.port_id, q.max_timestamp;
+RENAME TABLE bill_port_counters_tmp TO bill_port_counters;
+ALTER TABLE bill_data ADD id int NOT NULL PRIMARY KEY AUTO_INCREMENT FIRST;
+DELETE bill_data FROM bill_data INNER JOIN (SELECT bill_id, timestamp, MIN(id) as first_id FROM bill_data GROUP BY bill_id, timestamp HAVING COUNT(id) > 1) q ON bill_data.bill_id = q.bill_id AND bill_data.timestamp = q.timestamp;
+ALTER TABLE bill_data DROP id;
+ALTER TABLE bill_data ADD PRIMARY KEY (bill_id, timestamp);
+ALTER TABLE bill_data DROP INDEX bill_id_2;
+DROP TABLE port_in_measurements;
+DROP TABLE port_out_measurements;