Refactor Datastores to allow future improvements. OpenTSDB Tags. (#11283)

* Datastores to object oriented code, using the Laravel IoC container

Change instantiation
better DI
move OpenTSDB

Small re-orgs

remove unused stuff

Fix graphs and other scripts

Use DI for all except rrd

fix up connection error handling

Add tests, fix up a "few" things
Add Config::forget()

Style fixes

Don't reference legacy code

remove accidental code paste

Add datastores phpunit groups

some tests

* rebase fixes

* some test fixes

* shorter tests

* shorter tests

* Don't except when rrdtool can't be started.

* restore tests

* fix rrd tests

* fix iterable change upstream

* fix isValidDataset

* fix invalid data bug

* fix mysql incorrect ds

* fix issue with data that is too long

* use regular data_update()

* Use log facade

* OpenTSDB mis-ordered arguments fix

* Making a singleton with different options makes different singletons.  Just use the global config settings to disable datastores.

* only filter tags for datastores that won't it don't modify the tags permanently

* Update copyrights to include original authors.

* Stats for all datastores

* Fix mysql sends different rrd / other ds names

* fix snmp last stats not initialized
remove unused function

* remove unused function and move single use function closer to its use

* InfluxDB does not need to update null or U values.
Skip write if all fields are empty

* Fix smart value checks

* fix style issues

* Make sure port data is stored the same way as before for Graphite and OpenTSDB
Add ifIndex tag to all to be compatible

* Missed rrdtool_tune() call

* Test update WIP

* OpenTSDB now includes tags

* fix style

LibreNMS/Data/Measure/Measurement.php

@@ -0,0 +1,81 @@
+<?php
+/**
+ * Measurement.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Measure;
+
+class Measurement
+{
+    private $start;
+    private $type;
+    private $duration;
+
+    private function __construct(string $type)
+    {
+        $this->type = $type;
+        $this->start = microtime(true);
+    }
+
+    /**
+     * Start the timer for a new operation
+     *
+     * @param string $type
+     * @return static
+     */
+    public static function start(string $type)
+    {
+        return new static($type);
+    }
+
+    /**
+     * End the timer for this operation
+     *
+     * @return $this
+     */
+    public function end()
+    {
+        $this->duration = microtime(true) - $this->start;
+        return $this;
+    }
+
+    /**
+     * Get the duration of the operation
+     *
+     * @return float
+     */
+    public function getDuration()
+    {
+        return $this->duration;
+    }
+
+    /**
+     * Get the type of the operation
+     *
+     * @return string
+     */
+    public function getType()
+    {
+        return $this->type;
+    }
+}

LibreNMS/Data/Measure/MeasurementCollection.php

@@ -0,0 +1,75 @@
+<?php
+/**
+ * MeasurementCollection.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Measure;
+
+use Illuminate\Support\Collection;
+
+class MeasurementCollection extends Collection
+{
+    public function getTotalCount()
+    {
+        return $this->sumStat('getCount');
+    }
+
+    public function getTotalDuration()
+    {
+        return $this->sumStat('getDuration');
+    }
+
+    public function getCountDiff()
+    {
+        return $this->sumStat('getCountDiff');
+    }
+
+    public function getDurationDiff()
+    {
+        return $this->sumStat('getDurationDiff');
+    }
+
+    public function checkpoint()
+    {
+        $this->each->checkpoint();
+    }
+
+    public function record(Measurement $measurement)
+    {
+        $type = $measurement->getType();
+
+        if (!$this->has($type)) {
+            $this->put($type, new MeasurementSummary($type));
+        }
+
+        $this->get($type)->add($measurement);
+    }
+
+    private function sumStat($function)
+    {
+        return $this->reduce(function ($sum, $measurement) use ($function) {
+            $sum += $measurement->$function();
+            return $sum;
+        }, 0);
+    }
+}

LibreNMS/Data/Measure/MeasurementSummary.php

@@ -0,0 +1,92 @@
+<?php
+/**
+ * MeasurementSummary.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Measure;
+
+class MeasurementSummary
+{
+    private $type;
+    private $count = 0;
+    private $duration = 0.0;
+
+    private $checkpointCount = 0;
+    private $checkpointDuration = 0.0;
+
+    public function __construct(string $type)
+    {
+        $this->type = $type;
+    }
+
+    public function add(Measurement $measurement)
+    {
+        $this->count++;
+        $this->duration += $measurement->getDuration();
+    }
+
+    /**
+     * Get the measurement summary
+     * ['count' => #, 'duration' => s]
+     *
+     * @return array
+     */
+    public function get()
+    {
+        return [
+            'count' => $this->count,
+            'duration' => $this->duration,
+        ];
+    }
+
+    public function getCount()
+    {
+        return $this->count;
+    }
+
+    public function getType()
+    {
+        return $this->type;
+    }
+
+    public function getDuration()
+    {
+        return $this->duration;
+    }
+
+    public function checkpoint()
+    {
+        $this->checkpointCount = $this->count;
+        $this->checkpointDuration = $this->duration;
+    }
+
+    public function getCountDiff()
+    {
+        return $this->count - $this->checkpointCount;
+    }
+
+    public function getDurationDiff()
+    {
+        return $this->duration - $this->checkpointDuration;
+    }
+}

LibreNMS/Data/Store/BaseDatastore.php

@@ -0,0 +1,64 @@
+<?php
+/**
+ * BaseDatastore.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Store;
+
+use LibreNMS\Data\Measure\Measurement;
+use LibreNMS\Data\Measure\MeasurementCollection;
+use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
+
+abstract class BaseDatastore implements DatastoreContract
+{
+    private $stats;
+
+    public function __construct()
+    {
+        $this->stats = new MeasurementCollection();
+    }
+
+    /**
+     * Checks if the datastore wants rrdtags to be sent when issuing put()
+     *
+     * @return boolean
+     */
+    public function wantsRrdTags()
+    {
+        return true;
+    }
+
+    /**
+     * Record statistics for operation
+     * @param Measurement $stat
+     */
+    protected function recordStatistic(Measurement $stat)
+    {
+        $this->stats->record($stat);
+    }
+
+    public function getStats()
+    {
+        return $this->stats;
+    }
+}

LibreNMS/Data/Store/Datastore.php

@@ -0,0 +1,159 @@
+<?php
+/**
+ * Datastore.php
+ *
+ * Aggregates all enabled datastores and dispatches data to them
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Store;
+
+use LibreNMS\Config;
+use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
+
+class Datastore
+{
+    protected $stores;
+
+    /**
+     * Initialize and create the Datastore(s)
+     *
+     * @param array $options
+     * @return DatastoreContract
+     */
+    public static function init($options = [])
+    {
+        $opts = [
+            'r' => 'rrd.enable',
+            'f' => 'influxdb.enable',
+            'p' => 'prometheus.enable',
+            'g' => 'graphite.enable',
+        ];
+        foreach ($opts as $opt => $setting) {
+            if (isset($options[$opt])) {
+                Config::set($setting, false);
+            }
+        }
+
+        return app('Datastore');
+    }
+
+    public static function terminate()
+    {
+        \Rrd::close();
+    }
+
+    /**
+     * Datastore constructor.
+     * @param array $datastores Implement DatastoreInterface
+     */
+    public function __construct($datastores)
+    {
+        $this->stores = $datastores;
+    }
+
+    /**
+     * Disable a datastore for the rest of this run
+     *
+     * @param string $name
+     */
+    public function disable($name)
+    {
+        $store = app("LibreNMS\\Data\\Store\\$name");
+        $position = array_search($store, $this->stores);
+        if ($position !== false) {
+            c_echo("[%g$name Disabled%n]\n");
+            unset($this->stores[$position]);
+        } else {
+            c_echo("[%g$name is not a valid datastore name%n]\n");
+        }
+    }
+
+    /**
+     * Datastore-independent function which should be used for all polled metrics.
+     *
+     * RRD Tags:
+     *   rrd_def     RrdDefinition
+     *   rrd_name    array|string: the rrd filename, will be processed with rrd_name()
+     *   rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
+     *   rrd_step             int: rrd step, defaults to 300
+     *
+     * @param array $device
+     * @param string $measurement Name of this measurement
+     * @param array $tags tags for the data (or to control rrdtool)
+     * @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
+     *                            single values are allowed and will be paired with $measurement
+     */
+    public function put($device, $measurement, $tags, $fields)
+    {
+        // convenience conversion to allow calling with a single value, so, e.g., these are equivalent:
+        // data_update($device, 'mymeasurement', $tags, 1234);
+        //     AND
+        // data_update($device, 'mymeasurement', $tags, array('mymeasurement' => 1234));
+        if (!is_array($fields)) {
+            $fields = [$measurement => $fields];
+        }
+
+        foreach ($this->stores as $store) {
+            /** @var DatastoreContract $store */
+            // rrdtool_data_update() will only use the tags it deems relevant, so we pass all of them.
+            // However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
+            $temp_tags = $store->wantsRrdTags() ? $tags : $this->rrdTagFilter($tags);
+
+            $store->put($device, $measurement, $temp_tags, $fields);
+        }
+    }
+
+    /**
+     * Filter all elements with keys that start with 'rrd_'
+     *
+     * @param array $arr input array
+     * @return array Copy of $arr with all keys beginning with 'rrd_' removed.
+     */
+    private function rrdTagFilter($arr)
+    {
+        $result = [];
+        foreach ($arr as $k => $v) {
+            if (strpos($k, 'rrd_') === 0) {
+                continue;
+            }
+            $result[$k] = $v;
+        }
+        return $result;
+    }
+
+    /**
+     * Get all the active data stores
+     *
+     * @return array
+     */
+    public function getStores()
+    {
+        return $this->stores;
+    }
+
+    public function getStats()
+    {
+        return array_reduce($this->stores, function ($result, DatastoreContract $store) {
+            $result[$store->getName()] = $store->getStats();
+            return $result;
+        }, []);
+    }
+}

LibreNMS/Data/Store/Graphite.php

@@ -0,0 +1,146 @@
+<?php
+/**
+ * Graphite.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @copyright  2017 Falk Stern <https://github.com/fstern/>
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Store;
+
+use Carbon\Carbon;
+use LibreNMS\Config;
+use LibreNMS\Data\Measure\Measurement;
+use Log;
+
+class Graphite extends BaseDatastore
+{
+    protected $connection;
+
+    protected $prefix;
+
+    public function __construct(\Socket\Raw\Factory $socketFactory)
+    {
+        parent::__construct();
+        $host = Config::get('graphite.host');
+        $port = Config::get('graphite.port', 2003);
+        try {
+            $this->connection = $socketFactory->createClient("$host:$port");
+        } catch (\Exception $e) {
+            d_echo($e->getMessage());
+        }
+
+        if ($this->connection) {
+            Log::notice("Graphite connection made to $host");
+        } else {
+            Log::error("Graphite connection to $host has failed!");
+        }
+
+        $this->prefix = Config::get('graphite.prefix', '');
+    }
+
+    public function getName()
+    {
+        return 'Graphite';
+    }
+
+    public static function isEnabled()
+    {
+        return Config::get('graphite.enable', false);
+    }
+
+    /**
+     * Datastore-independent function which should be used for all polled metrics.
+     *
+     * RRD Tags:
+     *   rrd_def     RrdDefinition
+     *   rrd_name    array|string: the rrd filename, will be processed with rrd_name()
+     *   rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
+     *   rrd_step             int: rrd step, defaults to 300
+     *
+     * @param array $device
+     * @param string $measurement Name of this measurement
+     * @param array $tags tags for the data (or to control rrdtool)
+     * @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
+     *                            single values are allowed and will be paired with $measurement
+     */
+    public function put($device, $measurement, $tags, $fields)
+    {
+        if (!$this->connection) {
+            d_echo("Graphite Error: not connected\n");
+            return;
+        }
+
+        $timestamp = Carbon::now()->timestamp;
+
+        if ($measurement == 'ports') {
+            $measurement = 'ports|' . $tags['ifName'];
+        }
+
+        // metrics will be built as prefix.hostname.measurement.field value timestamp
+        // metric fields can not contain . as this is used by graphite as a field separator
+        $hostname = preg_replace('/\./', '_', $device['hostname']);
+        $measurement = preg_replace(['/\./', '/\//'], '_', $measurement);
+        $measurement = preg_replace('/\|/', '.', $measurement);
+        $measurement_name = preg_replace('/\./', '_', $tags['rrd_name']);
+        if (is_array($measurement_name)) {
+            $ms_name = implode(".", $measurement_name);
+        } else {
+            $ms_name = $measurement_name;
+        }
+        // remove the port-id tags from the metric
+        if (preg_match('/^port-id\d+/', $ms_name)) {
+            $ms_name = "";
+        }
+
+        foreach ($fields as $k => $v) {
+            // Send zero for fields without values
+            if (empty($v)) {
+                $v = 0;
+            }
+            $metric = implode(".", array_filter([$this->prefix, $hostname, $measurement, $ms_name, $k]));
+            $this->writeData($metric, $v, $timestamp);
+        }
+    }
+
+    /**
+     * @param $metric
+     * @param $value
+     * @param $timestamp
+     */
+    private function writeData($metric, $value, $timestamp)
+    {
+        try {
+            $stat = Measurement::start('write');
+
+            // Further sanitize the full metric before sending, whitespace isn't allowed
+            $metric = preg_replace('/\s+/', '_', $metric);
+
+            $line = implode(" ", [$metric, $value, $timestamp]);
+            Log::debug("Sending to Graphite: $line\n");
+            $this->connection->write("$line\n");
+
+            $this->recordStatistic($stat->end());
+        } catch (\Socket\Raw\Exception $e) {
+            Log::error('Graphite write error: ' . $e->getMessage());
+        }
+    }
+}

LibreNMS/Data/Store/InfluxDB.php

@@ -0,0 +1,189 @@
+<?php
+/**
+ * InfluxDB.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @copyright  2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Store;
+
+use InfluxDB\Client;
+use InfluxDB\Driver\UDP;
+use LibreNMS\Config;
+use LibreNMS\Data\Measure\Measurement;
+use Log;
+
+class InfluxDB extends BaseDatastore
+{
+    /** @var \InfluxDB\Database $connection */
+    private $connection;
+
+    public function __construct(\InfluxDB\Database $influx)
+    {
+        parent::__construct();
+        $this->connection = $influx;
+
+        // if the database doesn't exist, create it.
+        try {
+            if (!$influx->exists()) {
+                $influx->create();
+            }
+        } catch (\Exception $e) {
+            Log::warning('InfluxDB: Could not create database');
+        }
+    }
+
+    public function getName()
+    {
+        return 'InfluxDB';
+    }
+
+    public static function isEnabled()
+    {
+        return Config::get('influxdb.enable', false);
+    }
+
+    /**
+     * Datastore-independent function which should be used for all polled metrics.
+     *
+     * RRD Tags:
+     *   rrd_def     RrdDefinition
+     *   rrd_name    array|string: the rrd filename, will be processed with rrd_name()
+     *   rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
+     *   rrd_step             int: rrd step, defaults to 300
+     *
+     * @param array $device
+     * @param string $measurement Name of this measurement
+     * @param array $tags tags for the data (or to control rrdtool)
+     * @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
+     *                            single values are allowed and will be paired with $measurement
+     */
+    public function put($device, $measurement, $tags, $fields)
+    {
+        $stat = Measurement::start('write');
+        $tmp_fields = [];
+        $tmp_tags['hostname'] = $device['hostname'];
+        foreach ($tags as $k => $v) {
+            $v = preg_replace(['/ /', '/,/', '/=/'], ['\ ', '\,', '\='], $v);
+            if (empty($v)) {
+                $v = '_blank_';
+            }
+            $tmp_tags[$k] = $v;
+        }
+        foreach ($fields as $k => $v) {
+            if ($k == 'time') {
+                $k = 'rtime';
+            }
+
+            if (($value = $this->forceType($v)) !== null) {
+                $tmp_fields[$k] = $value;
+            }
+        }
+
+        if (empty($tmp_fields)) {
+            Log::warning('All fields empty, skipping update', ['orig_fields' => $fields]);
+            return;
+        }
+
+        Log::debug('InfluxDB data: ', [
+            'measurement' => $measurement,
+            'tags' => $tmp_tags,
+            'fields' => $tmp_fields,
+        ]);
+
+        try {
+            $points = [
+                new \InfluxDB\Point(
+                    $measurement,
+                    null, // the measurement value
+                    $tmp_tags,
+                    $tmp_fields // optional additional fields
+                )
+            ];
+
+            $this->connection->writePoints($points);
+            $this->recordStatistic($stat->end());
+        } catch (\Exception $e) {
+            Log::error('InfluxDB exception: ' . $e->getMessage());
+            Log::debug($e->getTraceAsString());
+        }
+    }
+
+    /**
+     * Create a new client and select the database
+     *
+     * @return \InfluxDB\Database
+     */
+    public static function createFromConfig()
+    {
+        $host = Config::get('influxdb.host', 'localhost');
+        $transport = Config::get('influxdb.transport', 'http');
+        $port = Config::get('influxdb.port', 8086);
+        $db = Config::get('influxdb.db', 'librenms');
+        $username = Config::get('influxdb.username', '');
+        $password = Config::get('influxdb.password', '');
+        $timeout = Config::get('influxdb.timeout', 0);
+        $verify_ssl = Config::get('influxdb.verifySSL', false);
+
+        $client = new Client($host, $port, $username, $password, $transport == 'https', $verify_ssl, $timeout, $timeout);
+
+        if ($transport == 'udp') {
+            $client->setDriver(new UDP($host, $port));
+        }
+
+        return $client->selectDB($db);
+    }
+
+    private function forceType($data)
+    {
+        /*
+         * It is not trivial to detect if something is a float or an integer, and
+         * therefore may cause breakages on inserts.
+         * Just setting every number to a float gets around this, but may introduce
+         * inefficiencies.
+         * I've left the detection statement in there for a possible change in future,
+         * but currently everything just gets set to a float.
+         */
+
+        if (is_numeric($data)) {
+            // If it is an Integer
+            if (ctype_digit($data)) {
+                return floatval($data);
+                // Else it is a float
+            } else {
+                return floatval($data);
+            }
+        }
+
+        return $data === 'U' ? null : $data;
+    }
+
+    /**
+     * Checks if the datastore wants rrdtags to be sent when issuing put()
+     *
+     * @return boolean
+     */
+    public function wantsRrdTags()
+    {
+        return false;
+    }
+}

LibreNMS/Data/Store/OpenTSDB.php

@@ -0,0 +1,145 @@
+<?php
+/**
+ * OpenTSDB.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @copyright  2017 Yacine Benamsili <https://github.com/yac01/librenms.git>
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Store;
+
+use Carbon\Carbon;
+use LibreNMS\Config;
+use LibreNMS\Data\Measure\Measurement;
+use Log;
+
+class OpenTSDB extends BaseDatastore
+{
+    /** @var \Socket\Raw\Socket $connection */
+    protected $connection;
+
+    public function __construct(\Socket\Raw\Factory $socketFactory)
+    {
+        parent::__construct();
+        $host = Config::get('opentsdb.host');
+        $port = Config::get('opentsdb.port', 2181);
+        try {
+            $this->connection = $socketFactory->createClient("$host:$port");
+        } catch (\Socket\Raw\Exception $e) {
+            Log::debug('OpenTSDB Error: ' . $e->getMessage());
+        }
+
+        if ($this->connection) {
+            Log::notice('Connected to OpenTSDB');
+        } else {
+            Log::error('Connection to OpenTSDB has failed!');
+        }
+    }
+
+    public function getName()
+    {
+        return 'OpenTSDB';
+    }
+
+    /**
+     * Datastore-independent function which should be used for all polled metrics.
+     *
+     * RRD Tags:
+     *   rrd_def     RrdDefinition
+     *   rrd_name    array|string: the rrd filename, will be processed with rrd_name()
+     *   rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
+     *   rrd_step             int: rrd step, defaults to 300
+     *
+     * @param array $device
+     * @param string $measurement Name of this measurement
+     * @param array $tags tags for the data (or to control rrdtool)
+     * @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
+     *                            single values are allowed and will be paired with $measurement
+     */
+    public function put($device, $measurement, $tags, $fields)
+    {
+        if (!$this->connection) {
+            Log::error("OpenTSDB Error: not connected\n");
+            return;
+        }
+
+        $flag = Config::get('opentsdb.co');
+        $timestamp = Carbon::now()->timestamp;
+        $tmp_tags = "hostname=".$device['hostname'];
+
+        foreach ($tags as $k => $v) {
+            $v = str_replace(array(' ',',','='), '_', $v);
+            if (!empty($v)) {
+                $tmp_tags = $tmp_tags ." ". $k ."=".$v;
+            }
+        }
+
+        if ($measurement == 'ports') {
+            foreach ($fields as $k => $v) {
+                $measurement = $k;
+                if ($flag == true) {
+                    $measurement = $measurement.".".$device['co'];
+                }
+
+                $this->putData('port.'.$measurement, $timestamp, $v, $tmp_tags);
+            }
+        } else {
+            if ($flag == true) {
+                $measurement = $measurement . '.' . $device['co'];
+            }
+
+            foreach ($fields as $k => $v) {
+                $tmp_tags_key = $tmp_tags . " " . "key" . "=" . $k;
+                $this->putData($measurement, $timestamp, $v, $tmp_tags_key);
+            }
+        }
+    }
+
+    private function putData($measurement, $timestamp, $value, $tags)
+    {
+        try {
+            $stat = Measurement::start('put');
+
+            $line = sprintf('put net.%s %d %f %s', strtolower($measurement), $timestamp, $value, $tags);
+            Log::debug("Sending to OpenTSDB: $line\n");
+            $this->connection->write("$line\n"); // send $line into OpenTSDB
+
+            $this->recordStatistic($stat->end());
+        } catch (\Socket\Raw\Exception $e) {
+            Log::error('OpenTSDB Error: ' . $e->getMessage());
+        }
+    }
+
+    public static function isEnabled()
+    {
+        return Config::get('opentsdb.enable', false);
+    }
+
+    /**
+     * Checks if the datastore wants rrdtags to be sent when issuing put()
+     *
+     * @return boolean
+     */
+    public function wantsRrdTags()
+    {
+        return false;
+    }
+}

LibreNMS/Data/Store/Prometheus.php

@@ -0,0 +1,132 @@
+<?php
+/**
+ * Prometheus.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2020 Tony Murray
+ * @copyright  2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Store;
+
+use GuzzleHttp\Exception\GuzzleException;
+use LibreNMS\Config;
+use LibreNMS\Data\Measure\Measurement;
+use Log;
+
+class Prometheus extends BaseDatastore
+{
+    private $client;
+    private $base_uri;
+    private $default_opts;
+    private $enabled;
+
+    public function __construct(\GuzzleHttp\Client $client)
+    {
+        parent::__construct();
+        $this->client = $client;
+
+        $url = Config::get('prometheus.url');
+        $job = Config::get('prometheus.job', 'librenms');
+        $this->base_uri = "$url/metrics/job/$job/instance/";
+
+        $this->default_opts = [
+            'headers' => ['Content-Type' => 'text/plain'],
+        ];
+        if ($proxy = get_proxy()) {
+            $this->default_opts['proxy'] = $proxy;
+        }
+
+        $this->enabled = self::isEnabled();
+    }
+
+    public function getName()
+    {
+        return 'Prometheus';
+    }
+
+    public static function isEnabled()
+    {
+        return Config::get('prometheus.enable', false);
+    }
+
+    public function put($device, $measurement, $tags, $fields)
+    {
+        $stat = Measurement::start('put');
+        // skip if needed
+        if (!$this->enabled) {
+            return;
+        }
+
+        try {
+            $vals = "";
+            $promtags = "/measurement/" . $measurement;
+
+            foreach ($fields as $k => $v) {
+                if ($v !== null) {
+                    $vals .= "$k $v\n";
+                }
+            }
+
+            foreach ($tags as $t => $v) {
+                if ($v !== null) {
+                    $promtags .= "/$t/$v";
+                }
+            }
+            $options = $this->getDefaultOptions();
+            $options['body'] = $vals;
+
+            $promurl = $this->base_uri . $device['hostname'] . $promtags;
+            $promurl = str_replace(" ", "-", $promurl); // Prometheus doesn't handle tags with spaces in url
+
+            Log::debug("Prometheus put $promurl: ", [
+                'measurement' => $measurement,
+                'tags' => $tags,
+                'fields' => $fields,
+                'vals' => $vals,
+            ]);
+
+            $result = $this->client->request('POST', $promurl, $options);
+
+            $this->recordStatistic($stat->end());
+
+            if ($result->getStatusCode() !== 200) {
+                Log::error('Prometheus Error: ' . $result->getReasonPhrase());
+            }
+        } catch (GuzzleException $e) {
+            Log::error("Prometheus Exception: " . $e->getMessage());
+        }
+    }
+
+    private function getDefaultOptions()
+    {
+        return $this->default_opts;
+    }
+
+    /**
+     * Checks if the datastore wants rrdtags to be sent when issuing put()
+     *
+     * @return boolean
+     */
+    public function wantsRrdTags()
+    {
+        return false;
+    }
+}

LibreNMS/Data/Store/Rrd.php

@@ -0,0 +1,496 @@
+<?php
+/**
+ * Rrd.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Data\Store;
+
+use LibreNMS\Config;
+use LibreNMS\Data\Measure\Measurement;
+use LibreNMS\Exceptions\FileExistsException;
+use LibreNMS\Proc;
+use Log;
+
+class Rrd extends BaseDatastore
+{
+    private $disabled = false;
+
+    /** @var Proc $sync_process */
+    private $sync_process;
+    /** @var Proc $async_process */
+    private $async_process;
+    private $rrd_dir;
+    private $version;
+    private $rrdcached;
+    private $rra;
+    private $step;
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->rrdcached = Config::get('rrdcached', false);
+
+        $this->init();
+        $this->rrd_dir = Config::get('rrd_dir', Config::get('install_dir') . '/rrd');
+        $this->step = Config::get('rrd.step', 300);
+        $this->rra = Config::get(
+            'rrd_rra',
+            'RRA:AVERAGE:0.5:1:2016 RRA:AVERAGE:0.5:6:1440 RRA:AVERAGE:0.5:24:1440 RRA:AVERAGE:0.5:288:1440 ' .
+            ' RRA:MIN:0.5:1:720 RRA:MIN:0.5:6:1440     RRA:MIN:0.5:24:775     RRA:MIN:0.5:288:797 ' .
+            ' RRA:MAX:0.5:1:720 RRA:MAX:0.5:6:1440     RRA:MAX:0.5:24:775     RRA:MAX:0.5:288:797 ' .
+            ' RRA:LAST:0.5:1:1440 '
+        );
+        $this->version = Config::get('rrdtool_version', '1.4');
+    }
+
+    public function getName()
+    {
+        return 'RRD';
+    }
+
+    public static function isEnabled()
+    {
+        return Config::get('rrd.enable', true);
+    }
+
+    /**
+     * Opens up a pipe to RRDTool using handles provided
+     *
+     * @param bool $dual_process start an additional process that's output should be read after every command
+     * @return bool the process(s) have been successfully started
+     */
+    public function init($dual_process = true)
+    {
+        $command = Config::get('rrdtool', 'rrdtool') . ' -';
+
+        $descriptor_spec = [
+            0 => ['pipe', 'r'], // stdin  is a pipe that the child will read from
+            1 => ['pipe', 'w'], // stdout is a pipe that the child will write to
+            2 => ['pipe', 'w'], // stderr is a pipe that the child will write to
+        ];
+
+        $cwd = Config::get('rrd_dir');
+
+        if (!$this->isSyncRunning()) {
+            $this->sync_process = new Proc($command, $descriptor_spec, $cwd);
+        }
+
+        if ($dual_process && !$this->isAsyncRunning()) {
+            $this->async_process = new Proc($command, $descriptor_spec, $cwd);
+            $this->async_process->setSynchronous(false);
+        }
+
+        return $this->isSyncRunning() && ($dual_process ? $this->isAsyncRunning() : true);
+    }
+
+
+    public function isSyncRunning()
+    {
+        return isset($this->sync_process) && $this->sync_process->isRunning();
+    }
+
+    public function isAsyncRunning()
+    {
+        return isset($this->async_process) && $this->async_process->isRunning();
+    }
+
+    /**
+     * Close all open rrdtool processes.
+     * This should be done before exiting
+     */
+    public function close()
+    {
+        if ($this->isSyncRunning()) {
+            $this->sync_process->close('quit');
+        }
+        if ($this->isAsyncRunning()) {
+            $this->async_process->close('quit');
+        }
+    }
+
+    /**
+     * rrdtool backend implementation of data_update
+     *
+     * Tags:
+     *   rrd_def     RrdDefinition
+     *   rrd_name    array|string: the rrd filename, will be processed with rrd_name()
+     *   rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
+     *   rrd_step             int: rrd step, defaults to 300
+     *
+     * @param array $device device array
+     * @param string $measurement the name of this measurement (if no rrd_name tag is given, this will be used to name the file)
+     * @param array $tags tags to pass additional info to rrdtool
+     * @param array $fields data values to update
+     */
+    public function put($device, $measurement, $tags, $fields)
+    {
+        $rrd_name = isset($tags['rrd_name']) ? $tags['rrd_name'] : $measurement;
+        $step = isset($tags['rrd_step']) ? $tags['rrd_step'] : $this->step;
+        if (!empty($tags['rrd_oldname'])) {
+            self::renameFile($device, $tags['rrd_oldname'], $rrd_name);
+        }
+
+        if (isset($tags['rrd_proxmox_name'])) {
+            $pmxvars = $tags['rrd_proxmox_name'];
+            $rrd = self::proxmoxName($pmxvars['pmxcluster'], $pmxvars['vmid'], $pmxvars['vmport']);
+        } else {
+            $rrd = self::name($device['hostname'], $rrd_name);
+        }
+
+        if (isset($tags['rrd_def'])) {
+            $rrd_def = $tags['rrd_def'];
+
+            // filter out data not in the definition
+            $fields = array_filter($fields, function ($key) use ($rrd_def) {
+                $valid = $rrd_def->isValidDataset($key);
+                if (!$valid) {
+                    Log::warning("RRD warning: unused data sent $key");
+                }
+                return $valid;
+            }, ARRAY_FILTER_USE_KEY);
+
+            if (!$this->checkRrdExists($rrd)) {
+                $newdef = "--step $step $rrd_def $this->rra";
+                $this->command('create', $rrd, $newdef);
+            }
+        }
+
+        $this->update($rrd, $fields);
+    }
+
+    /**
+     * Updates an rrd database at $filename using $options
+     * Where $options is an array, each entry which is not a number is replaced with "U"
+     *
+     * @internal
+     * @param string $filename
+     * @param array $data
+     * @return array|string
+     */
+    public function update($filename, $data)
+    {
+        $values = [];
+        // Do some sanitation on the data if passed as an array.
+
+        if (is_array($data)) {
+            $values[] = 'N';
+            foreach ($data as $v) {
+                if (!is_numeric($v)) {
+                    $v = 'U';
+                }
+
+                $values[] = $v;
+            }
+
+            $data = implode(':', $values);
+            return $this->command('update', $filename, $data);
+        } else {
+            return 'Bad options passed to rrdtool_update';
+        }
+    } // rrdtool_update
+
+    /**
+     * Modify an rrd file's max value and trim the peaks as defined by rrdtool
+     *
+     * @param string $type only 'port' is supported at this time
+     * @param string $filename the path to the rrd file
+     * @param integer $max the new max value
+     * @return bool
+     */
+    public function tune($type, $filename, $max)
+    {
+        $fields = [];
+        if ($type === 'port') {
+            if ($max < 10000000) {
+                return false;
+            }
+            $max = $max / 8;
+            $fields = [
+                'INOCTETS',
+                'OUTOCTETS',
+                'INERRORS',
+                'OUTERRORS',
+                'INUCASTPKTS',
+                'OUTUCASTPKTS',
+                'INNUCASTPKTS',
+                'OUTNUCASTPKTS',
+                'INDISCARDS',
+                'OUTDISCARDS',
+                'INUNKNOWNPROTOS',
+                'INBROADCASTPKTS',
+                'OUTBROADCASTPKTS',
+                'INMULTICASTPKTS',
+                'OUTMULTICASTPKTS'
+            ];
+        }
+        if (count($fields) > 0) {
+            $options = "--maximum " . implode(":$max --maximum ", $fields) . ":$max";
+            $this->command('tune', $filename, $options);
+        }
+        return true;
+    } // rrdtool_tune
+
+    /**
+     * Generates a filename for a proxmox cluster rrd
+     *
+     * @param $pmxcluster
+     * @param $vmid
+     * @param $vmport
+     * @return string full path to the rrd.
+     */
+    public function proxmoxName($pmxcluster, $vmid, $vmport)
+    {
+        $pmxcdir = join('/', [$this->rrd_dir, 'proxmox', self::safeName($pmxcluster)]);
+        // this is not needed for remote rrdcached
+        if (!is_dir($pmxcdir)) {
+            mkdir($pmxcdir, 0775, true);
+        }
+
+        return join('/', [$pmxcdir, self::safeName($vmid . '_netif_' . $vmport . '.rrd')]);
+    }
+
+    /**
+     * rename an rrdfile, can only be done on the LibreNMS server hosting the rrd files
+     *
+     * @param array $device Device object
+     * @param string|array $oldname RRD name array as used with rrd_name()
+     * @param string|array $newname RRD name array as used with rrd_name()
+     * @return bool indicating rename success or failure
+     */
+    public function renameFile($device, $oldname, $newname)
+    {
+        $oldrrd = self::name($device['hostname'], $oldname);
+        $newrrd = self::name($device['hostname'], $newname);
+        if (is_file($oldrrd) && !is_file($newrrd)) {
+            if (rename($oldrrd, $newrrd)) {
+                log_event("Renamed $oldrrd to $newrrd", $device, "poller", 1);
+                return true;
+            } else {
+                log_event("Failed to rename $oldrrd to $newrrd", $device, "poller", 5);
+                return false;
+            }
+        } else {
+            // we don't need to rename the file
+            return true;
+        }
+    }
+
+    /**
+     * Generates a filename based on the hostname (or IP) and some extra items
+     *
+     * @param string $host Host name
+     * @param array|string $extra Components of RRD filename - will be separated with "-", or a pre-formed rrdname
+     * @param string $extension File extension (default is .rrd)
+     * @return string the name of the rrd file for $host's $extra component
+     */
+    public function name($host, $extra, $extension = ".rrd")
+    {
+        $filename = self::safeName(is_array($extra) ? implode("-", $extra) : $extra);
+        return implode("/", [$this->dirFromHost($host), $filename . $extension]);
+    }
+
+    /**
+     * Generates a path based on the hostname (or IP)
+     *
+     * @param string $host Host name
+     * @return string the name of the rrd directory for $host
+     */
+    public function dirFromHost($host)
+    {
+        $host = str_replace(':', '_', trim($host, '[]'));
+        return implode("/", [$this->rrd_dir, $host]);
+    }
+
+    /**
+     * Generates and pipes a command to rrdtool
+     *
+     * @internal
+     * @param string $command create, update, updatev, graph, graphv, dump, restore, fetch, tune, first, last, lastupdate, info, resize, xport, flushcached
+     * @param string $filename The full patth to the rrd file
+     * @param string $options rrdtool command options
+     * @return array the output of stdout and stderr in an array
+     * @throws \Exception thrown when the rrdtool process(s) cannot be started
+     */
+    private function command($command, $filename, $options)
+    {
+        global $vdebug;
+        $stat = Measurement::start($this->coalesceStatisticType($command));
+
+        try {
+            $cmd = self::buildCommand($command, $filename, $options);
+        } catch (FileExistsException $e) {
+            Log::debug("RRD[%g$filename already exists%n]", ['color' => true]);
+            return [null, null];
+        }
+
+        Log::debug("RRD[%g$cmd%n]", ['color' => true]);
+
+        // do not write rrd files, but allow read-only commands
+        $ro_commands = ['graph', 'graphv', 'dump', 'fetch', 'first', 'last', 'lastupdate', 'info', 'xport'];
+        if ($this->disabled && !in_array($command, $ro_commands)) {
+            if (!Config::get('hide_rrd_disabled')) {
+                Log::debug('[%rRRD Disabled%n]', ['color' => true]);
+            }
+            return [null, null];
+        }
+
+        // send the command!
+        if ($command == 'last' && $this->init(false)) {
+            // send this to our synchronous process so output is guaranteed
+            $output = $this->sync_process->sendCommand($cmd);
+        } elseif ($this->init()) {
+            // don't care about the return of other commands, so send them to the faster async process
+            $output = $this->async_process->sendCommand($cmd);
+        } else {
+            Log::error('rrdtool could not start');
+        }
+
+        if ($vdebug) {
+            echo 'RRDtool Output: ';
+            echo $output[0];
+            echo $output[1];
+        }
+
+        $this->recordStatistic($stat->end());
+        return $output;
+    }
+
+    /**
+     * Build a command for rrdtool
+     * Shortens the filename as needed
+     * Determines if --daemon and -O should be used
+     *
+     * @internal
+     * @param string $command The base rrdtool command.  Usually create, update, last.
+     * @param string $filename The full path to the rrd file
+     * @param string $options Options for the command possibly including the rrd definition
+     * @return string returns a full command ready to be piped to rrdtool
+     * @throws FileExistsException if rrdtool <1.4.3 and the rrd file exists locally
+     */
+    private function buildCommand($command, $filename, $options)
+    {
+        if ($command == 'create') {
+            // <1.4.3 doesn't support -O, so make sure the file doesn't exist
+            if (version_compare($this->version, '1.4.3', '<')) {
+                if (is_file($filename)) {
+                    throw new FileExistsException();
+                }
+            } else {
+                $options .= ' -O';
+            }
+        }
+
+        // no remote for create < 1.5.5 and tune < 1.5
+        if ($this->rrdcached &&
+            !($command == 'create' && version_compare($this->version, '1.5.5', '<')) &&
+            !($command == 'tune' && $this->rrdcached && version_compare($this->version, '1.5', '<'))
+        ) {
+            // only relative paths if using rrdcached
+            $filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
+            $options = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $options);
+
+            return "$command $filename $options --daemon " . $this->rrdcached;
+        }
+
+        return "$command $filename $options";
+    }
+
+    /**
+     * Checks if the rrd file exists on the server
+     * This will perform a remote check if using rrdcached and rrdtool >= 1.5
+     *
+     * @param string $filename full path to the rrd file
+     * @return bool whether or not the passed rrd file exists
+     */
+    public function checkRrdExists($filename)
+    {
+        if ($this->rrdcached && version_compare($this->version, '1.5', '>=')) {
+            $chk = $this->command('last', $filename, '');
+            $filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
+            return !str_contains(implode($chk), "$filename': No such file or directory");
+        } else {
+            return is_file($filename);
+        }
+    }
+
+    /**
+     * Generates a graph file at $graph_file using $options
+     * Opens its own rrdtool pipe.
+     *
+     * @param string $graph_file
+     * @param string $options
+     * @return integer
+     */
+    public function graph($graph_file, $options)
+    {
+        if ($this->init(false)) {
+            $cmd = $this->buildCommand('graph', $graph_file, $options);
+
+            $output = implode($this->sync_process->sendCommand($cmd));
+
+            d_echo("<p>$cmd</p>\n<p>command returned ($output)</p>");
+
+            return $output;
+        } else {
+            return 0;
+        }
+    }
+
+    public function __destruct()
+    {
+        $this->close();
+    }
+
+    /**
+     * Remove invalid characters from the rrd file name
+     *
+     * @param string $name
+     * @return string
+     */
+    public static function safeName($name)
+    {
+        return (string)preg_replace('/[^a-zA-Z0-9,._\-]/', '_', $name);
+    }
+
+    /**
+     * Remove invalid characters from the rrd description
+     *
+     * @param string $descr
+     * @return string
+     */
+    public static function safeDescr($descr)
+    {
+        return (string)preg_replace('/[^a-zA-Z0-9,._\-\/\ ]/', ' ', $descr);
+    }
+
+    /**
+     * Only track update and create primarily, just put all others in an "other" bin
+     *
+     * @param $type
+     * @return string
+     */
+    private function coalesceStatisticType($type)
+    {
+        return ($type == 'update' || $type == 'create') ? $type : 'other';
+    }
+}

LibreNMS/Interfaces/Data/Datastore.php

@@ -0,0 +1,74 @@
+<?php
+/**
+ * Datastore.php
+ *
+ * Interface for datastores. Will be used to send them data through the put() method
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Interfaces\Data;
+
+interface Datastore
+{
+    /**
+     * Check if this is enabled by the configuration
+     *
+     * @return boolean
+     */
+    public static function isEnabled();
+
+    /**
+     * Checks if the datastore wants rrdtags to be sent when issuing put()
+     *
+     * @return boolean
+     */
+    public function wantsRrdTags();
+
+    /**
+     * The name of this datastore
+     *
+     * @return string
+     */
+    public function getName();
+
+    /**
+     * Array of stats should be [type => [count => n, time => s]]
+     *
+     * @return array
+     */
+    public function getStats();
+
+    /**
+     * Datastore-independent function which should be used for all polled metrics.
+     *
+     * RRD Tags:
+     *   rrd_def     RrdDefinition
+     *   rrd_name    array|string: the rrd filename, will be processed with rrd_name()
+     *   rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
+     *   rrd_step             int: rrd step, defaults to 300
+     *
+     * @param array $device
+     * @param string $measurement Name of this measurement
+     * @param array $tags tags for the data (or to control rrdtool)
+     * @param array|mixed $fields The data to update in an associative array, the order must be consistent with rrd_def,
+     *                            single values are allowed and will be paired with $measurement
+     */
+    public function put($device, $measurement, $tags, $fields);
+}

LibreNMS/RRD/RrdDefinition.php

@@ -30,8 +30,9 @@ use LibreNMS\Exceptions\InvalidRrdTypeException;
 
 class RrdDefinition
 {
-    private static $types = array('GAUGE', 'DERIVE', 'COUNTER', 'ABSOLUTE', 'DCOUNTER', 'DDERIVE');
+    private static $types = ['GAUGE', 'DERIVE', 'COUNTER', 'ABSOLUTE', 'DCOUNTER', 'DDERIVE'];
-    private $dataSets = array();
+    private $dataSets = [];
+    private $skipNameCheck = false;
 
     /**
      * Make a new empty RrdDefinition
@@ -58,14 +59,14 @@ class RrdDefinition
             d_echo("DS must be set to a non-empty string.");
         }
 
-        $ds = array();
+        $name = $this->escapeName($name);
-        $ds[] = $this->escapeName($name);
+        $this->dataSets[$name] = [
-        $ds[] = $this->checkType($type);
+            $name,
-        $ds[] = is_null($heartbeat) ? Config::get('rrd.heartbeat') : $heartbeat;
+            $this->checkType($type),
-        $ds[] = is_null($min) ? 'U' : $min;
+            is_null($heartbeat) ? Config::get('rrd.heartbeat') : $heartbeat,
-        $ds[] = is_null($max) ? 'U' : $max;
+            is_null($min) ? 'U' : $min,
-
+            is_null($max) ? 'U' : $max,
-        $this->dataSets[] = $ds;
+        ];
 
         return $this;
     }
@@ -82,6 +83,22 @@ class RrdDefinition
         }, '');
     }
 
+    /**
+     * Check if the give dataset name is valid for this definition
+     *
+     * @param $name
+     * @return bool
+     */
+    public function isValidDataset($name)
+    {
+        return $this->skipNameCheck || isset($this->dataSets[$name]);
+    }
+
+    public function disableNameChecking()
+    {
+        $this->skipNameCheck = true;
+    }
+
     /**
      * Check that the data set type is valid.
      *

LibreNMS/Util/GitHub.php

@@ -367,7 +367,10 @@ GRAPHQL;
         }
 
         if (!isset($previous_release['published_at'])) {
-            throw new Exception("Could not find previous release tag.");
+            throw new Exception(
+                $previous_release['message'] ??
+                "Could not find previous release tag. ($this->from)"
+            );
         }
 
         $this->getPullRequests($previous_release['published_at']);

LibreNMS/Validations/RrdCheck.php

@@ -58,7 +58,7 @@ class RrdCheck extends BaseValidation
         $screenpad = 0;
 
         foreach ($rrd_iterator as $filename => $file) {
-            $rrd_test_result = rrdtest($filename, $output, $error);
+            $rrd_test_result = $this->test($filename, $output, $error);
 
             $loopcount++;
             if (($loopcount % 50) == 0) {
@@ -80,4 +80,44 @@ class RrdCheck extends BaseValidation
         echo "\033[" . $screenpad . "D";
         echo "Status: " . $loopcount . "/" . $rrd_total . " - Complete\n";
     }
+
+    /**
+     * Run rrdtool info on a file path
+     *
+     * @param string $path Path to pass to rrdtool info
+     * @param string $stdOutput Variable to recieve the output of STDOUT
+     * @param string $stdError Variable to recieve the output of STDERR
+     *
+     * @return int exit code
+     *
+     **/
+    private function test($path, &$stdOutput, &$stdError)
+    {
+        //rrdtool info <escaped rrd path>
+        $command = Config::get('rrdtool') . ' info ' . escapeshellarg($path);
+        $process = proc_open(
+            $command,
+            array (
+                0 => array('pipe', 'r'),
+                1 => array('pipe', 'w'),
+                2 => array('pipe', 'w'),
+            ),
+            $pipes
+        );
+
+        if (!is_resource($process)) {
+            throw new \RuntimeException('Could not create a valid process');
+        }
+
+        $status = proc_get_status($process);
+        while ($status['running']) {
+            usleep(2000); // Sleep 2000 microseconds or 2 milliseconds
+            $status = proc_get_status($process);
+        }
+
+        $stdOutput = stream_get_contents($pipes[1]);
+        $stdError  = stream_get_contents($pipes[2]);
+        proc_close($process);
+        return $status['exitcode'];
+    }
 }

app/Facades/Rrd.php

@@ -0,0 +1,36 @@
+<?php
+/**
+ * Rrd.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace App\Facades;
+
+use Illuminate\Support\Facades\Facade;
+
+class Rrd extends Facade
+{
+    protected static function getFacadeAccessor()
+    {
+        return 'LibreNMS\Data\Store\Rrd';
+    }
+}

app/Jobs/PingCheck.php

@@ -260,7 +260,7 @@ class PingCheck implements ShouldQueue
             $rules->runRules($device->device_id);
 
             // add data to rrd
-            data_update($device->toArray(), 'ping-perf', $this->rrd_tags, ['ping' => $device->last_ping_timetaken]);
+            app('Datastore')->put($device->toArray(), 'ping-perf', $this->rrd_tags, ['ping' => $device->last_ping_timetaken]);
 
             // done with this device
             $this->complete($device->hostname);

app/Providers/DatastoreServiceProvider.php

@@ -0,0 +1,74 @@
+<?php
+/**
+ * DatastoreServiceProvider.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace App\Providers;
+
+use Illuminate\Contracts\Foundation\Application;
+use Illuminate\Support\ServiceProvider;
+use LibreNMS\Data\Store\Datastore;
+use LibreNMS\Interfaces\Data\Datastore as DatastoreContract;
+
+class DatastoreServiceProvider extends ServiceProvider
+{
+    protected $namespace = 'LibreNMS\\Data\\Store\\';
+    protected $stores = [
+        'LibreNMS\Data\Store\Graphite',
+        'LibreNMS\Data\Store\InfluxDB',
+        'LibreNMS\Data\Store\OpenTSDB',
+        'LibreNMS\Data\Store\Prometheus',
+        'LibreNMS\Data\Store\Rrd',
+    ];
+
+    public function register()
+    {
+        // set up bindings
+        foreach ($this->stores as $store) {
+            $this->app->singleton($store);
+        }
+
+        // bind the Datastore object
+        $this->app->singleton('Datastore', function (Application $app, $options) {
+            // only tag datastores enabled by config
+            $stores = array_filter($this->stores, function ($store) {
+                /** @var DatastoreContract $store */
+                return $store::isEnabled();
+            });
+
+            $app->tag($stores, ['datastore']);
+
+            return new Datastore(iterator_to_array($app->tagged('datastore')));
+        });
+
+        // additional bindings
+        $this->registerInflux();
+    }
+
+    public function registerInflux()
+    {
+        $this->app->singleton('InfluxDB\Database', function ($app) {
+            return \LibreNMS\Data\Store\InfluxDB::createFromConfig();
+        });
+    }
+}

check-services.php

@@ -13,31 +13,19 @@
  * the source code distribution for details.
  */
 
+use LibreNMS\Data\Store\Datastore;
+
 $init_modules = array();
 require __DIR__ . '/includes/init.php';
 
-$options = getopt('d::h:f:;');
+$options = getopt('drfpgh:');
 if (set_debug(isset($options['d']))) {
     echo "DEBUG!\n";
 }
 
-if (isset($options['f'])) {
-    \LibreNMS\Config::set('noinfluxdb', true);
-}
-
-if (isset($options['p'])) {
-    $prometheus = false;
-}
-
-if (\LibreNMS\Config::get('noinfluxdb') !== true && \LibreNMS\Config::get('influxdb.enable') === true) {
-    $influxdb = influxdb_connect();
-} else {
-    $influxdb = false;
-}
-
 $poller_start = microtime(true);
 
-rrdtool_initialize();
+$datastore = Datastore::init($options);
 
 echo "Starting service polling run:\n\n";
 $polled_services = 0;
@@ -96,4 +84,4 @@ $string = $argv[0] . " " . date(\LibreNMS\Config::get('dateformat.compact'))
     ." - $polled_services services polled in $poller_time secs";
 d_echo("$string\n");
 
-rrdtool_close();
+Datastore::terminate();

composer.json

@@ -30,6 +30,7 @@
         "ext-session": "*",
         "ext-xml": "*",
         "amenadiel/jpgraph": "^3.6",
+        "clue/socket-raw": "^1.4",
         "dapphp/radius": "^2.0",
         "darkghosthunter/larapoke": "^3.1",
         "doctrine/dbal": "^2.9",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "af1e9508013c6187a1ebaf4beaf59223",
+    "content-hash": "206e47998ed4d125251600e26fe67462",
     "packages": [
         {
             "name": "amenadiel/jpgraph",
@@ -61,6 +61,61 @@
             ],
             "time": "2018-10-14T21:43:30+00:00"
         },
+        {
+            "name": "clue/socket-raw",
+            "version": "v1.4.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/clue/php-socket-raw.git",
+                "reference": "00ab102d061f6cdb895e79dd4d69140c7bda31cc"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/clue/php-socket-raw/zipball/00ab102d061f6cdb895e79dd4d69140c7bda31cc",
+                "reference": "00ab102d061f6cdb895e79dd4d69140c7bda31cc",
+                "shasum": ""
+            },
+            "require": {
+                "ext-sockets": "*",
+                "php": ">=5.3"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^7.0 || ^6.0 || ^5.2 || ^4.8.35"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Socket\\Raw\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Lück",
+                    "email": "christian@clue.engineering"
+                }
+            ],
+            "description": "Simple and lightweight OOP wrapper for PHP's low-level sockets extension (ext-sockets)",
+            "homepage": "https://github.com/clue/php-socket-raw",
+            "keywords": [
+                "Socket",
+                "client",
+                "datagram",
+                "dgram",
+                "icmp",
+                "ipv6",
+                "server",
+                "stream",
+                "tcp",
+                "udg",
+                "udp",
+                "unix"
+            ],
+            "time": "2019-10-28T12:32:07+00:00"
+        },
         {
             "name": "dapphp/radius",
             "version": "2.5.4",

config/app.php

@@ -180,6 +180,7 @@ return [
         App\Providers\RouteServiceProvider::class,
         App\Providers\ComposerServiceProvider::class,
         App\Providers\ViewServiceProvider::class,
+        App\Providers\DatastoreServiceProvider::class,
 
         /*
          * LibreNMS Service Providers...
@@ -241,6 +242,7 @@ return [
         // LibreNMS
         'Permissions' => \App\Facades\Permissions::class,
         'DeviceCache' => \App\Facades\DeviceCache::class,
+        'Rrd' => App\Facades\Rrd::class,
     ],
 
 ];

html/graph.php

@@ -9,6 +9,9 @@
  * @copyright  (C) 2006 - 2012 Adam Armstrong
  */
 
+use LibreNMS\Authentication\LegacyAuth;
+use LibreNMS\Data\Store\Datastore;
+
 $start = microtime(true);
 
 $init_modules = array('web', 'graphs', 'auth');
@@ -22,11 +25,9 @@ if (!$auth) {
 
 set_debug(isset($_GET['debug']));
 
-rrdtool_initialize(false);
-
 require \LibreNMS\Config::get('install_dir') . '/includes/html/graphs/graph.inc.php';
 
-rrdtool_close();
+Datastore::terminate();
 
 if ($debug) {
     echo '<br />';

includes/common.php

@@ -436,7 +436,7 @@ function getidbyname($hostname)
 
 function safename($name)
 {
-    return preg_replace('/[^a-zA-Z0-9,._\-]/', '_', $name);
+    return \LibreNMS\Data\Store\Rrd::safeName($name);
 }
 
 /**
@@ -446,7 +446,7 @@ function safename($name)
  */
 function safedescr($descr)
 {
-    return preg_replace('/[^a-zA-Z0-9,._\-\/\ ]/', ' ', $descr);
+    return \LibreNMS\Data\Store\Rrd::safeDescr($descr);
 }
 
 function zeropad($num, $length = 2)

includes/datastore.inc.php

@@ -18,25 +18,6 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-/**
- * Filter all elements with keys that start with 'rrd_'
- *
- * @param array $arr input array
- * @return array Copy of $arr with all keys beginning with 'rrd_' removed.
- */
-function rrd_array_filter($arr)
-{
-    $result = array();
-    foreach ($arr as $k => $v) {
-        if (strpos($k, 'rrd_') === 0) {
-            continue;
-        }
-        $result[$k] = $v;
-    }
-    return $result;
-} // rrd_array_filter
-
-
 /**
  * Datastore-independent function which should be used for all polled metrics.
  *
@@ -54,20 +35,6 @@ function rrd_array_filter($arr)
  */
 function data_update($device, $measurement, $tags, $fields)
 {
-    // convenience conversion to allow calling with a single value, so, e.g., these are equivalent:
+    $datastore = app('Datastore');
-    // data_update($device, 'mymeasurement', $tags, 1234);
+    $datastore->put($device, $measurement, $tags, $fields);
-    //     AND
+}
-    // data_update($device, 'mymeasurement', $tags, array('mymeasurement' => 1234));
-    if (!is_array($fields)) {
-        $fields = array($measurement => $fields);
-    }
-
-    // rrdtool_data_update() will only use the tags it deems relevant, so we pass all of them.
-    // However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
-
-    rrdtool_data_update($device, $measurement, $tags, $fields);
-    prometheus_push($device, $measurement, rrd_array_filter($tags), $fields);
-    influx_update($device, $measurement, rrd_array_filter($tags), $fields);
-    opentsdb_update($device, $measurement, rrd_array_filter($tags), $fields);
-    graphite_update($device, $measurement, $tags, $fields);
-} // data_update

includes/functions.php

@@ -344,26 +344,6 @@ function percent_colour($perc)
     return sprintf('#%02x%02x%02x', $r, $b, $b);
 }
 
-// Returns the last in/out errors value in RRD
-function interface_errors($rrd_file, $period = '-1d')
-{
-    $errors = array();
-
-    $cmd = Config::get('rrdtool') . " fetch -s $period -e -300s $rrd_file AVERAGE | grep : | cut -d\" \" -f 4,5";
-    $data = trim(shell_exec($cmd));
-    $in_errors = 0;
-    $out_errors = 0;
-    foreach (explode("\n", $data) as $entry) {
-        list($in, $out) = explode(" ", $entry);
-        $in_errors += ($in * 300);
-        $out_errors += ($out * 300);
-    }
-    $errors['in'] = round($in_errors);
-    $errors['out'] = round($out_errors);
-
-    return $errors;
-}
-
 /**
  * @param $device
  * @return string the logo image path for this device. Images are often wide, not square.
@@ -1542,30 +1522,6 @@ function function_check($function)
     return function_exists($function);
 }
 
-function force_influx_data($data)
-{
-   /*
-    * It is not trivial to detect if something is a float or an integer, and
-    * therefore may cause breakages on inserts.
-    * Just setting every number to a float gets around this, but may introduce
-    * inefficiencies.
-    * I've left the detection statement in there for a possible change in future,
-    * but currently everything just gets set to a float.
-    */
-
-    if (is_numeric($data)) {
-        // If it is an Integer
-        if (ctype_digit($data)) {
-            return floatval($data);
-        // Else it is a float
-        } else {
-            return floatval($data);
-        }
-    } else {
-        return $data;
-    }
-}// end force_influx_data
-
 /**
  * Try to determine the address family (IPv4 or IPv6) associated with an SNMP
  * transport specifier (like "udp", "udp6", etc.).
@@ -1662,50 +1618,6 @@ function dnslookup($device, $type = false, $return = false)
     return $record[0][$return];
 }//end dnslookup
 
-
-
-
-/**
- * Run rrdtool info on a file path
- *
- * @param string $path Path to pass to rrdtool info
- * @param string $stdOutput Variable to recieve the output of STDOUT
- * @param string $stdError Variable to recieve the output of STDERR
- *
- * @return int exit code
- *
-**/
-
-function rrdtest($path, &$stdOutput, &$stdError)
-{
-    //rrdtool info <escaped rrd path>
-    $command = Config::get('rrdtool') . ' info ' . escapeshellarg($path);
-    $process = proc_open(
-        $command,
-        array (
-            0 => array('pipe', 'r'),
-            1 => array('pipe', 'w'),
-            2 => array('pipe', 'w'),
-        ),
-        $pipes
-    );
-
-    if (!is_resource($process)) {
-        throw new \RuntimeException('Could not create a valid process');
-    }
-
-    $status = proc_get_status($process);
-    while ($status['running']) {
-        usleep(2000); // Sleep 2000 microseconds or 2 milliseconds
-        $status = proc_get_status($process);
-    }
-
-    $stdOutput = stream_get_contents($pipes[1]);
-    $stdError  = stream_get_contents($pipes[2]);
-    proc_close($process);
-    return $status['exitcode'];
-}
-
 /**
  * Create a new state index.  Update translations if $states is given.
  *
@@ -2062,10 +1974,9 @@ function get_toner_levels($device, $raw_value, $capacity)
  */
 function initStats()
 {
-    global $snmp_stats, $rrd_stats;
+    global $snmp_stats, $snmp_stats_last;
-    global $snmp_stats_last, $rrd_stats_last;
 
-    if (!isset($snmp_stats, $rrd_stats)) {
+    if (!isset($snmp_stats)) {
         $snmp_stats = array(
             'ops' => array(
                 'snmpget' => 0,
@@ -2079,20 +1990,6 @@ function initStats()
             )
         );
         $snmp_stats_last = $snmp_stats;
-
-        $rrd_stats = array(
-            'ops' => array(
-                'update' => 0,
-                'create' => 0,
-                'other' => 0,
-            ),
-            'time' => array(
-                'update' => 0.0,
-                'create' => 0.0,
-                'other' => 0.0,
-            ),
-        );
-        $rrd_stats_last = $rrd_stats;
     }
 }
 
@@ -2103,25 +2000,29 @@ function initStats()
  */
 function printChangedStats($update_only = false)
 {
-    global $snmp_stats, $db_stats, $rrd_stats;
+    global $snmp_stats, $db_stats;
-    global $snmp_stats_last, $db_stats_last, $rrd_stats_last;
+    global $snmp_stats_last, $db_stats_last;
-
+    $output = sprintf(
-    if (!$update_only) {
+        ">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs]",
-        printf(
-            ">> SNMP: [%d/%.2fs] MySQL: [%d/%.2fs] RRD: [%d/%.2fs]\n",
         array_sum($snmp_stats['ops']) - array_sum($snmp_stats_last['ops']),
         array_sum($snmp_stats['time']) - array_sum($snmp_stats_last['time']),
         array_sum($db_stats['ops']) - array_sum($db_stats_last['ops']),
-            array_sum($db_stats['time']) - array_sum($db_stats_last['time']),
+        array_sum($db_stats['time']) - array_sum($db_stats_last['time'])
-            array_sum($rrd_stats['ops']) - array_sum($rrd_stats_last['ops']),
-            array_sum($rrd_stats['time']) - array_sum($rrd_stats_last['time'])
     );
+
+    foreach (app('Datastore')->getStats() as $datastore => $stats) {
+        /** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
+        $output .= sprintf(" %s: [%d/%.2fs]", $datastore, $stats->getCountDiff(), $stats->getDurationDiff());
+        $stats->checkpoint();
+    }
+
+    if (!$update_only) {
+        echo $output . PHP_EOL;
     }
 
     // make a new checkpoint
     $snmp_stats_last = $snmp_stats;
     $db_stats_last = $db_stats;
-    $rrd_stats_last = $rrd_stats;
 }
 
 /**
@@ -2129,7 +2030,7 @@ function printChangedStats($update_only = false)
  */
 function printStats()
 {
-    global $snmp_stats, $db_stats, $rrd_stats;
+    global $snmp_stats, $db_stats;
 
     if ($snmp_stats) {
         printf(
@@ -2167,40 +2068,16 @@ function printStats()
         );
     }
 
-    if ($rrd_stats) {
+    foreach (app('Datastore')->getStats() as $datastore => $stats) {
-        printf(
+        /** @var \LibreNMS\Data\Measure\MeasurementCollection $stats */
-            "RRD [%d/%.2fs]: Update[%d/%.2fs] Create [%d/%.2fs] Other[%d/%.2fs]\n",
+        printf("%s [%d/%.2fs]:", $datastore, $stats->getTotalCount(), $stats->getTotalDuration());
-            array_sum($rrd_stats['ops']),
+
-            array_sum($rrd_stats['time']),
+        foreach ($stats as $stat) {
-            $rrd_stats['ops']['update'],
+            /** @var \LibreNMS\Data\Measure\MeasurementSummary $stat */
-            $rrd_stats['time']['update'],
+            printf(" %s[%d/%.2fs]", ucfirst($stat->getType()), $stat->getCount(), $stat->getDuration());
-            $rrd_stats['ops']['create'],
+        }
-            $rrd_stats['time']['create'],
+        echo PHP_EOL;
-            $rrd_stats['ops']['other'],
-            $rrd_stats['time']['other']
-        );
     }
-}
-
-/**
- * Update statistics for rrd operations
- *
- * @param string $stat create, update, and other
- * @param float $start_time The time the operation started with 'microtime(true)'
- * @return float  The calculated run time
- */
-function recordRrdStatistic($stat, $start_time)
-{
-    global $rrd_stats;
-    initStats();
-
-    $stat = ($stat == 'update' || $stat == 'create') ? $stat : 'other';
-
-    $runtime = microtime(true) - $start_time;
-    $rrd_stats['ops'][$stat]++;
-    $rrd_stats['time'][$stat] += $runtime;
-
-    return $runtime;
 }
 
 /**

includes/graphite.inc.php

@@ -1,50 +0,0 @@
-<?php
-
-/*
- * LibreNMS
- *
- * Copyright (c) 2017 Falk Stern <https://github.com/fstern/ >
- *
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation, either version 3 of the License, or (at your
- * option) any later version.  Please see LICENSE.txt at the top level of
- * the source code distribution for details.
- */
-
-function graphite_update($device, $measurement, $tags, $fields)
-{
-    global $graphite;
-    if ($graphite != false) {
-        $timestamp = time();
-        $graphite_prefix = \LibreNMS\Config::get('graphite.prefix');
-        // metrics will be built as prefix.hostname.measurement.field value timestamp
-        // metric fields can not contain . as this is used by graphite as a field separator
-        $hostname = preg_replace('/\./', '_', $device['hostname']);
-        $measurement = preg_replace(array('/\./', '/\//'), '_', $measurement);
-        $measurement = preg_replace('/\|/', '.', $measurement);
-        $measurement_name = preg_replace('/\./', '_', $tags['rrd_name']);
-        if (is_array($measurement_name)) {
-            $ms_name = implode(".", $measurement_name);
-        } else {
-            $ms_name = $measurement_name;
-        }
-        // remove the port-id tags from the metric
-        if (preg_match('/^port-id\d+/', $ms_name)) {
-            $ms_name = "";
-        }
-
-        foreach ($fields as $k => $v) {
-            // Send zero for fields without values
-            if (empty($v)) {
-                $v = 0;
-            }
-            $metric = implode(".", array_filter(array($graphite_prefix, $hostname, $measurement, $ms_name, $k)));
-            // Further sanitize the full metric before sending, whitespace isn't allowed
-            $metric = preg_replace('/\s+/', '_', $metric);
-            $line = implode(" ", array($metric, $v, $timestamp));
-            d_echo("Sending $line\n");
-            fwrite($graphite, $line . "\n");
-        }
-    }
-}

includes/html/api_functions.inc.php

@@ -21,6 +21,7 @@ use Illuminate\Routing\Router;
 use Illuminate\Support\Facades\Validator;
 use LibreNMS\Alerting\QueryBuilderParser;
 use LibreNMS\Config;
+use LibreNMS\Data\Store\Datastore;
 use LibreNMS\Exceptions\InvalidIpException;
 use LibreNMS\Util\IPv4;
 
@@ -83,9 +84,8 @@ function api_get_graph(array $vars)
 
     ob_start();
 
-    rrdtool_initialize(false);
     include 'includes/html/graphs/graph.inc.php';
-    rrdtool_close();
+    Datastore::terminate();
 
     $image = ob_get_contents();
     ob_end_clean();

includes/html/forms/update-ifspeed.inc.php

@@ -42,7 +42,7 @@ if (!empty($ifName) && is_numeric($port_id) && is_numeric($port_id)) {
                 ($device_tune == "true" && $port_tune != 'false') ||
                 (\LibreNMS\Config::get('rrdtool_tune') == "true" && $port_tune != 'false' && $device_tune != 'false')) {
                 $rrdfile = get_port_rrdfile_path($device['hostname'], $port_id);
-                rrdtool_tune('port', $rrdfile, $speed);
+                Rrd::tune('port', $rrdfile, $speed);
             }
         }
         $status = 'ok';

includes/html/graphs/application/proxmox_traffic.inc.php

@@ -18,7 +18,7 @@
 
 require 'includes/html/graphs/common.inc.php';
 
-$proxmox_rrd = proxmox_rrd_name($vars['cluster'], $vars['vmid'], $vars['port']);
+$proxmox_rrd = Rrd::proxmoxName($vars['cluster'], $vars['vmid'], $vars['port']);
 
 if (rrdtool_check_rrd_exists($proxmox_rrd)) {
     $rrd_filename = $proxmox_rrd;

includes/influxdb.inc.php

@@ -1,89 +0,0 @@
-<?php
-
-use LibreNMS\Config;
-
-/*
- * LibreNMS
- *
- * Copyright (c) 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
- *
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation, either version 3 of the License, or (at your
- * option) any later version.  Please see LICENSE.txt at the top level of
- * the source code distribution for details.
- */
-
-function influxdb_connect()
-{
-    $influxdb_cred = '';
-    if (!empty(Config::get('influxdb.username')) && !empty(Config::get('influxdb.password'))) {
-        $influxdb_cred = Config::get('influxdb.username') . ':' . Config::get('influxdb.password') . '@';
-        d_echo('Using authentication for InfluxDB');
-    }
-    $influxdb_url = $influxdb_cred . Config::get('influxdb.host') . ':' . Config::get('influxdb.port') . '/' . Config::get('influxdb.db');
-    d_echo(Config::get('influxdb.transport') . " transport being used");
-    if (Config::get('influxdb.transport') == 'http') {
-        $influxdb_conn = 'influxdb';
-    } elseif (Config::get('influxdb.transport') == 'https') {
-        $influxdb_conn = 'https+influxdb';
-    } elseif (Config::get('influxdb.transport') == 'udp') {
-        $influxdb_conn = 'udp+influxdb';
-    } else {
-        echo 'InfluxDB support enabled but no valid transport details provided';
-        return false;
-    }
-
-    $db = \InfluxDB\Client::fromDSN($influxdb_conn . '://' . $influxdb_url, Config::get('influxdb.timeout'), Config::get('influxdb.verifySSL'));
-    return($db);
-}// end influxdb_connect
-
-function influx_update($device, $measurement, $tags, $fields)
-{
-    global $influxdb;
-    if ($influxdb !== false) {
-        $tmp_fields = array();
-        $tmp_tags['hostname'] = $device['hostname'];
-        foreach ($tags as $k => $v) {
-            $v = preg_replace(array('/ /','/,/','/=/'), array('\ ','\,','\='), $v);
-            if (empty($v)) {
-                $v = '_blank_';
-            }
-            $tmp_tags[$k] = $v;
-        }
-        foreach ($fields as $k => $v) {
-            if ($k == 'time') {
-                $k = 'rtime';
-            }
-            $tmp_fields[$k] = force_influx_data($v);
-            if ($tmp_fields[$k] === null) {
-                unset($tmp_fields[$k]);
-            }
-        }
-        
-        d_echo("\nInfluxDB data:\n");
-        d_echo($measurement);
-        d_echo($tmp_tags);
-        d_echo($tmp_fields);
-        d_echo("\nEND\n");
-
-        if (Config::get('noinfluxdb') !== true) {
-            $points = array(
-                new InfluxDB\Point(
-                    $measurement,
-                    null, // the measurement value
-                    $tmp_tags,
-                    $tmp_fields // optional additional fields
-                )
-            );
-            try {
-                $result = $influxdb->writePoints($points);
-            } catch (Exception $e) {
-                d_echo("Caught exception: " . $e->getMessage() . PHP_EOL);
-                d_echo($e->getTrace());
-            }
-        } else {
-            c_echo("[%gInfluxDB Disabled%n]\n");
-        }//end if
-    }//end if
-}// end influx_update

includes/init.php

@@ -57,10 +57,6 @@ if (!function_exists('module_selected')) {
 require_once $install_dir . '/includes/common.php';
 require_once $install_dir . '/includes/dbFacile.php';
 require_once $install_dir . '/includes/rrdtool.inc.php';
-require_once $install_dir . '/includes/influxdb.inc.php';
-require_once $install_dir . '/includes/prometheus.inc.php';
-require_once $install_dir . '/includes/opentsdb.inc.php';
-require_once $install_dir . '/includes/graphite.inc.php';
 require_once $install_dir . '/includes/datastore.inc.php';
 require_once $install_dir . '/includes/billing.php';
 require_once $install_dir . '/includes/syslog.php';

includes/opentsdb.inc.php

@@ -1,62 +0,0 @@
-<?php
-/*
- * LibreNMS
- *
- * Copyright (c) 2017 Yacine Benamsili <https://github.com/yac01/librenms.git >
- *
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation, either version 3 of the License, or (at your
- * option) any later version.  Please see LICENSE.txt at the top level of
- * the source code distribution for details.
- */
-
-function opentsdb_update($device, $measurement, $tags, $fields)
-{
-
-    global $opentsdb;
-    if (\LibreNMS\Config::get('opentsdb.enable') == true) {
-        if ($opentsdb != true) {
-            $opentsdb = fsockopen(\LibreNMS\Config::get('opentsdb.host'), \LibreNMS\Config::get('opentsdb.port'));
-        }
-        if ($opentsdb == true) {
-            d_echo("Connection to OpenTSDB is done\n");
-        } else {
-            d_echo("Connection to OpenTSDB has failed\n");
-        }
-
-        $flag = \LibreNMS\Config::get('opentsdb.co');
-          $timestamp = time();
-          $tmp_tags = "hostname=".$device['hostname'];
-
-        foreach ($tags as $k => $v) {
-                 $v = str_replace(array(' ',',','='), '_', $v);
-            if (!empty($v)) {
-                $tmp_tags = $tmp_tags ." ". $k ."=".$v;
-            }
-        }
-
-        if ($measurement == 'port') {
-            foreach ($fields as $k => $v) {
-                     $measurement = $k;
-                if ($flag == true) {
-                     $measurement = $measurement.".".$device['co'];
-                }
-                     $line = sprintf('put net.port.%s %d %f %s', strtolower($measurement), $timestamp, $v, $tmp_tags);
-                     d_echo("Sending to OPenTSDB: $line\n");
-                     fwrite($opentsdb, $line . "\n"); // send $line into OpenTSDB
-            }
-        } else {
-            if ($flag == true) {
-                $measurement = $measurement.'.'.$device['co'];
-            }
-
-            foreach ($fields as $k => $v) {
-                     $tmp_tags_key = $tmp_tags ." ". "key" ."=".$k;
-                     $line = sprintf('put net.%s %d %f %s', strtolower($measurement), $timestamp, $v, $tmp_tags_key);
-                     d_echo("Sending to OPenTSDB: $line\n");
-                     fwrite($opentsdb, $line . "\n"); // send $line into OpenTSDB
-            }
-        }
-    }
-}

includes/polling/applications/mysql.inc.php

@@ -219,6 +219,8 @@ $mapping_status = array(
 
 $rrd_name = array('app', $name, $app_id, 'status');
 $rrd_def = new RrdDefinition();
+// because this sends different names for rrd and compared to other datastores, disable $fields name checks
+$rrd_def->disableNameChecking();
 
 $fields = array();
 foreach ($mapping_status as $desc => $id) {

includes/polling/applications/smart.inc.php

@@ -47,83 +47,34 @@ while (isset($lines[$int])) {
         $id196, $id197, $id198, $id199, $id231, $id233, $completed, $interrupted, $read_failure,
         $unknown_failure, $extended, $short, $conveyance, $selective)=explode(",", $lines[$int]);
 
-    if (is_int($id5)) {
-        $id=null;
-    }
-    if (is_int($id10)) {
-        $id10=null;
-    }
-    if (is_int($id173)) {
-        $id173=null;
-    }
-    if (is_int($id177)) {
-        $id177=null;
-    }
-    if (is_int($id183)) {
-        $id183=null;
-    }
-    if (is_int($id184)) {
-        $id184=null;
-    }
-    if (is_int($id187)) {
-        $id187=null;
-    }
-    if (is_int($id188)) {
-        $id188=null;
-    }
-    if (is_int($id190)) {
-        $id190=null;
-    }
-    if (is_int($id194)) {
-        $id194=null;
-    }
-    if (is_int($id196)) {
-        $id196=null;
-    }
-    if (is_int($id197)) {
-        $id197=null;
-    }
-    if (is_int($id198)) {
-        $id198=null;
-    }
-    if (is_int($id199)) {
-        $id199=null;
-    }
-    if (is_int($id231)) {
-        $id231=null;
-    }
-    if (is_int($id233)) {
-        $id233=null;
-    }
-
     $rrd_name = array('app', $name, $app_id, $disk);
 
-    $fields = array(
+    $fields = [
-        'id5'=>$id5,
+        'id5' => is_numeric($id5) ? $id5 : null,
-        'id10'=>$id10,
+        'id10' => is_numeric($id10) ? $id10 : null,
-        'id173'=>$id173,
+        'id173' => is_numeric($id173) ? $id173 : null,
-        'id177'=>$id177,
+        'id177' => is_numeric($id177) ? $id177 : null,
-        'id183'=>$id183,
+        'id183' => is_numeric($id183) ? $id183 : null,
-        'id184'=>$id184,
+        'id184' => is_numeric($id184) ? $id184 : null,
-        'id187'=>$id187,
+        'id187' => is_numeric($id187) ? $id187 : null,
-        'id188'=>$id188,
+        'id188' => is_numeric($id188) ? $id188 : null,
-        'id190'=>$id190,
+        'id190' => is_numeric($id190) ? $id190 : null,
-        'id194'=>$id194,
+        'id194' => is_numeric($id194) ? $id194 : null,
-        'id196'=>$id196,
+        'id196' => is_numeric($id196) ? $id196 : null,
-        'id197'=>$id197,
+        'id197' => is_numeric($id197) ? $id197 : null,
-        'id198'=>$id198,
+        'id198' => is_numeric($id198) ? $id198 : null,
-        'id199'=>$id199,
+        'id199' => is_numeric($id199) ? $id199 : null,
-        'id231'=>$id231,
+        'id231' => is_numeric($id231) ? $id231 : null,
-        'id233'=>$id233,
+        'id233' => is_numeric($id233) ? $id233 : null,
-        'completed'=>$completed,
+        'completed' => is_numeric($completed) ? $completed : null,
-        'interrupted'=>$interrupted,
+        'interrupted' => is_numeric($interrupted) ? $interrupted : null,
-        'readfailure'=>$read_failure,
+        'readfailure' => is_numeric($read_failure) ? $read_failure : null,
-        'unknownfail'=>$unknown_failure,
+        'unknownfail' => is_numeric($unknown_failure) ? $unknown_failure : null,
-        'extended'=>$extended,
+        'extended' => is_numeric($extended) ? $extended : null,
-        'short'=>$short,
+        'short' => is_numeric($short) ? $short : null,
-        'conveyance'=>$conveyance,
+        'conveyance' => is_numeric($conveyance) ? $conveyance : null,
-        'selective'=>$selective
+        'selective' => is_numeric($selective) ? $selective : null
-    );
+    ];
 
     $metrics[$disk] = $fields;
     $tags = array('name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name);

includes/polling/ipSystemStats.inc.php

@@ -105,7 +105,7 @@ if ($data) {
             if (strstr($stats[$oid], 'No') || strstr($stats[$oid], 'd') || strstr($stats[$oid], 's')) {
                 $stats[$oid] = '0';
             }
-            $fields[$oid] = $stats[$oid];
+            $fields[$oid_ds] = $stats[$oid];
         }
 
         $tags = compact('af', 'rrd_name', 'rrd_def');

includes/polling/netstats/netstats-icmp.inc.php

@@ -43,7 +43,7 @@ if (!starts_with($device['os'], array('Snom', 'asa'))) {
         $fields = array();
         foreach ($oids as $oid) {
             $rrd_def->addDataset($oid, 'COUNTER', null, 100000000000);
-            $fields[$oid] = isset($data[$oid]) ? $data[$oid] : 'U';
+            $fields[substr($oid, 0, 19)] = isset($data[$oid]) ? $data[$oid] : 'U';
         }
 
         $tags = compact('rrd_def');

includes/polling/netstats/netstats-snmp.inc.php

@@ -45,7 +45,7 @@ if ($device['os'] != 'Snom') {
         $fields = array();
         foreach ($oids as $oid) {
             $rrd_def->addDataset($oid, 'COUNTER', null, 100000000000);
-            $fields[$oid] = isset($data[0][$oid]) ? $data[0][$oid] : 'U';
+            $fields[substr($oid, 0, 19)] = isset($data[0][$oid]) ? $data[0][$oid] : 'U';
         }
 
         $tags = compact('rrd_def');

includes/polling/ports.inc.php

@@ -823,16 +823,7 @@ foreach ($ports as $port) {
                 'OUTMULTICASTPKTS' => $this_port['ifOutMulticastPkts'],
             );
 
-            if ($tune_port === true) {
+            // non rrd stats (will be filtered)
-                rrdtool_tune('port', $rrdfile, $this_port['ifSpeed']);
-            }
-
-            $port_descr_type = $port['port_descr_type'];
-            $ifName = $port['ifName'];
-            $ifAlias = $port['ifAlias'];
-            $tags = compact('ifName', 'ifAlias', 'port_descr_type', 'rrd_name', 'rrd_def');
-            rrdtool_data_update($device, 'ports', $tags, $fields);
-
             $fields['ifInUcastPkts_rate'] = $port['ifInUcastPkts_rate'];
             $fields['ifOutUcastPkts_rate'] = $port['ifOutUcastPkts_rate'];
             $fields['ifInErrors_rate'] = $port['ifInErrors_rate'];
@@ -844,10 +835,20 @@ foreach ($ports as $port) {
             $fields['ifInBits_rate'] = $port['stats']['ifInBits_rate'];
             $fields['ifOutBits_rate'] = $port['stats']['ifOutBits_rate'];
 
-            prometheus_push($device, 'ports', rrd_array_filter($tags), $fields);
+            if ($tune_port === true) {
-            influx_update($device, 'ports', rrd_array_filter($tags), $fields);
+                Rrd::tune('port', $rrdfile, $this_port['ifSpeed']);
-            graphite_update($device, 'ports|' . $ifName, $tags, $fields);
+            }
-            opentsdb_update($device, 'port', array('ifName' => $this_port['ifName'], 'ifIndex' => getPortRrdName($port_id)), $fields);
+
+            $tags = [
+                'ifName' => $port['ifName'],
+                'ifAlias' => $port['ifAlias'],
+                'ifIndex' => $port['ifIndex'],
+                'port_descr_type' => $port['port_descr_type'],
+                'rrd_name' => $rrd_name,
+                'rrd_def' => $rrd_def,
+            ];
+            app('Datastore')->put($device, 'ports', $tags, $fields);
+
             // End Update IF-MIB
             // Update PAgP
             if ($this_port['pagpOperationMode'] || $port['pagpOperationMode']) {

includes/polling/ports/cisco-if-extension.inc.php

@@ -49,5 +49,5 @@ if (isset($this_port['cieIfInRuntsErrs'])) {
      */
     $ifName = $port['ifName'];
     $tags = compact('ifName', 'rrd_name', 'rrd_def');
-    rrdtool_data_update($device, 'drops', $tags, $rrd_data);
+    data_update($device, 'drops', $tags, $rrd_data);
 }

includes/prometheus.inc.php

@@ -1,75 +0,0 @@
-<?php
-
-use LibreNMS\Config;
-
-/*
- * LibreNMS
- *
- * Copyright (c) 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
- *
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation, either version 3 of the License, or (at your
- * option) any later version.  Please see LICENSE.txt at the top level of
- * the source code distribution for details.
- */
-
-function prometheus_push($device, $measurement, $tags, $fields)
-{
-    global $prometheus;
-    if (Config::get('prometheus.enable') === true) {
-        if ($prometheus !== false) {
-            try {
-                $ch = curl_init();
-
-                set_curl_proxy($ch);
-                $vals = "";
-                $promtags = "/measurement/".$measurement;
-
-                foreach ($fields as $k => $v) {
-                    if ($v !== null) {
-                        $vals = $vals . "$k $v\n";
-                    }
-                }
-                
-                foreach ($tags as $t => $v) {
-                    if ($v !== null) {
-                        $promtags = $promtags . "/$t/$v";
-                    }
-                }
-
-                $promurl = Config::get('prometheus.url') . '/metrics/job/' . Config::get('prometheus.job') . '/instance/' . $device['hostname'] . $promtags;
-                $promurl = str_replace(" ", "-", $promurl); // Prometheus doesn't handle tags with spaces in url
-        
-                d_echo("\nPrometheus data:\n");
-                d_echo($measurement);
-                d_echo($tags);
-                d_echo($fields);
-                d_echo($vals);
-                d_echo($promurl);
-                d_echo("\nEND\n");
-
-                curl_setopt($ch, CURLOPT_URL, $promurl);
-                curl_setopt($ch, CURLOPT_POST, 1);
-                curl_setopt($ch, CURLOPT_POSTFIELDS, $vals);
-                curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
- 
-                $headers = array();
-                $headers[] = "Content-Type: test/plain";
-                curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
-
-        
-                curl_exec($ch);
- 
-                if (curl_errno($ch)) {
-                    d_echo('Error:' . curl_error($ch));
-                }
-            } catch (Exception $e) {
-                d_echo("Caught exception: " . $e->getMessage() . PHP_EOL);
-                d_echo($e->getTrace());
-            }
-        } else {
-            c_echo("[%gPrometheus Push Disabled%n]\n");
-        }//end if
-    }//end if
-}// end prometheus_push

includes/rrdtool.inc.php

@@ -24,71 +24,6 @@
  * @author     Tony Murray <murraytony@gmail.com>
  */
 
-use LibreNMS\Config;
-use LibreNMS\Exceptions\FileExistsException;
-use LibreNMS\Proc;
-
-/**
- * Opens up a pipe to RRDTool using handles provided
- *
- * @param bool $dual_process start an additional process that's output should be read after every command
- * @return bool the process(s) have been successfully started
- */
-function rrdtool_initialize($dual_process = true)
-{
-    global $rrd_sync_process, $rrd_async_process;
-
-    $command = Config::get('rrdtool') . ' -';
-
-    $descriptor_spec = array(
-        0 => array('pipe', 'r'), // stdin  is a pipe that the child will read from
-        1 => array('pipe', 'w'), // stdout is a pipe that the child will write to
-        2 => array('pipe', 'w'), // stderr is a pipe that the child will write to
-    );
-
-    $cwd = Config::get('rrd_dir');
-
-    if (!rrdtool_running($rrd_sync_process)) {
-        $rrd_sync_process = new Proc($command, $descriptor_spec, $cwd);
-    }
-
-    if ($dual_process && !rrdtool_running($rrd_async_process)) {
-        $rrd_async_process = new Proc($command, $descriptor_spec, $cwd);
-        $rrd_async_process->setSynchronous(false);
-    }
-
-    return rrdtool_running($rrd_sync_process) && ($dual_process ? rrdtool_running($rrd_async_process) : true);
-}
-
-/**
- * Checks if the variable is a running rrdtool process
- *
- * @param $process
- * @return bool
- */
-function rrdtool_running(&$process)
-{
-    return isset($process) && $process instanceof Proc && $process->isRunning();
-}
-
-/**
- * Close all open rrdtool processes.
- * This should be done before exiting a script that has called rrdtool_initilize()
- */
-function rrdtool_close()
-{
-    global $rrd_sync_process, $rrd_async_process;
-    /** @var Proc $rrd_sync_process */
-    /** @var Proc $rrd_async_process */
-
-    if (rrdtool_running($rrd_sync_process)) {
-        $rrd_sync_process->close('quit');
-    }
-    if (rrdtool_running($rrd_async_process)) {
-        $rrd_async_process->close('quit');
-    }
-}
-
 /**
  * Generates a graph file at $graph_file using $options
  * Opens its own rrdtool pipe.
@@ -99,123 +34,10 @@ function rrdtool_close()
  */
 function rrdtool_graph($graph_file, $options)
 {
-    global $debug, $rrd_sync_process;
+    return Rrd::graph($graph_file, $options);
-    /** @var Proc $rrd_sync_process */
-
-    if (rrdtool_initialize(false)) {
-        $cmd = rrdtool_build_command('graph', $graph_file, $options);
-
-        $output = implode($rrd_sync_process->sendCommand($cmd));
-
-        if ($debug) {
-            echo "<p>$cmd</p>";
-            echo "<p>command returned ($output)</p>";
-        }
-
-        return $output;
-    } else {
-        return 0;
-    }
 }
 
 
-/**
- * Generates and pipes a command to rrdtool
- *
- * @internal
- * @param string $command create, update, updatev, graph, graphv, dump, restore, fetch, tune, first, last, lastupdate, info, resize, xport, flushcached
- * @param string $filename The full patth to the rrd file
- * @param string $options rrdtool command options
- * @return array the output of stdout and stderr in an array
- * @throws FileExistsException thrown when a create command is set to rrdtool < 1.4 and the rrd already exists
- * @throws Exception thrown when the rrdtool process(s) cannot be started
- */
-function rrdtool($command, $filename, $options)
-{
-    global $debug, $vdebug, $rrd_async_process, $rrd_sync_process;
-    /** @var Proc $rrd_sync_process */
-    /** @var Proc $rrd_async_process */
-
-    $start_time = microtime(true);
-
-    try {
-        $cmd = rrdtool_build_command($command, $filename, $options);
-    } catch (FileExistsException $e) {
-        c_echo('RRD[%g' . $filename . " already exists%n]\n", $debug);
-        return array(null, null);
-    }
-
-    c_echo("RRD[%g$cmd%n]\n", $debug);
-
-    // do not write rrd files, but allow read-only commands
-    $ro_commands = array('graph', 'graphv', 'dump', 'fetch', 'first', 'last', 'lastupdate', 'info', 'xport');
-    if (!empty(Config::get('norrd')) && !in_array($command, $ro_commands)) {
-        c_echo('[%rRRD Disabled%n]', !Config::get('hide_rrd_disabled'));
-        return array(null, null);
-    }
-
-    // send the command!
-    if ($command == 'last' && rrdtool_initialize(false)) {
-        // send this to our synchronous process so output is guaranteed
-        $output = $rrd_sync_process->sendCommand($cmd);
-    } elseif (rrdtool_initialize()) {
-        // don't care about the return of other commands, so send them to the faster async process
-        $output = $rrd_async_process->sendCommand($cmd);
-    } else {
-        throw new Exception('rrdtool could not start');
-    }
-
-    if ($vdebug) {
-        echo 'RRDtool Output: ';
-        echo $output[0];
-        echo $output[1];
-    }
-
-    recordRrdStatistic($command, $start_time);
-    return $output;
-}
-
-/**
- * Build a command for rrdtool
- * Shortens the filename as needed
- * Determines if --daemon and -O should be used
- *
- * @internal
- * @param string $command The base rrdtool command.  Usually create, update, last.
- * @param string $filename The full path to the rrd file
- * @param string $options Options for the command possibly including the rrd definition
- * @return string returns a full command ready to be piped to rrdtool
- * @throws FileExistsException if rrdtool <1.4.3 and the rrd file exists locally
- */
-function rrdtool_build_command($command, $filename, $options)
-{
-    if ($command == 'create') {
-        // <1.4.3 doesn't support -O, so make sure the file doesn't exist
-        if (version_compare(Config::get('rrdtool_version', '1.4'), '1.4.3', '<')) {
-            if (is_file($filename)) {
-                throw new FileExistsException();
-            }
-        } else {
-            $options .= ' -O';
-        }
-    }
-
-    // no remote for create < 1.5.5 and tune < 1.5
-    $rrdtool_version = Config::get('rrdtool_version', '1.4');
-    if (Config::get('rrdcached') &&
-        !($command == 'create' && version_compare($rrdtool_version, '1.5.5', '<')) &&
-        !($command == 'tune' && Config::get('rrdcached') && version_compare($rrdtool_version, '1.5', '<'))
-    ) {
-        // only relative paths if using rrdcached
-        $filename = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $filename);
-        $options = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $options);
-
-        return "$command $filename $options --daemon " . Config::get('rrdcached');
-    }
-
-    return "$command $filename $options";
-}
-
 /**
  * Checks if the rrd file exists on the server
  * This will perform a remote check if using rrdcached and rrdtool >= 1.5
@@ -225,46 +47,9 @@ function rrdtool_build_command($command, $filename, $options)
  */
 function rrdtool_check_rrd_exists($filename)
 {
-    if (Config::get('rrdcached') && version_compare(Config::get('rrdtool_version', '1.4'), '1.5', '>=')) {
+    return Rrd::checkRrdExists($filename);
-        $chk = rrdtool('last', $filename, '');
-        $filename = str_replace([Config::get('rrd_dir') . '/', Config::get('rrd_dir')], '', $filename);
-        return !str_contains(implode($chk), "$filename': No such file or directory");
-    } else {
-        return is_file($filename);
-    }
 }
 
-/**
- * Updates an rrd database at $filename using $options
- * Where $options is an array, each entry which is not a number is replaced with "U"
- *
- * @internal
- * @param string $filename
- * @param array $data
- * @return array|string
- */
-function rrdtool_update($filename, $data)
-{
-    $values = array();
-    // Do some sanitation on the data if passed as an array.
-
-    if (is_array($data)) {
-        $values[] = 'N';
-        foreach ($data as $v) {
-            if (!is_numeric($v)) {
-                $v = 'U';
-            }
-
-            $values[] = $v;
-        }
-
-        $data = implode(':', $values);
-        return rrdtool('update', $filename, $data);
-    } else {
-        return 'Bad options passed to rrdtool_update';
-    }
-} // rrdtool_update
-
 
 /**
  * Escapes strings for RRDtool
@@ -312,11 +97,9 @@ function rrdtool_escape($string, $length = null)
  */
 function rrd_name($host, $extra, $extension = ".rrd")
 {
-    $filename = safename(is_array($extra) ? implode("-", $extra) : $extra);
+    return Rrd::name($host, $extra, $extension);
-    return implode("/", array(get_rrd_dir($host), $filename.$extension));
 } // rrd_name
 
-
 /**
  * Generates a path based on the hostname (or IP)
  *
@@ -329,107 +112,6 @@ function get_rrd_dir($host)
     return implode("/", [Config::get('rrd_dir'), $host]);
 } // rrd_dir
 
-
-/**
- * Generates a filename for a proxmox cluster rrd
- *
- * @param $pmxcluster
- * @param $vmid
- * @param $vmport
- * @return string full path to the rrd.
- */
-function proxmox_rrd_name($pmxcluster, $vmid, $vmport)
-{
-    $pmxcdir = join('/', [Config::get('rrd_dir'), 'proxmox', safename($pmxcluster)]);
-    // this is not needed for remote rrdcached
-    if (!is_dir($pmxcdir)) {
-        mkdir($pmxcdir, 0775, true);
-    }
-
-    return join('/', array($pmxcdir, safename($vmid.'_netif_'.$vmport.'.rrd')));
-}
-
-/**
- * Modify an rrd file's max value and trim the peaks as defined by rrdtool
- *
- * @param string $type only 'port' is supported at this time
- * @param string $filename the path to the rrd file
- * @param integer $max the new max value
- * @return bool
- */
-function rrdtool_tune($type, $filename, $max)
-{
-    $fields = array();
-    if ($type === 'port') {
-        if ($max < 10000000) {
-            return false;
-        }
-        $max = $max / 8;
-        $fields = array(
-            'INOCTETS',
-            'OUTOCTETS',
-            'INERRORS',
-            'OUTERRORS',
-            'INUCASTPKTS',
-            'OUTUCASTPKTS',
-            'INNUCASTPKTS',
-            'OUTNUCASTPKTS',
-            'INDISCARDS',
-            'OUTDISCARDS',
-            'INUNKNOWNPROTOS',
-            'INBROADCASTPKTS',
-            'OUTBROADCASTPKTS',
-            'INMULTICASTPKTS',
-            'OUTMULTICASTPKTS'
-        );
-    }
-    if (count($fields) > 0) {
-        $options = "--maximum " . implode(":$max --maximum ", $fields) . ":$max";
-        rrdtool('tune', $filename, $options);
-    }
-    return true;
-} // rrdtool_tune
-
-
-/**
- * rrdtool backend implementation of data_update
- *
- * Tags:
- *   rrd_def     RrdDefinition
- *   rrd_name    array|string: the rrd filename, will be processed with rrd_name()
- *   rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
- *   rrd_step             int: rrd step, defaults to 300
- *
- * @param array $device device array
- * @param string $measurement the name of this measurement (if no rrd_name tag is given, this will be used to name the file)
- * @param array $tags tags to pass additional info to rrdtool
- * @param array $fields data values to update
- */
-function rrdtool_data_update($device, $measurement, $tags, $fields)
-{
-    $rrd_name = $tags['rrd_name'] ?: $measurement;
-    $step = $tags['rrd_step'] ?: Config::get('rrd.step');
-    $oldname = $tags['rrd_oldname'];
-    if (!empty($oldname)) {
-        rrd_file_rename($device, $oldname, $rrd_name);
-    }
-
-    if (isset($tags['rrd_proxmox_name'])) {
-        $pmxvars = $tags['rrd_proxmox_name'];
-        $rrd = proxmox_rrd_name($pmxvars['pmxcluster'], $pmxvars['vmid'], $pmxvars['vmport']);
-    } else {
-        $rrd = rrd_name($device['hostname'], $rrd_name);
-    }
-
-    if (isset($tags['rrd_def']) && !rrdtool_check_rrd_exists($rrd)) {
-        $newdef = "--step $step " . $tags['rrd_def'] . Config::get('rrd_rra');
-        rrdtool('create', $rrd, $newdef);
-    }
-
-    rrdtool_update($rrd, $fields);
-} // rrdtool_data_update
-
-
 /**
  * rename an rrdfile, can only be done on the LibreNMS server hosting the rrd files
  *
@@ -440,18 +122,5 @@ function rrdtool_data_update($device, $measurement, $tags, $fields)
  */
 function rrd_file_rename($device, $oldname, $newname)
 {
-    $oldrrd = rrd_name($device['hostname'], $oldname);
+    return Rrd::renameFile($device, $oldname, $newname);
-    $newrrd = rrd_name($device['hostname'], $newname);
-    if (is_file($oldrrd) && !is_file($newrrd)) {
-        if (rename($oldrrd, $newrrd)) {
-            log_event("Renamed $oldrrd to $newrrd", $device, "poller", 1);
-            return true;
-        } else {
-            log_event("Failed to rename $oldrrd to $newrrd", $device, "poller", 5);
-            return false;
-        }
-    } else {
-        // we don't need to rename the file
-        return true;
-    }
 } // rrd_file_rename

ping.php

@@ -37,14 +37,4 @@ if (isset($options['g'])) {
     $groups = [];
 }
 
-if (Config::get('base_url') !== true && \LibreNMS\Config::get('influxdb.enable') === true) {
-    $influxdb = influxdb_connect();
-} else {
-    $influxdb = false;
-}
-
-rrdtool_initialize();
-
 PingCheck::dispatch($groups);
-
-rrdtool_close();

poll-billing.php

@@ -11,6 +11,8 @@
  * @copyright  (C) 2006 - 2012 Adam Armstrong
  */
 
+use LibreNMS\Data\Store\Datastore;
+
 $init_modules = array();
 require __DIR__ . '/includes/init.php';
 
@@ -22,6 +24,7 @@ if (isset($argv[1]) && is_numeric($argv[1])) {
 }
 
 set_debug(isset($options['d']));
+Datastore::init();
 
 // Wait for schema update, as running during update can break update
 if (get_db_schema() < 107) {
@@ -29,8 +32,6 @@ if (get_db_schema() < 107) {
     exit(1);
 }
 
-rrdtool_initialize();
-
 $poller_start = microtime(true);
 echo "Starting Polling Session ... \n\n";
 
@@ -164,4 +165,4 @@ if ($poller_time > 300) {
 }
 echo "\nCompleted in $poller_time sec\n";
 
-rrdtool_close();
+Datastore::terminate();

poller.php

@@ -28,6 +28,7 @@
 
 use LibreNMS\Config;
 use LibreNMS\Alert\AlertRules;
+use LibreNMS\Data\Store\Datastore;
 
 $init_modules = ['polling', 'alerts', 'laravel'];
 require __DIR__ . '/includes/init.php';
@@ -117,44 +118,11 @@ EOH;
     update_os_cache(true); // Force update of OS Cache
 }
 
-if (isset($options['r'])) {
-    Config::set('norrd', true);
-}
-
-if (isset($options['f'])) {
-    Config::set('noinfluxdb', true);
-}
-
-if (isset($options['p'])) {
-    $prometheus = false;
-}
-
-if (isset($options['g'])) {
-    Config::set('nographite', true);
-}
-
-if (Config::get('base_url') !== true && Config::get('influxdb.enable') === true) {
-    $influxdb = influxdb_connect();
-} else {
-    $influxdb = false;
-}
-
-if (Config::get('base_url') !== true && Config::get('graphite.enable') === true) {
-    $graphite = fsockopen(Config::get('graphite.host'), Config::get('graphite.port'));
-    if ($graphite !== false) {
-        echo "Connection made to " . Config::get('graphite.host') . " for Graphite support\n";
-    } else {
-        echo "Connection to " . Config::get('graphite.host') . " has failed, Graphite support disabled\n";
-        Config::set('nographite', true);
-    }
-} else {
-    $graphite = false;
-}
-
 // If we've specified modules with -m, use them
 $module_override = parse_modules('poller', $options);
+set_debug($debug);
 
-rrdtool_initialize();
+$datastore = Datastore::init($options);
 
 echo "Starting polling run:\n\n";
 $polled_devices = 0;
@@ -186,10 +154,6 @@ $poller_end  = microtime(true);
 $poller_run  = ($poller_end - $poller_start);
 $poller_time = substr($poller_run, 0, 5);
 
-if ($graphite !== false) {
-    fclose($graphite);
-}
-
 if ($polled_devices) {
     dbInsert(array(
         'type' => 'poll',
@@ -209,8 +173,7 @@ if (!isset($options['q'])) {
 }
 
 logfile($string);
-rrdtool_close();
+Datastore::terminate();
-
 // Remove this for testing
 // print_r(get_defined_vars());
 

scripts/tune_port.php

@@ -4,8 +4,6 @@
 $init_modules = array();
 require realpath(__DIR__ . '/..') . '/includes/init.php';
 
-rrdtool_initialize();
-
 $options = getopt('h:p:');
 
 $hosts = str_replace('*', '%', mres($options['h']));
@@ -30,8 +28,8 @@ foreach (dbFetchRows("SELECT `device_id`,`hostname` FROM `devices` WHERE `hostna
     foreach (dbFetchRows("SELECT `port_id`,`ifIndex`,`ifName`,`ifSpeed` FROM `ports` WHERE `ifName` LIKE ? AND `device_id` = ?", array('%'.$ports.'%',$device['device_id'])) as $port) {
         echo "Tuning port " . $port['ifName'].".......\n";
         $rrdfile = get_port_rrdfile_path($device['hostname'], $port['port_id']);
-        rrdtool_tune('port', $rrdfile, $port['ifSpeed']);
+        Rrd::tune('port', $rrdfile, $port['ifSpeed']);
     }
 }
 
-rrdtool_close();
+Rrd::close();

tests/ConfigTest.php

@@ -200,4 +200,22 @@ class ConfigTest extends TestCase
         $function($config);
         $this->config->setValue($config);
     }
+
+    public function testForget()
+    {
+        Config::set('forget.me', 'now');
+        $this->assertTrue(Config::has('forget.me'));
+
+        Config::forget('forget.me');
+        $this->assertFalse(Config::has('forget.me'));
+    }
+
+    public function testForgetSubtree()
+    {
+        Config::set('forget.me.sub', 'yep');
+        $this->assertTrue(Config::has('forget.me.sub'));
+
+        Config::forget('forget.me');
+        $this->assertFalse(Config::has('forget.me.sub'));
+    }
 }

tests/RrdtoolTest.php

@@ -26,6 +26,7 @@
 namespace LibreNMS\Tests;
 
 use LibreNMS\Config;
+use LibreNMS\Data\Store\Rrd;
 
 class RrdtoolTest extends TestCase
 {
@@ -36,25 +37,26 @@ class RrdtoolTest extends TestCase
         Config::set('rrdtool_version', '1.4');
         Config::set('rrd_dir', '/opt/librenms/rrd');
 
-        $cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('create /opt/librenms/rrd/f o', $cmd);
 
-        $cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('tune /opt/librenms/rrd/f o', $cmd);
 
-        $cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('update /opt/librenms/rrd/f o', $cmd);
 
 
+        $this->app->forgetInstance(Rrd::class);
         Config::set('rrdtool_version', '1.6');
 
-        $cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('create /opt/librenms/rrd/f o -O', $cmd);
 
-        $cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('tune /opt/librenms/rrd/f o', $cmd);
 
-        $cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('update /opt/librenms/rrd/f o', $cmd);
     }
 
@@ -64,25 +66,25 @@ class RrdtoolTest extends TestCase
         Config::set('rrdtool_version', '1.4');
         Config::set('rrd_dir', '/opt/librenms/rrd');
 
-        $cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('create /opt/librenms/rrd/f o', $cmd);
 
-        $cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('tune /opt/librenms/rrd/f o', $cmd);
 
-        $cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('update f o --daemon server:42217', $cmd);
 
-
+        $this->app->forgetInstance(Rrd::class);
         Config::set('rrdtool_version', '1.6');
 
-        $cmd = rrdtool_build_command('create', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('create', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('create f o -O --daemon server:42217', $cmd);
 
-        $cmd = rrdtool_build_command('tune', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('tune', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('tune f o --daemon server:42217', $cmd);
 
-        $cmd = rrdtool_build_command('update', '/opt/librenms/rrd/f', 'o');
+        $cmd = $this->buildCommandProxy('update', '/opt/librenms/rrd/f', 'o');
         $this->assertEquals('update f o --daemon server:42217', $cmd);
     }
 
@@ -93,6 +95,16 @@ class RrdtoolTest extends TestCase
 
         $this->expectException('LibreNMS\Exceptions\FileExistsException');
         // use this file, since it is guaranteed to exist
-        rrdtool_build_command('create', __FILE__, 'o');
+        $this->buildCommandProxy('create', __FILE__, 'o');
+    }
+
+    private function buildCommandProxy($command, $filename, $options)
+    {
+        // todo better tests
+        $reflection = new \ReflectionClass(Rrd::class);
+        $method = $reflection->getMethod('buildCommand');
+        $method->setAccessible(true);
+
+        return $method->invokeArgs($this->app->make(Rrd::class), func_get_args());
     }
 }

tests/Unit/Data/DatastoreTest.php

@@ -0,0 +1,81 @@
+<?php
+/**
+ * DatastoreTest.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Tests\Unit\Data;
+
+use LibreNMS\Config;
+use LibreNMS\Tests\TestCase;
+
+/**
+ * @group datastores
+ */
+class DatastoreTest extends TestCase
+{
+    public function setUp() : void
+    {
+        parent::setUp();
+
+        Config::forget([
+            'graphite',
+            'influxdb',
+            'opentsdb',
+            'prometheus',
+            'rrd',
+        ]);
+    }
+
+    public function testDefaultInitialization()
+    {
+        $ds = $this->app->make('Datastore');
+        $stores = $ds->getStores();
+        $this->assertCount(1, $stores, 'Incorrect number of default stores enabled');
+
+        $this->assertEquals('LibreNMS\Data\Store\Rrd', get_class($stores[0]), 'The default enabled store should be Rrd');
+    }
+
+    public function testInitialization()
+    {
+        Config::set('rrd.enable', false);
+        Config::set('graphite.enable', true);
+        Config::set('influxdb.enable', true);
+        Config::set('opentsdb.enable', true);
+        Config::set('prometheus.enable', true);
+
+        $ds = $this->app->make('Datastore');
+        $stores = $ds->getStores();
+        $this->assertCount(4, $stores, 'Incorrect number of default stores enabled');
+
+        $enabled = array_map('get_class', $stores);
+
+        $expected_enabled = [
+            'LibreNMS\Data\Store\Graphite',
+            'LibreNMS\Data\Store\InfluxDB',
+            'LibreNMS\Data\Store\OpenTSDB',
+            'LibreNMS\Data\Store\Prometheus',
+        ];
+
+        $this->assertEquals($expected_enabled, $enabled, 'Expected all non-default stores to be initialized');
+    }
+}

tests/Unit/Data/GraphiteStoreTest.php

@@ -0,0 +1,108 @@
+<?php
+/**
+ * GraphiteStoreTest.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Tests\Unit\Data;
+
+use Carbon\Carbon;
+use LibreNMS\Data\Store\Graphite;
+use LibreNMS\Tests\TestCase;
+
+/**
+ * @group datastores
+ */
+class GraphiteStoreTest extends TestCase
+{
+    protected $timestamp = 997464400;
+
+    public function setUp() : void
+    {
+        parent::setUp();
+
+        // fix the date
+        Carbon::setTestNow(Carbon::createFromTimestamp($this->timestamp));
+    }
+
+    public function tearDown() : void
+    {
+        // restore Carbon:now() to normal
+        Carbon::setTestNow();
+
+        parent::tearDown();
+    }
+
+    public function testSocketConnectError()
+    {
+        $mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
+
+        $mockFactory->shouldReceive('createClient')
+            ->andThrow('Socket\Raw\Exception', 'Failed to handle connect exception');
+
+        new Graphite($mockFactory);
+    }
+
+    public function testSocketWriteError()
+    {
+        $mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
+        $graphite = $this->mockGraphite($mockSocket);
+
+        $mockSocket->shouldReceive('write')
+            ->andThrow('Socket\Raw\Exception', 'Did not handle socket exception');
+
+        $graphite->put(['hostname' => 'test'], 'fake', ['rrd_name' => 'name'], ['one' => 1]);
+    }
+
+    public function testSimpleWrite()
+    {
+        $mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
+        $graphite = $this->mockGraphite($mockSocket);
+
+        $device = ['hostname' => 'testhost'];
+        $measurement = 'testmeasure';
+        $tags = ['rrd_name' => 'rrd_name', 'ifName' => 'testifname', 'type' => 'testtype'];
+        $fields = ['ifIn' => 234234, 'ifOut' => 53453];
+
+        $mockSocket->shouldReceive('write')
+            ->with("testhost.testmeasure.rrd_name.ifIn 234234 $this->timestamp\n");
+        $mockSocket->shouldReceive('write')
+            ->with("testhost.testmeasure.rrd_name.ifOut 53453 $this->timestamp\n");
+        $graphite->put($device, $measurement, $tags, $fields);
+    }
+
+
+    /**
+     * @param $mockSocket
+     * @return Graphite
+     */
+    private function mockGraphite($mockSocket)
+    {
+        $mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
+
+        $mockFactory->shouldReceive('createClient')
+            ->andReturn($mockSocket);
+
+        $graphite = new Graphite($mockFactory);
+        return $graphite;
+    }
+}

tests/Unit/Data/InfluxDBStoreTest.php

@@ -0,0 +1,66 @@
+<?php
+/**
+ * InfluxStoreTest.php *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Tests\Unit\Data;
+
+use InfluxDB\Point;
+use LibreNMS\Config;
+use LibreNMS\Data\Store\InfluxDB;
+use LibreNMS\Tests\TestCase;
+
+/**
+ * @group datastores
+ */
+class InfluxDBStoreTest extends TestCase
+{
+    public function testBadSettings()
+    {
+        Config::set('influxdb.host', '');
+        Config::set('influxdb.port', 'abc');
+        $influx = new InfluxDB(InfluxDB::createFromConfig());
+
+        \Log::shouldReceive('debug');
+        \Log::shouldReceive('error')->once()->with('InfluxDB exception: Unable to parse URI: http://:0'); // the important one
+        $influx->put(['hostname' => 'test'], 'fake', [], ['one' => 1]);
+    }
+
+    public function testSimpleWrite()
+    {
+        // Create a mock of the Random Interface
+        $mock = \Mockery::mock(\InfluxDB\Database::class);
+
+        $mock->shouldReceive('exists')->once()->andReturn(true);
+        $influx = new InfluxDB($mock);
+
+        $device = ['hostname' => 'testhost'];
+        $measurement = 'testmeasure';
+        $tags = ['ifName' => 'testifname', 'type' => 'testtype'];
+        $fields = ['ifIn' => 234234, 'ifOut' => 53453];
+
+        $expected = [new Point($measurement, null, ['hostname' => $device['hostname']] + $tags, $fields)];
+
+        $mock->shouldReceive('writePoints')->withArgs([$expected])->once();
+        $influx->put($device, $measurement, $tags, $fields);
+    }
+}

tests/Unit/Data/OpenTSDBStoreTest.php

@@ -0,0 +1,124 @@
+<?php
+/**
+ * OpenTSDBStoreTest.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Tests\Unit\Data;
+
+use Carbon\Carbon;
+use GuzzleHttp\Client;
+use LibreNMS\Data\Store\OpenTSDB;
+use LibreNMS\Tests\TestCase;
+
+/**
+ * @group datastores
+ */
+class OpenTSDBStoreTest extends TestCase
+{
+    protected $timestamp = 990464400;
+
+    public function setUp() : void
+    {
+        parent::setUp();
+
+        // fix the date
+        Carbon::setTestNow(Carbon::createFromTimestamp($this->timestamp));
+    }
+
+    public function tearDown() : void
+    {
+        // restore Carbon:now() to normal
+        Carbon::setTestNow();
+
+        parent::tearDown();
+    }
+
+    public function testSocketConnectError()
+    {
+        $mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
+
+        $mockFactory->shouldReceive('createClient')
+            ->andThrow('Socket\Raw\Exception', 'Failed to handle connect exception');
+
+        new OpenTSDB($mockFactory);
+    }
+
+    public function testSocketWriteError()
+    {
+        $mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
+        $opentsdb = $this->mockOpenTSDB($mockSocket);
+
+        $mockSocket->shouldReceive('write')
+            ->andThrow('Socket\Raw\Exception', 'Did not handle socket exception');
+
+        $opentsdb->put(['hostname' => 'test'], 'fake', [], ['one' => 1]);
+    }
+
+    public function testSimpleWrite()
+    {
+        $mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
+        $opentsdb = $this->mockOpenTSDB($mockSocket);
+
+        $device = ['hostname' => 'testhost'];
+        $measurement = 'testmeasure';
+        $tags = ['ifName' => 'testifname', 'type' => 'testtype'];
+        $fields = ['ifIn' => 234234, 'ifOut' => 53453];
+
+        $mockSocket->shouldReceive('write')
+            ->with("put net.testmeasure $this->timestamp 234234.000000 hostname=testhost ifName=testifname type=testtype key=ifIn\n");
+        $mockSocket->shouldReceive('write')
+            ->with("put net.testmeasure $this->timestamp 53453.000000 hostname=testhost ifName=testifname type=testtype key=ifOut\n");
+        $opentsdb->put($device, $measurement, $tags, $fields);
+    }
+
+    public function testPortWrite()
+    {
+        $mockSocket = \Mockery::mock(\Socket\Raw\Socket::class);
+        $opentsdb = $this->mockOpenTSDB($mockSocket);
+
+        $device = ['hostname' => 'testhost'];
+        $measurement = 'ports';
+        $tags = ['ifName' => 'testifname', 'type' => 'testtype'];
+        $fields = ['ifIn' => 897238, 'ifOut' => 2342];
+
+        $mockSocket->shouldReceive('write')
+            ->with("put net.port.ifin $this->timestamp 897238.000000 hostname=testhost ifName=testifname type=testtype\n");
+        $mockSocket->shouldReceive('write')
+            ->with("put net.port.ifout $this->timestamp 2342.000000 hostname=testhost ifName=testifname type=testtype\n");
+        $opentsdb->put($device, $measurement, $tags, $fields);
+    }
+
+    /**
+     * @param $mockSocket
+     * @return OpenTSDB
+     */
+    private function mockOpenTSDB($mockSocket)
+    {
+        $mockFactory = \Mockery::mock(\Socket\Raw\Factory::class);
+
+        $mockFactory->shouldReceive('createClient')
+            ->andReturn($mockSocket);
+
+        return new OpenTSDB($mockFactory);
+    }
+}

tests/Unit/Data/PrometheusStoreTest.php

@@ -0,0 +1,105 @@
+<?php
+/**
+ * PrometheusStoreTest.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2018 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Tests\Unit\Data;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Exception\RequestException;
+use GuzzleHttp\Handler\MockHandler;
+use GuzzleHttp\HandlerStack;
+use GuzzleHttp\Middleware;
+use GuzzleHttp\Psr7\Request;
+use GuzzleHttp\Psr7\Response;
+use LibreNMS\Config;
+use LibreNMS\Data\Store\Prometheus;
+use LibreNMS\Tests\TestCase;
+
+/**
+ * @group datastores
+ */
+class PrometheusStoreTest extends TestCase
+{
+    public function setUp() : void
+    {
+        parent::setUp();
+
+        Config::set('prometheus.enable', true);
+        Config::set('prometheus.url', 'http://fake:9999');
+    }
+
+    public function testFailWrite()
+    {
+        $stack = HandlerStack::create(new MockHandler([
+            new Response(422, [], 'Bad response'),
+            new RequestException("Exception thrown", new Request('POST', 'test'))
+        ]));
+
+        $client = new Client(['handler' => $stack]);
+        $prometheus = new Prometheus($client);
+
+        \Log::shouldReceive('debug');
+        \Log::shouldReceive('error')->once()->with("Prometheus Exception: Client error: `POST http://fake:9999/metrics/job/librenms/instance/test/measurement/none` resulted in a `422 Unprocessable Entity` response:\nBad response\n");
+        \Log::shouldReceive('error')->once()->with('Prometheus Exception: Exception thrown');
+        $prometheus->put(['hostname' => 'test'], 'none', [], ['one' => 1]);
+        $prometheus->put(['hostname' => 'test'], 'none', [], ['one' => 1]);
+    }
+
+    public function testSimpleWrite()
+    {
+        $stack = HandlerStack::create(new MockHandler([
+            new Response(200),
+        ]));
+
+        $container = [];
+        $history = Middleware::history($container);
+
+        $stack->push($history);
+        $client = new Client(['handler' => $stack]);
+        $prometheus = new Prometheus($client);
+
+
+        $device = ['hostname' => 'testhost'];
+        $measurement = 'testmeasure';
+        $tags = ['ifName' => 'testifname', 'type' => 'testtype'];
+        $fields = ['ifIn' => 234234, 'ifOut' => 53453];
+
+        \Log::shouldReceive('debug');
+        \Log::shouldReceive('error')->times(0);
+
+        $prometheus->put($device, $measurement, $tags, $fields);
+
+        $this->assertCount(1, $container, 'Did not receive the expected number of requests');
+
+        /** @var Request $request */
+        $request = $container[0]['request'];
+
+
+        $this->assertEquals('POST', $request->getMethod());
+        $this->assertEquals("/metrics/job/librenms/instance/testhost/measurement/testmeasure/ifName/testifname/type/testtype", $request->getUri()->getPath());
+        $this->assertEquals('fake', $request->getUri()->getHost());
+        $this->assertEquals(9999, $request->getUri()->getPort());
+        $this->assertEquals("ifIn 234234\nifOut 53453\n", (string)$request->getBody());
+    }
+}