feature: Added elasticsearch transport and docs (#6687)

* Add elasticsearch transport and docs

* Fix formatting

* Fix formatting

* Remove librenms string from sql file

* Add missing states

* Use ifAlias not ifDescr

* Update Alerting.md

* Remove unused es_enabled flag

* Fix storage column names

* Fix processor columns

* Rename 194.sql to 193.sql

doc/Extensions/Alerting.md

@@ -34,6 +34,7 @@ Table of Content:
     - [Cisco Spark](#transports-ciscospark)
     - [SMSEagle](#transports-smseagle)
     - [Syslog](#transports-syslog)
+    - [Elasticsearch](#transports-elasticsearch)
 - [Entities](#entities)
     - [Devices](#entity-devices)
     - [BGP Peers](#entity-bgppeers)
@@ -679,15 +680,38 @@ $config['alert']['transports']['syslog']['syslog_facility'] = 3;
 ```
 ~
 
+## <a name="transports-elasticsearch">Elasticsearch</a>
+
+You can have LibreNMS emit alerts to an elasticsearch database. Each fault will be sent as a separate document.
+The index pattern uses strftime() formatting.
+The proxy setting uses the proxy set in config.php if true and does not if false; this allows you to use local servers.
+
+~
+```php
+$config['alert']['transports']['elasticsearch']['es_host']   = '127.0.0.1';
+$config['alert']['transports']['elasticsearch']['es_port']  = 9200;
+$config['alert']['transports']['elasticsearch']['es_index']  = 'librenms-%Y.%m.%d';
+$config['alert']['transports']['elasticsearch']['es_proxy'] = false;
+```
+~
+
 # <a name="entities">Entities
 
 Entities as described earlier are based on the table and column names within the database, if you are unsure of what the entity is you want then have a browse around inside MySQL using `show tables` and `desc <tablename>`.
 
 ## <a name="entity-devices">Devices</a>
 
-__devices.hostname__ = The devices hostname.
+__devices.hostname__ = The device hostname.
 
-__devices.location__ = The devices location.
+__devices.sysName__ = The device sysName.
+
+__devices.sysDescr__ = The device sysDescr.
+
+__devices.hardware__ = The device hardware.
+
+__devices.version__ = The device os version.
+
+__devices.location__ = The device location.
 
 __devices.status__ = The status of the device, 1 = up, 0 = down.
 

html/includes/forms/test-transport.inc.php

@@ -20,11 +20,15 @@ if (is_admin() === false) {
 $transport = mres($_POST['transport']);
 
 require_once $config['install_dir'].'/includes/alerts.inc.php';
-$tmp = array(dbFetchRow('select device_id,hostname from devices order by device_id asc limit 1'));
+$tmp = array(dbFetchRow('select device_id,hostname,sysDescr,version,hardware,location from devices order by device_id asc limit 1'));
 $tmp['contacts'] = GetContacts($tmp);
 $obj = array(
     "hostname"  => $tmp[0]['hostname'],
     "device_id" => $tmp[0]['device_id'],
+    "sysDescr" => $tmp[0]['sysDescr'],
+    "version" => $tmp[0]['version'],
+    "hardware" => $tmp[0]['hardware'],
+    "location" => $tmp[0]['location'],
     "title"     => "Testing transport from ".$config['project_name'],
     "elapsed"   => "11s",
     "id"        => "000",
@@ -33,6 +37,7 @@ $obj = array(
     "severity"  => "critical",
     "rule"      => "%macros.device = 1",
     "name"      => "Test-Rule",
+    "string"      => "#1: test => string;",
     "timestamp" => date("Y-m-d H:i:s"),
     "contacts"  => $tmp['contacts'],
     "state"     => "1",

html/pages/settings/alerting.inc.php

@@ -1348,6 +1348,58 @@ echo '
                             <input id="syslog_facility" class="form-control" type="text" name="global-config-input" value="'.$syslog_facility['config_value'].'" data-config_id="'.$syslog_facility['config_id'].'">
                             <span class="form-control-feedback">
     <i class="fa" aria-hidden="true"></i>
+</span>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>';
+$es_host     = get_config_by_name('alert.transports.elasticsearch.es_host');
+$es_port    = get_config_by_name('alert.transports.elasticsearch.es_port');
+$es_index    = get_config_by_name('alert.transports.elasticsearch.es_index');
+$es_proxy   = get_config_by_name('alert.transports.elasticsearch.es_proxy');
+echo '
+        <div class="panel panel-default">
+            <div class="panel-heading">
+                <h4 class="panel-title">
+                    <a data-toggle="collapse" data-parent="#accordion" href="#es_transport_expand"><i class="fa fa-caret-down"></i> Elasticsearch transport</a> <button name="test-alert" id="test-alert" type="button" data-transport="elasticsearch" class="btn btn-primary btn-xs pull-right">Test transport</button>
+                </h4>
+            </div>
+            <div id="es_transport_expand" class="panel-collapse collapse">
+                <div class="panel-body">
+                    <div class="form-group has-feedback">
+                        <label for="es_host" class="col-sm-4 control-label">Elasticsearch Host </label>
+                        <div class="col-sm-4">
+                            <input id="es_host" class="form-control" type="text" name="global-config-input" value="'.$es_host['config_value'].'" data-config_id="'.$es_host['config_id'].'">
+                            <span class="form-control-feedback">
+    <i class="fa" aria-hidden="true"></i>
+</span>
+                        </div>
+                    </div>
+                    <div class="form-group has-feedback">
+                        <label for="es_port" class="col-sm-4 control-label">Elasticsearch Port </label>
+                        <div class="col-sm-4">
+                            <input id="es_port" class="form-control" type="text" name="global-config-input" value="'.$es_port['config_value'].'" data-config_id="'.$es_port['config_id'].'">
+                            <span class="form-control-feedback">
+    <i class="fa" aria-hidden="true"></i>
+</span>
+                        </div>
+                    </div>
+                    <div class="form-group has-feedback">
+                        <label for="es_index" class="col-sm-4 control-label">Elasticsearch Index Pattern </label>
+                        <div class="col-sm-4">
+                            <input id="es_index" class="form-control" type="text" name="global-config-input" value="'.$es_index['config_value'].'" data-config_id="'.$es_index['config_id'].'">
+                            <span class="form-control-feedback">
+    <i class="fa" aria-hidden="true"></i>
+</span>
+                        </div>
+                    </div>
+                    <div class="form-group has-feedback">
+                        <label for="es_proxy" class="col-sm-4 control-label">Use proxy if configured? </label>
+                        <div class="col-sm-4">
+                            <input id="es_proxy" type="checkbox" name="global-config-check" '.$es_proxy['config_value'].' data-on-text="Yes" data-off-text="No" data-size="small" data-config_id="'.$es_proxy['config_id'].'">
+                            <span class="form-control-feedback">
+    <i class="fa" aria-hidden="true"></i>
 </span>
                         </div>
                     </div>

includes/alerts.inc.php

@@ -444,11 +444,14 @@ function DescribeAlert($alert)
 {
     $obj         = array();
     $i           = 0;
-    $device      = dbFetchRow('SELECT hostname, sysName, location, purpose, notes, uptime FROM devices WHERE device_id = ?', array($alert['device_id']));
+    $device      = dbFetchRow('SELECT hostname, sysName, sysDescr, hardware, version, location, purpose, notes, uptime FROM devices WHERE device_id = ?', array($alert['device_id']));
     $tpl         = dbFetchRow('SELECT `template`,`title`,`title_rec` FROM `alert_templates` JOIN `alert_template_map` ON `alert_template_map`.`alert_templates_id`=`alert_templates`.`id` WHERE `alert_template_map`.`alert_rule_id`=?', array($alert['rule_id']));
     $default_tpl = "%title\r\nSeverity: %severity\r\n{if %state == 0}Time elapsed: %elapsed\r\n{/if}Timestamp: %timestamp\r\nUnique-ID: %uid\r\nRule: {if %name}%name{else}%rule{/if}\r\n{if %faults}Faults:\r\n{foreach %faults}  #%key: %value.string\r\n{/foreach}{/if}Alert sent to: {foreach %contacts}%value <%key> {/foreach}";
     $obj['hostname']     = $device['hostname'];
     $obj['sysName']      = $device['sysName'];
+    $obj['sysDescr']      = $device['sysDescr'];
+    $obj['hardware']      = $device['hardware'];
+    $obj['version']      = $device['version'];
     $obj['location']     = $device['location'];
     $obj['uptime']       = $device['uptime'];
     $obj['uptime_short'] = formatUptime($device['uptime'], 'short');

includes/alerts/transport.elasticsearch.php

@@ -0,0 +1,173 @@
+/* LibreNMS 
+ *
+ * Copyright (C) 2017 Paul Blasquez <pblasquez@gmail.com>
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+
+$es_host = '127.0.0.1';
+$es_port = 9200;
+$index = strftime("librenms-%Y.%m.%d");
+$type = 'alert';
+$severity = $obj['severity'];
+$device = device_by_id_cache($obj['device_id']); // for event logging
+
+if (!empty($opts['es_host'])) {
+    if (preg_match("/[a-zA-Z]/", $opts['es_host'])) {
+        $es_host = gethostbyname($opts['es_host']);
+        if ($es_host === $opts['es_host']) {
+          return "Alphanumeric hostname found but does not resolve to an IP.";
+        }
+    } elseif (filter_var($opts['es_host'], FILTER_VALIDATE_IP)) {
+        $es_host = $opts['es_host'];
+    } else {
+        return "Elasticsearch host is not a valid IP: " . $opts['es_host'];
+    }
+}
+
+if (!empty($opts['es_port']) && preg_match("/^\d+$/", $opts['es_port'])) {
+    $es_port = $opts['es_port'];
+}
+
+if (!empty($opts['es_index'])) {
+    $index = strftime($opts['es_index']);
+}
+
+$host = $es_host.':'.$es_port.'/'.$index.'/'.$type;
+
+switch( $obj['state'] ) {
+case 0:
+    $state = "ok";
+    break;
+case 1:
+    $state = $severity;
+    break;
+case 2:
+    $state = "acknowledged";
+    break;
+case 3:
+    $state = "worse";
+    break;
+case 4:
+    $state = "better";
+    break;
+}
+
+$data = array(
+    '@timestamp' => strftime("%Y-%m-%dT%T"),
+    "host" => gethostname(),
+    "location" => $obj['location'],
+    "title" => $obj['name'],
+    "message" => $obj['string'],
+    "device_id" => $obj['device_id'],
+    "device_name" => $obj['hostname'],
+    "device_hardware" => $obj['hardware'],
+    "device_version" => $obj['version'],
+    "state" => $state,
+    "severity" => $severity,
+    "first_occurrence" => $obj['timestamp'],
+    "entity_type" => "device",
+    "entity_id" => $obj['device_id'],
+    "entity_name" => $obj['hostname'],
+    "entity_descr" => $obj['sysDescr'],
+);
+
+if( !empty( $obj['faults'] ) ) {
+    foreach($obj['faults'] as $k => $v) {
+        $curl = curl_init();
+        $data['message'] = $v['string'];
+        switch (true) {
+            case (array_key_exists('port_id', $v)):
+                $data['entity_type'] = 'port';
+                $data['entity_id'] = $v['port_id'];
+                $data['entity_name'] = $v['ifName'];
+                $data['entity_descr'] = $v['ifAlias'];
+                break;
+            case (array_key_exists('sensor_id', $v)):
+                $data['entity_type'] = 'sensor';
+                $data['entity_id'] = $v['sensor_id'];
+                $data['entity_name'] = $v['sensor_class'];
+                $data['entity_descr'] = $v['sensor_descr'];
+                break;
+            case (array_key_exists('mempool_id', $v)):
+                $data['entity_type'] = 'mempool';
+                $data['entity_id'] = $v['mempool_id'];
+                $data['entity_name'] = $v['mempool_index'];
+                $data['entity_descr'] = $v['mempool_descr'];
+                break;
+            case (array_key_exists('storage_id', $v)):
+                $data['entity_type'] = 'storage';
+                $data['entity_id'] = $v['storage_id'];
+                $data['entity_name'] = $v['storage_index'];
+                $data['entity_descr'] = $v['storage_descr'];
+                break;
+            case (array_key_exists('processor_id', $v)):
+                $data['entity_type'] = 'processor';
+                $data['entity_id'] = $v['processor_id'];
+                $data['entity_name'] = $v['processor_type'];
+                $data['entity_descr'] = $v['processor_descr'];
+                break;
+            case (array_key_exists('bgpPeer_id', $v)):
+                $data['entity_type'] = 'bgp';
+                $data['entity_id'] = $v['bgpPeer_id'];
+                $data['entity_name'] = 'local: '.$v['bgpPeerLocalAddr'].' - AS'.$obj['bgpLocalAs'];
+                $data['entity_descr'] = 'remote: '.$v['bgpPeerIdentifier'].' - AS'.$v['bgpPeerRemoteAs'];
+                break;
+            case (array_key_exists('tunnel_id', $v)):
+                $data['entity_type'] = 'ipsec_tunnel';
+                $data['entity_id'] = $v['tunnel_id'];
+                $data['entity_name'] = $v['tunnel_name'];
+                $data['entity_descr'] = 'local: '.$v['local_addr'].':'.$v['local_port'].', remote: '.$v['peer_addr'].':'.$v['peer_port'];
+                break;
+            default:
+                $data['entity_type'] = 'generic';
+                break;
+        }
+        $alert_message = json_encode($data);
+        curl_setopt($curl, CURLOPT_HTTPHEADER, array(
+            'Content-Type: application/json')
+        );
+        if ($opts['es_proxy'] === true) {
+            set_curl_proxy($curl);
+        }
+        curl_setopt($curl, CURLOPT_URL, $host);
+        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
+        curl_setopt($curl, CURLOPT_POST,true);
+        curl_setopt($curl, CURLOPT_POSTFIELDS, $alert_message );
+    
+        $ret = curl_exec($curl);
+        $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
+        if( $code != 200 && $code != 201 ) {
+            return $host.' returned HTTP Status code '.$code.' for '.$alert_message;
+        }
+    }
+} else {
+    $curl = curl_init();
+    $alert_message = json_encode($data);
+    curl_setopt($curl, CURLOPT_HTTPHEADER, array(
+        'Content-Type: application/json')
+    );
+    if ($opts['es_proxy'] === true) {
+        set_curl_proxy($curl);
+    }
+    curl_setopt($curl, CURLOPT_URL, $host);
+    curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
+    curl_setopt($curl, CURLOPT_POST,true);
+    curl_setopt($curl, CURLOPT_POSTFIELDS, $alert_message );
+
+    $ret = curl_exec($curl);
+    $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
+    if( $code != 200 && $code != 201 ) {
+        return $host.' returned HTTP Status code '.$code.' for '.$alert_message;
+    }
+}
+return true;

sql-schema/193.sql

@@ -0,0 +1 @@
+INSERT INTO config (config_name,config_value,config_default,config_descr,config_group,config_group_order,config_sub_group,config_sub_group_order,config_hidden,config_disabled) values ('alert.transports.elasticsearch.es_host','','127.0.0.1','Elasticsearch Host','alerting',0,'transports',0,'0','0'),('alert.transports.elasticsearch.es_port','','9200','Elasticsearch Port','alerting',0,'transports',0,'0','0'),('alert.transports.elasticsearch.es_index','','','Elasticsearch Index','alerting',0,'transports',0,'0','0'),('alert.transports.elasticsearch.es_proxy','','FALSE','Use Proxy?','alerting',0,'transports',0,'0','0');