From c8c8bc3a81b9a250c77cb53fcd7baf652875f262 Mon Sep 17 00:00:00 2001
From: Filippo Giunchedi <filippog@users.noreply.github.com>
Date: Sat, 27 Mar 2021 16:01:48 +0100
Subject: [PATCH] Support multiple Alertmanager URLs (#12346)

---
 LibreNMS/Alert/Transport/Alertmanager.php | 59 +++++++++++++++++------
 doc/Alerting/Transports.md                |  9 ++--
 2 files changed, 51 insertions(+), 17 deletions(-)

diff --git a/LibreNMS/Alert/Transport/Alertmanager.php b/LibreNMS/Alert/Transport/Alertmanager.php
index 37e88eacc1..4262f2b392 100644
--- a/LibreNMS/Alert/Transport/Alertmanager.php
+++ b/LibreNMS/Alert/Transport/Alertmanager.php
@@ -35,7 +35,7 @@ class Alertmanager extends Transport
         return $this->contactAlertmanager($obj, $alertmanager_opts);
     }
 
-    public static function contactAlertmanager($obj, $api)
+    public function contactAlertmanager($obj, $api)
     {
         if ($obj['state'] == AlertState::RECOVERED) {
             $alertmanager_status = 'endsAt';
@@ -43,8 +43,6 @@ class Alertmanager extends Transport
             $alertmanager_status = 'startsAt';
         }
         $gen_url = (Config::get('base_url') . 'device/device=' . $obj['device_id']);
-        $host = ($api['url'] . '/api/v2/alerts');
-        $curl = curl_init();
         $alertmanager_msg = strip_tags($obj['msg']);
         $data = [[
             $alertmanager_status => date('c'),
@@ -61,26 +59,59 @@ class Alertmanager extends Transport
             ],
         ]];
 
+        $url = $api['url'];
         unset($api['url']);
         foreach ($api as $label => $value) {
             $data[0]['labels'][$label] = $value;
         }
 
-        $alert_message = json_encode($data);
-        curl_setopt($curl, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
+        return $this->postAlerts($url, $data);
+    }
+
+    public static function postAlerts($url, $data)
+    {
+        $curl = curl_init();
         set_curl_proxy($curl);
-        curl_setopt($curl, CURLOPT_URL, $host);
+        curl_setopt($curl, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
         curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
+        curl_setopt($curl, CURLOPT_TIMEOUT, 5);
+        curl_setopt($curl, CURLOPT_TIMEOUT_MS, 5000);
+        curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 5);
+        curl_setopt($curl, CURLOPT_CONNECTTIMEOUT_MS, 5000);
         curl_setopt($curl, CURLOPT_POST, true);
+
+        $alert_message = json_encode($data);
         curl_setopt($curl, CURLOPT_POSTFIELDS, $alert_message);
 
-        $ret = curl_exec($curl);
-        $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
-        if ($code != 200) {
-            return 'HTTP Status code ' . $code;
+        foreach (explode(',', $url) as $am) {
+            $post_url = ($am . '/api/v2/alerts');
+            curl_setopt($curl, CURLOPT_URL, $post_url);
+            $ret = curl_exec($curl);
+            if ($ret === false || curl_errno($curl)) {
+                logfile("Failed to contact $post_url: " . curl_error($curl));
+                continue;
+            }
+            $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
+            if ($code == 200) {
+                curl_close($curl);
+
+                return true;
+            }
         }
 
-        return true;
+        $err = "Unable to POST to Alertmanager at $post_url .";
+
+        if ($ret === false || curl_errno($curl)) {
+            $err .= ' cURL error: ' . curl_error($curl);
+        } else {
+            $err .= ' HTTP status: ' . curl_getinfo($curl, CURLINFO_HTTP_CODE);
+        }
+
+        curl_close($curl);
+
+        logfile($err);
+
+        return $err;
     }
 
     public static function configTemplate()
@@ -88,9 +119,9 @@ class Alertmanager extends Transport
         return [
             'config' => [
                 [
-                    'title' => 'Alertmanager URL',
+                    'title' => 'Alertmanager URL(s)',
                     'name' => 'alertmanager-url',
-                    'descr' => 'Alertmanager Webhook URL',
+                    'descr' => 'Alertmanager Webhook URL(s). Can contain comma-separated URLs',
                     'type' => 'text',
                 ],
                 [
@@ -101,7 +132,7 @@ class Alertmanager extends Transport
                 ],
             ],
             'validation' => [
-                'alertmanager-url' => 'required|url',
+                'alertmanager-url' => 'required|string',
             ],
         ];
     }
diff --git a/doc/Alerting/Transports.md b/doc/Alerting/Transports.md
index a3d91687ed..4228632fdd 100644
--- a/doc/Alerting/Transports.md
+++ b/doc/Alerting/Transports.md
@@ -62,17 +62,20 @@ of alerts of similar content for an array of hosts, whereas
 Alertmanager can group them by alert meta, ideally producing one
 single notice in case an issue occurs.
 
-It is possible to configure as much label values as required in
-Alertmanager Options section. Every label and it's value should be
+It is possible to configure as many label values as required in
+Alertmanager Options section. Every label and its value should be
 entered as a new line.
 
+Multiple Alertmanager URLs (comma separated) are supported. Each
+URL will be tried and the search will stop at the first success.
+
 [Alertmanager Docs](https://prometheus.io/docs/alerting/alertmanager/)
 
 **Example:**
 
 | Config | Example |
 | ------ | ------- |
-| Alertmanager URL      | http://alertmanager.example.com |
+| Alertmanager URL(s)   | http://alertmanager1.example.com,http://alertmanager2.example.com |
 | Alertmanager Options: | source=librenms <br/> customlabel=value |
 
 ## API