feat(apps/mysql): add error-state to non-responsive mysql-servers (#13993)

* feat(apps/mysql): add error-state to non-responsive mysql-servers

* feat(alerting): add alert-rule for offline mysql-servers

* fixup! feat(alerting): add alert-rule for offline mysql-servers

* chore(style): replaced ticks

* fixup! chore(style): replaced ticks

* feat: added migration to make app_status field bigger

* feat: moved error-detection to the application_update function

* fixup! feat: moved error-detection to the application_update function

* chore: updated schema

* chore: fix styling

* Cleaner way to handle the error states

* regex should not include OK

Co-authored-by: Tony Murray <murraytony@gmail.com>

database/migrations/2022_05_30_084932_update-app-status-length.php

@@ -0,0 +1,32 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class UpdateAppStatusLength extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('applications', function (Blueprint $table) {
+            $table->string('app_status', 1024)->change();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('applications', function (Blueprint $table) {
+            $table->string('app_status', 8)->change();
+        });
+    }
+}

includes/polling/functions.inc.php

@@ -459,12 +459,15 @@ function update_application($app, $response, $metrics = [], $status = '')
     ];
 
     if ($response != '' && $response !== false) {
+        // if the response indicates an error, set it and set app_status to the raw response
         if (Str::contains($response, [
             'Traceback (most recent call last):',
         ])) {
             $data['app_state'] = 'ERROR';
-        } elseif (in_array($response, ['OK', 'ERROR', 'LEGACY', 'UNSUPPORTED'])) {
+            $data['app_status'] = $response;
-            $data['app_state'] = $response;
+        } elseif (preg_match('/^(ERROR|LEGACY|UNSUPPORTED)/', $response, $matches)) {
+            $data['app_state'] = $matches[1];
+            $data['app_status'] = $response;
         } else {
             // should maybe be 'unknown' as state
             $data['app_state'] = 'OK';

misc/alert_rules.json

@@ -527,5 +527,10 @@
     "rule": "applications.app_type = \"suricata\" && application_metrics.metric = \".total_error_delta\" && application_metrics.value >= \"2\"",
     "name": "Suricata Packet Error >= 2%",
     "severity": "critical"
+  },
+  {
+    "rule": "applications.app_type = \"mysql\" && applications.app_state != \"OK\"",
+    "name": "MySQL Server not responding",
+    "severity":"critical"
   }
 ]

misc/db_schema.yaml

@@ -168,7 +168,7 @@ applications:
     - { Field: app_state, Type: varchar(32), 'Null': false, Extra: '', Default: UNKNOWN }
     - { Field: discovered, Type: tinyint, 'Null': false, Extra: '', Default: '0' }
     - { Field: app_state_prev, Type: varchar(32), 'Null': true, Extra: '' }
-    - { Field: app_status, Type: varchar(8), 'Null': false, Extra: '' }
+    - { Field: app_status, Type: varchar(1024), 'Null': false, Extra: '' }
     - { Field: timestamp, Type: timestamp, 'Null': false, Extra: 'on update CURRENT_TIMESTAMP', Default: CURRENT_TIMESTAMP }
     - { Field: app_instance, Type: varchar(255), 'Null': false, Extra: '' }
   Indexes: