fix: page/graph load speed: part 1 (#6611)

* fix: page/graph load speed: part 1
Speed up load_all_os() by caching the full $config['os'] array
Function time goes from 1.13648s -> 0.00073s
The cache is updated during discovery, but only if it is >22hrs old.
Additionally, the cache is not used and a full yaml load is forced if the cache is >24hrs old.
The cache time is controlled by $config['os_def_cache_time'], I don't see any reason anyone would change that time.

* Delete Session.php

* Include the cache directory.

cache/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

discovery.php

@@ -118,6 +118,8 @@ if (get_lock('schema') === false) {
     require 'includes/sql-schema/update.php';
 }
 
+update_os_cache(); // will only update if needed
+
 $discovered_devices = 0;
 
 if (!empty($config['distributed_poller_group'])) {

includes/common.php

@@ -1576,25 +1576,40 @@ function load_os(&$device)
 }
 
 /**
- * @param array $restricted
+ * Load all OS, optionally load just the OS used by existing devices
+ * Default cache time is 1 day. Controlled by os_def_cache_time.
+ *
+ * @param bool $existing Only load OS that have existing OS in the database
+ * @param bool $cached Load os definitions from the cache file
  */
-function load_all_os($restricted = array())
+function load_all_os($existing = false, $cached = true)
 {
     global $config;
-    if (!empty($restricted)) {
+    $cache_file = $config['install_dir'] . '/cache/os_defs.cache';
-        $list = $restricted;
+
-    } else {
+    if ($cached && is_file($cache_file) && (time() - filemtime($cache_file) < $config['os_def_cache_time'])) {
-        $list = glob($config['install_dir'].'/includes/definitions/*.yaml');
+        // Cached
-        if (count($list) == count($config['os'])) {
+        $os_defs = unserialize(file_get_contents($cache_file));
-            // already fully loaded
+
-            return;
+        if ($existing) {
-        }
+            // remove unneeded os
+            $os_defs = array_diff_key($os_defs, dbFetchColumn('SELECT DISTINCT(`os`) FROM `devices`'));
         }
 
-    foreach ($list as $file) {
+        $config['os'] = array_replace_recursive($os_defs, $config['os']);
-        $tmp = Symfony\Component\Yaml\Yaml::parse(
+    } else {
-            file_get_contents($file)
+        // load from yaml
-        );
+        if ($existing) {
+            $os_list = array_map(function ($os) use ($config) {
+                return $config['install_dir'] . '/includes/definitions/'. $os . '.yaml';
+            }, dbFetchColumn('SELECT DISTINCT(`os`) FROM `devices`'));
+        } else {
+            $os_list = glob($config['install_dir'].'/includes/definitions/*.yaml');
+        }
+
+        foreach ($os_list as $file) {
+            $tmp = Symfony\Component\Yaml\Yaml::parse(file_get_contents($file));
+
             if (isset($config['os'][$tmp['os']])) {
                 $config['os'][$tmp['os']] = array_replace_recursive($tmp, $config['os'][$tmp['os']]);
             } else {
@@ -1602,6 +1617,24 @@ function load_all_os($restricted = array())
             }
         }
     }
+}
+
+/**
+ * Update the OS cache file cache/os_defs.cache
+ */
+function update_os_cache()
+{
+    global $config;
+    $cache_file = $config['install_dir'] . '/cache/os_defs.cache';
+    $cache_keep_time = $config['os_def_cache_time'] - 7200; // 2hr buffer
+
+    if (!is_file($cache_file) || time() - filemtime($cache_file) > $cache_keep_time) {
+        d_echo('Updating os_def.cache... ');
+        load_all_os(false, false);
+        file_put_contents($cache_file, serialize($config['os']));
+        d_echo("Done\n");
+    }
+}
 
 /**
  * @param $scale
@@ -1704,7 +1737,7 @@ function get_user_pref($name, $default = null, $user_id = null)
 {
     global $user_prefs;
 
-    if (array_key_exists($name, $user_prefs)) {
+    if (is_array($user_prefs) && array_key_exists($name, $user_prefs)) {
         return $user_prefs[$name];
     }
 

includes/defaults.inc.php

@@ -191,6 +191,9 @@ $config['snmp']['v3'][0]['cryptoalgo'] = 'AES';
 // Devices must respond to icmp by default
 $config['icmp_check'] = true;
 
+// The amount of time to keep the OS cache
+$config['os_def_cache_time'] = 86400;
+
 // Autodiscovery Settings
 $config['autodiscovery']['xdp'] = true;
 $config['autodiscovery']['xdp_exclude']['sysdesc_regexp'][] = '/-K9W8-/'; // Cisco Lightweight Access Point

includes/init.php

@@ -152,12 +152,8 @@ if (module_selected('web', $init_modules)) {
         $config['title_image'] = 'images/librenms_logo_'.$config['site_style'].'.svg';
     }
     require $install_dir . '/html/includes/vars.inc.php';
-    $tmp_list = dbFetchRows('SELECT DISTINCT(`os`) FROM `devices`');
+
-    $os_list = array();
+    load_all_os(true);
-    foreach ($tmp_list as $k => $v) {
-        $os_list[] = $config['install_dir'].'/includes/definitions/'. $v['os'] . '.yaml';
-    }
-    load_all_os($os_list);
 }
 
 $console_color = new Console_Color2();

tests/OSDiscoveryTest.php

@@ -143,7 +143,10 @@ class OSDiscoveryTest extends \PHPUnit_Framework_TestCase
     {
         global $config;
 
-        load_all_os(); // make sure all OS are loaded
+        // make sure all OS are loaded
+        if (count($config['os']) < count(glob($config['install_dir'].'/includes/definitions/*.yaml'))) {
+            load_all_os();
+        }
 
         $excluded_os = array(
             'default',