Updated device cache support (#10795)

* Device Cache
also some additional device related models

* attribs from cache

* replace common attribute functions

* remove legacy cache usage
tidy up some collection manipulation
remove some unused or single use functions

* cleanup some items

* always return a device, to prevent bugs

* clear device cache when testing after each test

* fix double assignment

* Clean up function to take advantage of null object

LibreNMS/Cache/Device.php

@@ -0,0 +1,118 @@
+<?php
+/**
+ * Device.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2019 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace LibreNMS\Cache;
+
+class Device
+{
+    private $devices = [];
+    private $primary;
+
+    /**
+     * Gets the current primary device.
+     *
+     * @return \App\Models\Device
+     */
+    public function getPrimary() : \App\Models\Device
+    {
+        return self::get($this->primary);
+    }
+
+    /**
+     * Set the primary device.
+     * This will be fetched by getPrimary()
+     *
+     * @param int $device_id
+     */
+    public function setPrimary($device_id)
+    {
+        $this->primary = $device_id;
+    }
+
+    /**
+     * Get a device by device_id
+     *
+     * @param int $device_id
+     * @return \App\Models\Device
+     */
+    public function get($device_id) : \App\Models\Device
+    {
+        if (!array_key_exists($device_id, $this->devices)) {
+            return self::load($device_id);
+        }
+
+        return $this->devices[$device_id] ?: new \App\Models\Device;
+    }
+
+    /**
+     * Get a device by hostname
+     *
+     * @param string $hostname
+     * @return \App\Models\Device
+     */
+    public function getByHostname($hostname) : \App\Models\Device
+    {
+        $device_id = collect($this->devices)->pluck('device_id', 'hostname')->get($hostname);
+
+        if (!$device_id) {
+            return $this->load($hostname, 'hostname');
+        }
+
+        return $this->devices[$device_id] ?: new \App\Models\Device;
+    }
+
+    /**
+     * Ignore cache and load the device fresh from the database
+     *
+     * @param int $device_id
+     * @return \App\Models\Device
+     */
+    public function refresh($device_id) : \App\Models\Device
+    {
+        unset($this->devices[$device_id]);
+        return self::get($device_id);
+    }
+
+    /**
+     * Flush the cache
+     */
+    public function flush()
+    {
+        $this->devices = [];
+    }
+
+    private function load($value, $field = 'device_id')
+    {
+        $device = \App\Models\Device::query()->where($field, $value)->first();
+
+        if (!$device) {
+            return new \App\Models\Device;
+        }
+
+        $device->loadOs();
+        $this->devices[$device->device_id] = $device;
+        return $device;
+    }
+}

app/Facades/DeviceCache.php

@@ -0,0 +1,36 @@
+<?php
+/**
+ * DeviceCache.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2019 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace App\Facades;
+
+use Illuminate\Support\Facades\Facade;
+
+class DeviceCache extends Facade
+{
+    protected static function getFacadeAccessor()
+    {
+        return 'device-cache';
+    }
+}

app/Models/Device.php

@@ -7,6 +7,7 @@ use Fico7489\Laravel\Pivot\Traits\PivotEventTrait;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\ModelNotFoundException;
 use Illuminate\Database\Query\JoinClause;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Str;
 use LibreNMS\Exceptions\InvalidIpException;
 use LibreNMS\Util\IP;
@@ -334,6 +335,45 @@ class Device extends BaseModel
         $this->save();
     }
 
+    public function getAttrib($name)
+    {
+        return $this->attribs->pluck('attrib_value', 'attrib_type')->get($name);
+    }
+
+    public function setAttrib($name, $value)
+    {
+        $attrib = $this->attribs->first(function ($item) use ($name) {
+            return $item->attrib_type === $name;
+        });
+
+        if (!$attrib) {
+            $attrib = new DeviceAttrib(['attrib_type' => $name]);
+            $this->attribs->push($attrib);
+        }
+
+        $attrib->attrib_value = $value;
+        return (bool)$this->attribs()->save($attrib);
+    }
+
+    public function forgetAttrib($name)
+    {
+        $attrib_index = $this->attribs->search(function ($attrib) use ($name) {
+            return $attrib->attrib_type === $name;
+        });
+
+        if ($attrib_index !== false) {
+            $this->attribs->forget($attrib_index);
+            return (bool)$this->attribs->get($attrib_index)->delete();
+        }
+
+        return false;
+    }
+
+    public function getAttribs()
+    {
+        return $this->attribs->pluck('attrib_value', 'attrib_type')->toArray();
+    }
+
     // ---- Accessors/Mutators ----
 
     public function getIconAttribute($icon)
@@ -445,6 +485,11 @@ class Device extends BaseModel
         return $this->hasMany('App\Models\Alert', 'device_id');
     }
 
+    public function attribs()
+    {
+        return $this->hasMany('App\Models\DeviceAttrib', 'device_id');
+    }
+
     public function alertSchedules()
     {
         return $this->morphToMany('App\Models\AlertSchedule', 'alert_schedulable', 'alert_schedulables', 'schedule_id', 'schedule_id');
@@ -606,6 +651,11 @@ class Device extends BaseModel
         return $this->belongsToMany('App\Models\User', 'devices_perms', 'device_id', 'user_id');
     }
 
+    public function vrfLites()
+    {
+        return $this->hasMany('App\Models\VrfLite', 'device_id');
+    }
+
     public function vrfs()
     {
         return $this->hasMany('App\Models\Vrf', 'device_id');

app/Models/DeviceAttrib.php

@@ -0,0 +1,34 @@
+<?php
+/**
+ * DeviceAttrib.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2019 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace App\Models;
+
+class DeviceAttrib extends DeviceRelatedModel
+{
+    protected $table = 'devices_attribs';
+    protected $primaryKey = 'attrib_id';
+    public $timestamps = false;
+    protected $fillable = ['attrib_type', 'attrib_value'];
+}

app/Models/VrfLite.php

@@ -0,0 +1,33 @@
+<?php
+/**
+ * VrfLite.php
+ *
+ * -Description-
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2019 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+namespace App\Models;
+
+class VrfLite extends DeviceRelatedModel
+{
+    protected $table = 'vrf_lite_cisco';
+    protected $primaryKey = 'vrf_lite_cisco_id';
+    public $timestamps = false;
+}

app/Providers/AppServiceProvider.php

@@ -6,7 +6,6 @@ use Illuminate\Database\Eloquent\Relations\Relation;
 use Illuminate\Support\Facades\Blade;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\ServiceProvider;
-use Illuminate\Validation\Rule;
 use LibreNMS\Config;
 use LibreNMS\Permissions;
 use LibreNMS\Util\IP;
@@ -28,6 +27,9 @@ class AppServiceProvider extends ServiceProvider
         $this->app->singleton('permissions', function ($app) {
             return new Permissions();
         });
+        $this->app->singleton('device-cache', function ($app) {
+            return new \LibreNMS\Cache\Device();
+        });
     }
 
     /**

config/app.php

@@ -240,6 +240,7 @@ return [
 
         // LibreNMS
         'Permissions' => \App\Facades\Permissions::class,
+        'DeviceCache' => \App\Facades\DeviceCache::class,
     ],
 
 ];

discovery.php

@@ -117,6 +117,7 @@ if (!empty(\LibreNMS\Config::get('distributed_poller_group'))) {
 
 global $device;
 foreach (dbFetch("SELECT * FROM `devices` WHERE disabled = 0 AND snmp_disable = 0 $where ORDER BY device_id DESC", $sqlparams) as $device) {
+    DeviceCache::setPrimary($device['device_id']);
     $discovered_devices += (int)discover_device($device, $module_override);
 }
 

includes/common.php

@@ -16,6 +16,7 @@
  * the source code distribution for details.
  */
 
+use App\Models\Device;
 use LibreNMS\Config;
 use LibreNMS\Exceptions\InvalidIpException;
 use LibreNMS\Util\Git;
@@ -239,23 +240,6 @@ function get_port_by_ifIndex($device_id, $ifIndex)
     return dbFetchRow("SELECT * FROM `ports` WHERE `device_id` = ? AND `ifIndex` = ?", array($device_id, $ifIndex));
 }
 
-function get_all_devices()
-{
-    global $cache;
-    $devices = array();
-
-    // FIXME needs access control checks!
-    // FIXME respect $type (server, network, etc) -- needs an array fill in topnav.
-
-    if (isset($cache['devices']['hostname'])) {
-        $devices = array_keys($cache['devices']);
-    } else {
-        $devices = dbFetchRows("SELECT * FROM `devices` ORDER BY hostname");
-    }
-
-    return $devices;
-}
-
 function table_from_entity_type($type)
 {
     // Fuck you, english pluralisation.
@@ -347,10 +331,9 @@ function ifclass($ifOperStatus, $ifAdminStatus)
     return \LibreNMS\Util\Url::portLinkDisplayClass((object) ['ifOperStatus' => $ifOperStatus, 'ifAdminStatus' => $ifAdminStatus]);
 }
 
-function device_by_name($name, $refresh = 0)
+function device_by_name($name)
 {
-    // FIXME - cache name > id too.
+    return device_by_id_cache(getidbyname($name));
-    return device_by_id_cache(getidbyname($name), $refresh);
 }
 
 
@@ -365,29 +348,15 @@ function accesspoint_by_id($ap_id, $refresh = '0')
 
 function device_by_id_cache($device_id, $refresh = false)
 {
-    global $cache;
+    $model = $refresh ? DeviceCache::refresh($device_id) : DeviceCache::get($device_id);
 
-    if (!$refresh && isset($cache['devices']['id'][$device_id]) && is_array($cache['devices']['id'][$device_id])) {
+    $device = $model->toArray();
-        $device = $cache['devices']['id'][$device_id];
+    $device['location'] = $model->location->location;
-    } else {
+    $device['lat'] = $model->location->lat;
-        $device = dbFetchRow("SELECT `devices`.*, `location`, `lat`, `lng` FROM `devices` LEFT JOIN locations ON `devices`.location_id=`locations`.`id` WHERE `device_id` = ?", [$device_id]);
+    $device['lng'] = $model->location->lng;
-        $device['attribs'] = get_dev_attribs($device['device_id']);
+    $device['attribs'] = $model->getAttribs();
-        load_os($device);
+    $device['vrf_lite_cisco'] = $model->vrfLites->keyBy('context_name')->toArray();
 
-        //order vrf_lite_cisco with context, this will help to get the vrf_name and instance_name all the time
-        $vrfs_lite_cisco = dbFetchRows("SELECT * FROM `vrf_lite_cisco` WHERE `device_id` = ?", array($device_id));
-        if (!empty($vrfs_lite_cisco)) {
-            $device['vrf_lite_cisco'] = array();
-            foreach ($vrfs_lite_cisco as $vrf) {
-                $device['vrf_lite_cisco'][$vrf['context_name']] = $vrf;
-            }
-        }
-
-        if (!empty($device['ip'])) {
-            $device['ip'] = inet6_ntop($device['ip']);
-        }
-        $cache['devices']['id'][$device_id] = $device;
-    }
     return $device;
 }
 
@@ -419,17 +388,9 @@ function getifhost($id)
     return dbFetchCell("SELECT `device_id` from `ports` WHERE `port_id` = ?", array($id));
 }
 
-function gethostbyid($id)
+function gethostbyid($device_id)
 {
-    global $cache;
+    return DeviceCache::get($device_id)->hostname;
-
-    if (isset($cache['devices']['id'][$id]['hostname'])) {
-        $hostname = $cache['devices']['id'][$id]['hostname'];
-    } else {
-        $hostname = dbFetchCell("SELECT `hostname` FROM `devices` WHERE `device_id` = ?", array($id));
-    }
-
-    return $hostname;
 }
 
 function strgen($length = 16)
@@ -470,28 +431,7 @@ function getifdescrbyid($id)
 
 function getidbyname($hostname)
 {
-    global $cache;
+    return DeviceCache::getByHostname($hostname)->device_id;
-
-    if (isset($cache['devices']['hostname'][$hostname])) {
-        $id = $cache['devices']['hostname'][$hostname];
-    } else {
-        $id = dbFetchCell("SELECT `device_id` FROM `devices` WHERE `hostname` = ?", array($hostname));
-    }
-
-    return $id;
-}
-
-function gethostosbyid($id)
-{
-    global $cache;
-
-    if (isset($cache['devices']['id'][$id]['os'])) {
-        $os = $cache['devices']['id'][$id]['os'];
-    } else {
-        $os = dbFetchCell("SELECT `os` FROM `devices` WHERE `device_id` = ?", array($id));
-    }
-
-    return $os;
 }
 
 function safename($name)
@@ -516,21 +456,12 @@ function zeropad($num, $length = 2)
 
 function set_dev_attrib($device, $attrib_type, $attrib_value)
 {
-    if (dbFetchCell("SELECT COUNT(*) FROM devices_attribs WHERE `device_id` = ? AND `attrib_type` = ?", array($device['device_id'],$attrib_type))) {
+    return DeviceCache::get($device['device_id'])->setAttrib($attrib_type, $attrib_value);
-        $return = dbUpdate(array('attrib_value' => $attrib_value), 'devices_attribs', 'device_id=? and attrib_type=?', array($device['device_id'], $attrib_type));
-    } else {
-        $return = dbInsert(array('device_id' => $device['device_id'], 'attrib_type' => $attrib_type, 'attrib_value' => $attrib_value), 'devices_attribs');
-    }
-    return $return;
 }
 
-function get_dev_attribs($device)
+function get_dev_attribs($device_id)
 {
-    $attribs = array();
+    return DeviceCache::get($device_id)->getAttribs();
-    foreach (dbFetchRows("SELECT * FROM devices_attribs WHERE `device_id` = ?", array($device)) as $entry) {
-        $attribs[$entry['attrib_type']] = $entry['attrib_value'];
-    }
-    return $attribs;
 }
 
 function get_dev_entity_state($device)
@@ -543,36 +474,14 @@ function get_dev_entity_state($device)
     return $state;
 }
 
-function get_dev_attrib($device, $attrib_type, $attrib_value = '')
+function get_dev_attrib($device, $attrib_type)
 {
-    $sql = '';
+    return DeviceCache::get($device['device_id'])->getAttrib($attrib_type);
-    $params = array($device['device_id'], $attrib_type);
-    if (!empty($attrib_value)) {
-        $sql = " AND `attrib_value`=?";
-        array_push($params, $attrib_value);
-    }
-    if ($row = dbFetchRow("SELECT attrib_value FROM devices_attribs WHERE `device_id` = ? AND `attrib_type` = ? $sql", $params)) {
-        return $row['attrib_value'];
-    } else {
-        return null;
-    }
-}
-
-function is_dev_attrib_enabled($device, $attrib, $default = true)
-{
-    $val = get_dev_attrib($device, $attrib);
-    if ($val != null) {
-        // attribute is set
-        return ($val != 0);
-    } else {
-        // attribute not set
-        return $default;
-    }
 }
 
 function del_dev_attrib($device, $attrib_type)
 {
-    return dbDelete('devices_attribs', "`device_id` = ? AND `attrib_type` = ?", array($device['device_id'], $attrib_type));
+    return DeviceCache::get($device['device_id'])->forgetAttrib($attrib_type);
 }
 
 function formatRates($value, $round = '2', $sf = '3')

includes/discovery/functions.inc.php

@@ -126,9 +126,8 @@ function discover_device(&$device, $force_module = false)
 
     $valid = array();
     // Reset $valid array
-    $attribs = get_dev_attribs($device['device_id']);
+    $attribs = DeviceCache::getPrimary()->getAttribs();
     $device['attribs'] = $attribs;
-    $device['snmp_max_repeaters'] = $attribs['snmp_max_repeaters'];
 
     $device_start = microtime(true);
     // Start counting device poll time

includes/functions.php

@@ -2470,8 +2470,8 @@ function get_db_schema()
 function get_device_oid_limit($device)
 {
     // device takes priority
-    if ($device['snmp_max_oid'] > 0) {
+    if ($device['attribs']['snmp_max_oid'] > 0) {
-        return $device['snmp_max_oid'];
+        return $device['attribs']['snmp_max_oid'];
     }
 
     // then os

includes/html/object-cache.inc.php

@@ -5,13 +5,6 @@
 // It would be interesting to know where this is used.  It probably should have its own API.
 use LibreNMS\ObjectCache;
 
-foreach (dbFetchRows('SELECT * FROM `devices` ORDER BY `hostname`') as $device) {
-    $cache['devices']['hostname'][$device['hostname']] = $device['device_id'];
-    $cache['devices']['id'][$device['device_id']]      = $device;
-
-    $cache['device_types'][$device['type']]++;
-}
-
 $devices  = new ObjectCache('devices');
 $ports    = new ObjectCache('ports');
 $services = new ObjectCache('services');

includes/html/pages/device.inc.php

@@ -17,6 +17,7 @@ if (device_permitted($vars['device']) || $permitted_by_port) {
     }
     $select = array($tab => 'class="active"');
 
+    DeviceCache::setPrimary($vars['device']);
     $device = device_by_id_cache($vars['device']);
     $attribs = get_dev_attribs($device['device_id']);
     $device['attribs'] = $attribs;
@@ -381,7 +382,7 @@ if (device_permitted($vars['device']) || $permitted_by_port) {
         }
 
         if ($device_config_file) {
-            if (!get_dev_attrib($device, 'override_Oxidized_disable', 'true')) {
+            if (!get_dev_attrib($device, 'override_Oxidized_disable') === 'true') {
                 echo '<li class="'.$select['showconfig'].'">
                     <a href="'.generate_device_url($device, array('tab' => 'showconfig')).'">
                     <i class="fa fa-align-justify fa-lg icon-theme"  aria-hidden="true"></i> Config

includes/html/pages/device/edit/apps.inc.php

@@ -28,7 +28,7 @@ foreach ($applications as $app) {
     if (isset($enabled_apps[$app])) {
         $modifiers = ' checked';
         if ($enabled_apps[$app]
-            && is_dev_attrib_enabled($device, "poll_applications", Config::getOsSetting($device['os'], "poller_modules.applications"))
+            && (get_dev_attrib($device, 'poll_applications') || Config::getOsSetting($device['os'], "poller_modules.applications"))
         ) {
             $app_text .= '<span class="text-success"> (Discovered)</span>';
             $modifiers .= ' disabled';

includes/polling/functions.inc.php

@@ -232,14 +232,11 @@ function poll_device($device, $force_module = false)
 
     $device_start = microtime(true);
 
-    $attribs = get_dev_attribs($device['device_id']);
+    $attribs = DeviceCache::getPrimary()->getAttribs();
     $device['attribs'] = $attribs;
 
     load_os($device);
 
-    $device['snmp_max_repeaters'] = $attribs['snmp_max_repeaters'];
-    $device['snmp_max_oid'] = $attribs['snmp_max_oid'];
-
     unset($array);
 
     // Start counting device poll time

includes/snmp.inc.php

@@ -1381,7 +1381,7 @@ function snmpwalk_array_num($device, $oid, $indexes = 1)
  */
 function get_device_max_repeaters($device)
 {
-    return $device['snmp_max_repeaters'] ?:
+    return $device['attribs']['snmp_max_repeaters'] ??
         Config::getOsSetting($device['os'], 'snmp.max_repeaters', false);
 }
 

poller.php

@@ -164,6 +164,7 @@ if (!isset($query)) {
 }
 
 foreach (dbFetch($query) as $device) {
+    DeviceCache::setPrimary($device['device_id']);
     if ($device['os_group'] == 'cisco') {
         $device['vrf_lite_cisco'] = dbFetchRows("SELECT * FROM `vrf_lite_cisco` WHERE `device_id` = " . $device['device_id']);
     } else {

scripts/collect-port-polling.php

@@ -30,6 +30,8 @@ if (isset($options['help'])) {
     exit(0);
 }
 
+$where = '';
+$params = [];
 if (isset($options['h'])) {
     if (is_numeric($options['h'])) {
         $where = "AND `device_id` = ?";
@@ -43,10 +45,8 @@ if (isset($options['h'])) {
         $where = "AND `hostname` LIKE ?";
         $params = array(str_replace('*', '%', mres($options['h'])));
     }
-    $devices = dbFetch("SELECT * FROM `devices` WHERE status = 1 AND disabled = 0 $where ORDER BY `hostname` ASC", $params);
-} else {
-    $devices = get_all_devices();
 }
+$devices = dbFetchRows("SELECT * FROM `devices` WHERE status = 1 AND disabled = 0 $where ORDER BY `hostname` ASC", $params);
 
 if (isset($options['e'])) {
     if (!is_numeric($options['e']) || $options['e'] < 0) {

tests/OSDiscoveryTest.php

@@ -130,6 +130,7 @@ class OSDiscoveryTest extends TestCase
             'community' => $community,
             'os' => 'generic',
             'os_group' => '',
+            'attribs' => [],
         ];
     }
 

tests/OSModulesTest.php

@@ -25,6 +25,7 @@
 
 namespace LibreNMS\Tests;
 
+use DeviceCache;
 use LibreNMS\Config;
 use LibreNMS\Exceptions\FileNotFoundException;
 use LibreNMS\Exceptions\InvalidModuleException;
@@ -132,6 +133,8 @@ class OSModulesTest extends DBTestCase
                 . "\nOS $os: Polled $module data does not match that found in $filename"
             );
         }
+
+        DeviceCache::flush(); // clear cached devices
     }
 
     public function dumpedDataProvider()