Refactor FDB Tables to Laravel (#9669)

* Refactor FDB Tables to Laravel
Hopefully much better performance with large tables.
Better dns resolution (limit to 4 IPs tried per MAC)

* update style

* de-duplicate IPs

* fixed column width for mac and vlan

* Make DNS column visibility control whether or not we send the dns query.
Hide that column by default.
This commit is contained in:
Tony Murray
2019-01-20 08:43:36 -06:00
committed by GitHub
parent e2c4fa971e
commit ef41c9fd7d
14 changed files with 379 additions and 178 deletions

View File

@@ -94,7 +94,7 @@ abstract class PaginatedAjaxController extends Controller
* @param Model $model
* @return array|Collection|Model
*/
protected function formatItem($model)
public function formatItem($model)
{
return $model;
}
@@ -123,6 +123,7 @@ abstract class PaginatedAjaxController extends Controller
* @param Request $request
* @param Builder $query
* @param array $fields
* @return Builder
*/
protected function filter($request, $query, $fields)
{
@@ -135,6 +136,23 @@ abstract class PaginatedAjaxController extends Controller
}
}
}
return $query;
}
/**
* @param Request $request
* @param Builder $query
* @return Builder
*/
protected function sort($request, $query)
{
$sort = $request->get('sort', []);
foreach ($sort as $column => $direction) {
$query->orderBy($column, $direction);
}
return $query;
}
/**

View File

@@ -0,0 +1,277 @@
<?php
/**
* FdbTablesController.php
*
* FDB tables data for bootgrid display
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package LibreNMS
* @link http://librenms.org
* @copyright 2019 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace App\Http\Controllers\Table;
use App\Models\Ipv4Mac;
use App\Models\Port;
use App\Models\PortsFdb;
use App\Models\Vlan;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Http\Request;
use LibreNMS\Util\IP;
use LibreNMS\Util\Rewrite;
use LibreNMS\Util\Url;
class FdbTablesController extends TableController
{
protected $macCountCache = [];
protected $ipCache = [];
protected function rules()
{
return [
'port_id' => 'nullable|integer',
'device_id' => 'nullable|integer',
'serachby' => 'in:mac,vlan,dnsname,ip,description',
'dns' => 'nullable|in:true,false',
];
}
protected function filterFields($request)
{
return [
'ports_fdb.device_id' => 'device_id',
'ports_fdb.port_id' => 'port_id',
];
}
/**
* Defines the base query for this resource
*
* @param \Illuminate\Http\Request $request
* @return \Illuminate\Database\Eloquent\Builder|\Illuminate\Database\Query\Builder
*/
protected function baseQuery($request)
{
return PortsFdb::hasAccess($request->user())->with(['device', 'port', 'vlan'])->select('ports_fdb.*');
}
/**
* @param string $search
* @param Builder $query
* @param array $fields
* @return Builder|\Illuminate\Database\Query\Builder
*/
protected function search($search, $query, $fields = [])
{
if ($search = trim(\Request::get('searchPhrase'))) {
$mac_search = '%' . str_replace([':', ' ', '-', '.', '0x'], '', $search) . '%';
switch (\Request::get('searchby')) {
case 'mac':
return $query->where('ports_fdb.mac_address', 'like', $mac_search);
case 'vlan':
return $query->whereIn('ports_fdb.vlan_id', $this->findVlans($search));
case 'dnsname':
$search = gethostbyname($search);
// no break
case 'ip':
return $query->whereIn('ports_fdb.mac_address', $this->findMacs($search));
case 'description':
return $query->whereIn('ports_fdb.port_id', $this->findPorts($search));
default:
return $query->where(function ($query) use ($search, $mac_search) {
$query->where('ports_fdb.mac_address', 'like', $mac_search)
->orWhereIn('ports_fdb.port_id', $this->findPorts($search))
->orWhereIn('ports_fdb.vlan_id', $this->findVlans($search))
->orWhereIn('ports_fdb.mac_address', $this->findMacs($search));
});
}
}
return $query;
}
/**
* @param Request $request
* @param Builder $query
* @return Builder
*/
public function sort($request, $query)
{
$sort = $request->get('sort');
if (isset($sort['mac_address'])) {
$query->orderBy('mac_address', $sort['mac_address']);
}
if (isset($sort['device'])) {
$query->leftJoin('devices', 'ports_fdb.device_id', 'devices.device_id')
->orderBy('hostname', $sort['device']);
}
if (isset($sort['vlan'])) {
$query->leftJoin('vlans', 'ports_fdb.vlan_id', 'vlans.vlan_id')
->orderBy('vlan_vlan', $sort['vlan']);
}
if (isset($sort['interface'])) {
$query->leftJoin('ports', 'ports_fdb.port_id', 'ports.port_id')
->orderBy('ports.ifDescr', $sort['interface']);
}
if (isset($sort['description'])) {
$query->leftJoin('ports', 'ports_fdb.port_id', 'ports.port_id')
->orderBy('ports.ifDescr', $sort['description']);
}
return $query;
}
public function formatItem($fdb_entry)
{
$ip_info = $this->findIps($fdb_entry->mac_address);
$item = [
'device' => $fdb_entry->device ? Url::deviceLink($fdb_entry->device) : '',
'mac_address' => Rewrite::readableMac($fdb_entry->mac_address),
'ipv4_address' => $ip_info['ips']->implode(', '),
'interface' => '',
'vlan' => $fdb_entry->vlan ? $fdb_entry->vlan->vlan_vlan : '',
'description' => '',
'dnsname' => $ip_info['dns'],
];
if ($fdb_entry->port) {
$item['interface'] = Url::portLink($fdb_entry->port, $fdb_entry->port->getShortLabel());
$item['description'] = $fdb_entry->port->ifAlias;
if ($fdb_entry->port->ifInErrors > 0 || $fdb_entry->port->ifOutErrors > 0) {
$item['interface'] .= ' ' . Url::portLink($fdb_entry->port, '<i class="fa fa-flag fa-lg" style="color:red" aria-hidden="true"></i>');
}
if ($this->getMacCount($fdb_entry->port) == 1) {
// only one mac on this port, likely the endpoint
$item['interface'] .= ' <i class="fa fa-star fa-lg" style="color:green" aria-hidden="true" title="' . __('This indicates the most likely endpoint switchport') . '"></i>';
}
}
return $item;
}
/**
* @param string $ip
* @return Builder
*/
protected function findMacs($ip)
{
$port_id = \Request::get('port_id');
$device_id = \Request::get('device_id');
return Ipv4Mac::where('ipv4_address', 'like', "%$ip%")
->when($device_id, function ($query) use ($device_id) {
$query->where('device_id', $device_id);
})
->when($port_id, function ($query) use ($port_id) {
$query->where('port_id', $port_id);
})
->pluck('mac_address');
}
/**
* @param string $vlan
* @return Builder
*/
protected function findVlans($vlan)
{
$port_id = \Request::get('port_id');
$device_id = \Request::get('device_id');
return Vlan::where('vlan_vlan', $vlan)
->when($device_id, function ($query) use ($device_id) {
$query->where('device_id', $device_id);
})
->when($port_id, function ($query) use ($port_id) {
$query->whereIn('device_id', function ($query) use ($port_id) {
$query->select('device_id')->from('ports')->where('port_id', $port_id);
});
})
->pluck('vlan_id');
}
/**
* @param string $ifAlias
* @return Builder
*/
protected function findPorts($ifAlias)
{
$port_id = \Request::get('port_id');
$device_id = \Request::get('device_id');
return Port::where('ifAlias', 'like', "%$ifAlias%")
->when($device_id, function ($query) use ($device_id) {
$query->where('device_id', $device_id);
})
->when($port_id, function ($query) use ($port_id) {
$query->where('port_id', $port_id);
})
->pluck('port_id');
}
/**
* @param string $mac_address
* @return \Illuminate\Support\Collection
*/
protected function findIps($mac_address)
{
if (!isset($this->ipCache[$mac_address])) {
$ips = Ipv4Mac::where('mac_address', $mac_address)
->groupBy('ipv4_address')
->pluck('ipv4_address');
$dns = 'N/A';
// only fetch DNS if the column is visible
if (\Request::get('dns') == 'true') {
// don't try too many dns queries, this is the slowest part
foreach ($ips->take(3) as $ip) {
$hostname = gethostbyaddr($ip);
if (!IP::isValid($hostname)) {
$dns = $hostname;
break;
}
}
}
$this->ipCache[$mac_address] = [
'ips' => $ips,
'dns' => $dns,
];
}
return $this->ipCache[$mac_address];
}
/**
* @param Port $port
* @return int
*/
protected function getMacCount($port)
{
if (!isset($this->macCountCache[$port->port_id])) {
$this->macCountCache[$port->port_id] = $port->fdbEntries()->count();
}
return $this->macCountCache[$port->port_id];
}
}

View File

@@ -52,11 +52,7 @@ abstract class TableController extends PaginatedAjaxController
$this->search($request->get('searchPhrase'), $query, $this->searchFields($request));
$this->filter($request, $query, $this->filterFields($request));
$sort = $request->get('sort', $this->default_sort);
foreach ($sort as $column => $direction) {
$query->orderBy($column, $direction);
}
$this->sort($request, $query);
$limit = $request->get('rowCount', 25);
$page = $request->get('current', 1);

11
app/Models/Ipv4Mac.php Normal file
View File

@@ -0,0 +1,11 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
class Ipv4Mac extends Model
{
protected $table = 'ipv4_mac';
public $timestamps = false;
}

View File

@@ -211,6 +211,11 @@ class Port extends BaseModel
return $this->morphMany(Eventlog::class, 'events', 'type', 'reference');
}
public function fdbEntries()
{
return $this->hasMany('App\Models\PortsFdb', 'port_id', 'port_id');
}
public function users()
{
// FIXME does not include global read

32
app/Models/PortsFdb.php Normal file
View File

@@ -0,0 +1,32 @@
<?php
namespace App\Models;
class PortsFdb extends BaseModel
{
protected $table = 'ports_fdb';
protected $primaryKey = 'ports_fdb_id';
public $timestamps = false;
public function scopeHasAccess($query, User $user)
{
return $this->hasPortAccess($query, $user);
}
// ---- Define Relationships ----
public function device()
{
return $this->belongsTo('App\Models\Device', 'device_id', 'device_id');
}
public function port()
{
return $this->belongsTo('App\Models\Port', 'port_id', 'port_id');
}
public function vlan()
{
return $this->belongsTo('App\Models\Vlan', 'vlan_id', 'vlan_id');
}
}

View File

@@ -49,7 +49,6 @@ class PortsNac extends BaseModel
'time_elapsed',
];
public function scopeHasAccess($query, User $user)
{
return $this->hasPortAccess($query, $user);

10
app/Models/Vlan.php Normal file
View File

@@ -0,0 +1,10 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
class Vlan extends Model
{
public $timestamps = false;
}

View File

@@ -1,147 +0,0 @@
<?php
use LibreNMS\Authentication\LegacyAuth;
$param = array();
$select = "SELECT `F`.`port_id` AS `port_id`, `F`.`device_id`, `ifInErrors`, `ifOutErrors`, `ifOperStatus`,";
$select .= " `ifAdminStatus`, `ifAlias`, `ifDescr`, `mac_address`, `V`.`vlan_vlan` AS `vlan`,";
$select .= " `hostname`, `hostname` AS `device` , group_concat(`M`.`ipv4_address` SEPARATOR ', ') AS `ipv4_address`,";
$select .= " `P`.`ifDescr` AS `interface`";
$sql = " FROM `ports_fdb` AS `F`";
$sql .= " LEFT JOIN `devices` AS `D` USING(`device_id`)";
$sql .= " LEFT JOIN `ports` AS `P` USING(`port_id`, `device_id`)";
$sql .= " LEFT JOIN `vlans` AS `V` USING(`vlan_id`, `device_id`)";
// Add counter so we can ORDER BY the port_id with least amount of macs attached
$sql .= " LEFT JOIN ( SELECT `port_id`, COUNT(*) `portCount` FROM `ports_fdb` GROUP BY `port_id` ) AS `C` ON `C`.`port_id` = `F`.`port_id`";
$where = " WHERE 1";
if (!LegacyAuth::user()->hasGlobalRead()) {
$sql .= ' LEFT JOIN `devices_perms` AS `DP` USING (`device_id`)';
$where .= ' AND `DP`.`user_id`=?';
$param[] = LegacyAuth::id();
}
if (is_numeric($vars['device_id'])) {
$where .= ' AND `F`.`device_id`=?';
$param[] = $vars['device_id'];
}
if (is_numeric($vars['port_id'])) {
$where .= ' AND `F`.`port_id`=?';
$param[] = $vars['port_id'];
}
if (isset($vars['searchPhrase']) && !empty($vars['searchPhrase'])) {
$search = mres(trim($vars['searchPhrase']));
$mac_search = '%'.str_replace(array(':', ' ', '-', '.', '0x'), '', $search).'%';
if (isset($vars['searchby']) && $vars['searchby'] == 'vlan') {
$where .= ' AND `V`.`vlan_vlan` = ?';
$param[] = (int)$search;
} elseif (isset($vars['searchby']) && $vars['searchby'] == 'ip') {
$ip = $vars['searchPhrase'];
$ip_search = '%'.mres(trim($ip)).'%';
$sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)";
$where .= ' AND `M`.`ipv4_address` LIKE ?';
$param[] = $ip_search;
} elseif (isset($vars['searchby']) && $vars['searchby'] == 'dnsname') {
$ip = gethostbyname($vars['searchPhrase']);
$ip_search = '%'.mres(trim($ip)).'%';
$sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)";
$where .= ' AND `M`.`ipv4_address` LIKE ?';
$param[] = $ip_search;
} elseif (isset($vars['searchby']) && $vars['searchby'] == 'description') {
$desc_search = '%' . $search . '%';
$where .= ' AND `P`.`ifAlias` LIKE ?';
$param[] = $desc_search;
} elseif (isset($vars['searchby']) && $vars['searchby'] == 'mac') {
$where .= ' AND `F`.`mac_address` LIKE ?';
$param[] = $mac_search;
} else {
$sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)";
$where .= ' AND (`V`.`vlan_vlan` = ? OR `F`.`mac_address` LIKE ? OR `P`.`ifAlias` LIKE ? OR `M`.`ipv4_address` LIKE ?)';
$param[] = (int)$search;
$param[] = $mac_search;
$param[] = '%' . $search . '%';
$param[] = '%' . gethostbyname(trim($vars['searchPhrase'])) . '%';
}
}
$total = (int)dbFetchCell("SELECT COUNT(*) $sql $where", $param);
// Don't use ipv4_mac in count it will inflate the rows unless we aggregate it
// Except for ip search.
if (empty($vars['searchPhrase']) || isset($vars['searchby']) && $vars['searchby'] != 'ip' && $vars['searchby'] != 'dnsname') {
$sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)";
}
$sql .= $where;
$sql .= " GROUP BY `device_id`, `port_id`, `mac_address`, `vlan`";
// Get most likely endpoint port_id, used to add a visual marker for this element
// in the list
if (isset($vars['searchby']) && !empty($vars['searchPhrase']) && $vars['searchby'] != 'vlan') {
$countsql .= " ORDER BY `C`.`portCount` ASC LIMIT 1";
foreach (dbFetchRows($select . $sql . $countsql, $param) as $entry) {
$endpoint_portid = $entry['port_id'];
}
}
if (!isset($sort) || empty($sort)) {
$sort = '`C`.`portCount` ASC';
}
$sql .= " ORDER BY $sort";
if (isset($current)) {
$limit_low = (($current * $rowCount) - ($rowCount));
$limit_high = $rowCount;
}
if ($rowCount != -1) {
$sql .= " LIMIT $limit_low,$limit_high";
}
$response = array();
foreach (dbFetchRows($select . $sql, $param) as $entry) {
$entry = cleanPort($entry);
if (!$ignore) {
if ($entry['ifInErrors'] > 0 || $entry['ifOutErrors'] > 0) {
$error_img = generate_port_link(
$entry,
"<i class='fa fa-flag fa-lg' style='color:red' aria-hidden='true'></i>",
'port_errors'
);
} else {
$error_img = '';
}
if ($entry['port_id'] == $endpoint_portid) {
$endpoint_img = "<i class='fa fa-star fa-lg' style='color:green' aria-hidden='true' title='This indicates the most likely endpoint switchport'></i>";
$dnsname = gethostbyaddr(reset(explode(',', $entry['ipv4_address']))) ?: 'N/A';
} else {
$endpoint_img = '';
$dnsname = "N/A";
}
$response[] = array(
'device' => generate_device_link(device_by_id_cache($entry['device_id'])),
'mac_address' => formatMac($entry['mac_address']),
'ipv4_address' => $entry['ipv4_address'],
'interface' => generate_port_link($entry, makeshortif(fixifname($entry['label']))).' '.$error_img.' '.$endpoint_img,
'vlan' => $entry['vlan'],
'description' => $entry['ifAlias'],
'dnsname' => $dnsname,
);
}//end if
unset($ignore);
}//end foreach
$output = array(
'current' => $current,
'rowCount' => $rowCount,
'rows' => $response,
'total' => $total,
);
echo _json_encode($output);

View File

@@ -4,10 +4,11 @@ $no_refresh = true;
<table id="port-fdb" class="table table-condensed table-hover table-striped">
<thead>
<tr>
<th data-column-id="mac_address">MAC address</th>
<th data-column-id="ipv4_address">IPv4 Address</th>
<th data-column-id="mac_address" data-width="150px">MAC Address</th>
<th data-column-id="ipv4_address" data-sortable="false">IPv4 Address</th>
<th data-column-id="interface">Port</th>
<th data-column-id="vlan">Vlan</th>
<th data-column-id="vlan" data-width="60px">Vlan</th>
<th data-column-id="dnsname" data-sortable="false" data-visible="false">DNS Name</th>
</tr>
</thead>
</table>
@@ -19,11 +20,11 @@ var grid = $("#port-fdb").bootgrid({
post: function ()
{
return {
id: "fdb-search",
port_id: "<?php echo $port['port_id']; ?>"
port_id: "<?php echo $port['port_id']; ?>",
dns: $("#port-fdb").bootgrid("getColumnSettings")[4].visible
};
},
url: "ajax_table.php"
url: "ajax/table/fdb-tables"
});
</script>

View File

@@ -4,12 +4,12 @@ $no_refresh = true;
<table id="ports-fdb" class="table table-condensed table-hover table-striped">
<thead>
<tr>
<th data-column-id="mac_address">MAC address</th>
<th data-column-id="ipv4_address">IPv4 Address</th>
<th data-column-id="mac_address" data-width="150px">MAC Address</th>
<th data-column-id="ipv4_address" data-sortable="false">IPv4 Address</th>
<th data-column-id="interface">Port</th>
<th data-column-id="description">Description</th>
<th data-column-id="vlan">Vlan</th>
<th data-column-id="dnsname" data-visible="false">DNS Name</th>
<th data-column-id="vlan" data-width="60px">Vlan</th>
<th data-column-id="dnsname" data-sortable="false" data-visible="false">DNS Name</th>
</tr>
</thead>
</table>
@@ -21,11 +21,11 @@ var grid = $("#ports-fdb").bootgrid({
post: function ()
{
return {
id: "fdb-search",
device_id: "<?php echo $device['device_id']; ?>"
device_id: "<?php echo $device['device_id']; ?>",
dns: $("#ports-fdb").bootgrid("getColumnSettings")[5].visible
};
},
url: "ajax_table.php"
url: "ajax/table/fdb-tables"
});
</script>

View File

@@ -6,12 +6,12 @@
<thead>
<tr>
<th data-column-id="device">Device</th>
<th data-column-id="mac_address">MAC Address</th>
<th data-column-id="ipv4_address">IPv4 Address</th>
<th data-column-id="mac_address" data-width="150px">MAC Address</th>
<th data-column-id="ipv4_address" data-sortable="false">IPv4 Address</th>
<th data-column-id="interface">Port</th>
<th data-column-id="vlan">Vlan</th>
<th data-column-id="vlan" data-width="60px">Vlan</th>
<th data-column-id="description">Description</th>
<th data-column-id="dnsname">DNS Name</th>
<th data-column-id="dnsname" data-sortable="false" data-visible="false">DNS Name</th>
</tr>
</thead>
</table>
@@ -114,13 +114,13 @@ echo '"'.$vars['searchPhrase'].'"+';
post: function ()
{
return {
id: "fdb-search",
device_id: '<?php echo $vars['device_id']; ?>',
searchby: '<?php echo $vars['searchby']; ?>',
searchPhrase: '<?php echo $vars['searchPhrase']; ?>'
searchPhrase: '<?php echo $vars['searchPhrase']; ?>',
dns: $("#fdb-search").bootgrid("getColumnSettings")[6].visible
};
},
url: "ajax_table.php"
url: "ajax/table/fdb-tables"
});
</script>

View File

@@ -36,9 +36,7 @@ function rewrite_location($location)
function formatMac($mac)
{
$mac = preg_replace('/(..)(..)(..)(..)(..)(..)/', '\\1:\\2:\\3:\\4:\\5:\\6', $mac);
return $mac;
return \LibreNMS\Util\Rewrite::readableMac($mac);
}

View File

@@ -62,9 +62,10 @@ Route::group(['middleware' => ['auth', '2fa'], 'guard' => 'auth'], function () {
Route::group(['prefix' => 'table', 'namespace' => 'Table'], function () {
Route::post('customers', 'CustomersController');
Route::post('eventlog', 'EventlogController');
Route::post('fdb-tables', 'FdbTablesController');
Route::post('graylog', 'GraylogController');
Route::post('location', 'LocationController');
Route::post('port-nac', 'PortNacController');
Route::post('graylog', 'GraylogController');
Route::post('syslog', 'SyslogController');
});