From ef41c9fd7da7a5463f2116c0d49dc2b6bcf7c5da Mon Sep 17 00:00:00 2001 From: Tony Murray Date: Sun, 20 Jan 2019 08:43:36 -0600 Subject: [PATCH] Refactor FDB Tables to Laravel (#9669) * Refactor FDB Tables to Laravel Hopefully much better performance with large tables. Better dns resolution (limit to 4 IPs tried per MAC) * update style * de-duplicate IPs * fixed column width for mac and vlan * Make DNS column visibility control whether or not we send the dns query. Hide that column by default. --- .../Controllers/PaginatedAjaxController.php | 20 +- .../Controllers/Table/FdbTablesController.php | 277 ++++++++++++++++++ .../Controllers/Table/TableController.php | 6 +- app/Models/Ipv4Mac.php | 11 + app/Models/Port.php | 5 + app/Models/PortsFdb.php | 32 ++ app/Models/PortsNac.php | 1 - app/Models/Vlan.php | 10 + html/includes/table/fdb-search.inc.php | 147 ---------- html/pages/device/port/fdb.inc.php | 13 +- html/pages/device/ports/fdb.inc.php | 14 +- html/pages/search/fdb.inc.php | 14 +- includes/rewrites.php | 4 +- routes/web.php | 3 +- 14 files changed, 379 insertions(+), 178 deletions(-) create mode 100644 app/Http/Controllers/Table/FdbTablesController.php create mode 100644 app/Models/Ipv4Mac.php create mode 100644 app/Models/PortsFdb.php create mode 100644 app/Models/Vlan.php delete mode 100644 html/includes/table/fdb-search.inc.php diff --git a/app/Http/Controllers/PaginatedAjaxController.php b/app/Http/Controllers/PaginatedAjaxController.php index 5fd2f86a9e..7d5f779d09 100644 --- a/app/Http/Controllers/PaginatedAjaxController.php +++ b/app/Http/Controllers/PaginatedAjaxController.php @@ -94,7 +94,7 @@ abstract class PaginatedAjaxController extends Controller * @param Model $model * @return array|Collection|Model */ - protected function formatItem($model) + public function formatItem($model) { return $model; } @@ -123,6 +123,7 @@ abstract class PaginatedAjaxController extends Controller * @param Request $request * @param Builder $query * @param array $fields + * @return Builder */ protected function filter($request, $query, $fields) { @@ -135,6 +136,23 @@ abstract class PaginatedAjaxController extends Controller } } } + + return $query; + } + + /** + * @param Request $request + * @param Builder $query + * @return Builder + */ + protected function sort($request, $query) + { + $sort = $request->get('sort', []); + foreach ($sort as $column => $direction) { + $query->orderBy($column, $direction); + } + + return $query; } /** diff --git a/app/Http/Controllers/Table/FdbTablesController.php b/app/Http/Controllers/Table/FdbTablesController.php new file mode 100644 index 0000000000..4d60a24466 --- /dev/null +++ b/app/Http/Controllers/Table/FdbTablesController.php @@ -0,0 +1,277 @@ +. + * + * @package LibreNMS + * @link http://librenms.org + * @copyright 2019 Tony Murray + * @author Tony Murray + */ + +namespace App\Http\Controllers\Table; + +use App\Models\Ipv4Mac; +use App\Models\Port; +use App\Models\PortsFdb; +use App\Models\Vlan; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Http\Request; +use LibreNMS\Util\IP; +use LibreNMS\Util\Rewrite; +use LibreNMS\Util\Url; + +class FdbTablesController extends TableController +{ + protected $macCountCache = []; + protected $ipCache = []; + + protected function rules() + { + return [ + 'port_id' => 'nullable|integer', + 'device_id' => 'nullable|integer', + 'serachby' => 'in:mac,vlan,dnsname,ip,description', + 'dns' => 'nullable|in:true,false', + ]; + } + + protected function filterFields($request) + { + return [ + 'ports_fdb.device_id' => 'device_id', + 'ports_fdb.port_id' => 'port_id', + ]; + } + + /** + * Defines the base query for this resource + * + * @param \Illuminate\Http\Request $request + * @return \Illuminate\Database\Eloquent\Builder|\Illuminate\Database\Query\Builder + */ + protected function baseQuery($request) + { + return PortsFdb::hasAccess($request->user())->with(['device', 'port', 'vlan'])->select('ports_fdb.*'); + } + + /** + * @param string $search + * @param Builder $query + * @param array $fields + * @return Builder|\Illuminate\Database\Query\Builder + */ + protected function search($search, $query, $fields = []) + { + if ($search = trim(\Request::get('searchPhrase'))) { + $mac_search = '%' . str_replace([':', ' ', '-', '.', '0x'], '', $search) . '%'; + switch (\Request::get('searchby')) { + case 'mac': + return $query->where('ports_fdb.mac_address', 'like', $mac_search); + case 'vlan': + return $query->whereIn('ports_fdb.vlan_id', $this->findVlans($search)); + case 'dnsname': + $search = gethostbyname($search); + // no break + case 'ip': + return $query->whereIn('ports_fdb.mac_address', $this->findMacs($search)); + case 'description': + return $query->whereIn('ports_fdb.port_id', $this->findPorts($search)); + default: + return $query->where(function ($query) use ($search, $mac_search) { + $query->where('ports_fdb.mac_address', 'like', $mac_search) + ->orWhereIn('ports_fdb.port_id', $this->findPorts($search)) + ->orWhereIn('ports_fdb.vlan_id', $this->findVlans($search)) + ->orWhereIn('ports_fdb.mac_address', $this->findMacs($search)); + }); + } + } + + return $query; + } + + /** + * @param Request $request + * @param Builder $query + * @return Builder + */ + public function sort($request, $query) + { + $sort = $request->get('sort'); + + if (isset($sort['mac_address'])) { + $query->orderBy('mac_address', $sort['mac_address']); + } + + if (isset($sort['device'])) { + $query->leftJoin('devices', 'ports_fdb.device_id', 'devices.device_id') + ->orderBy('hostname', $sort['device']); + } + + if (isset($sort['vlan'])) { + $query->leftJoin('vlans', 'ports_fdb.vlan_id', 'vlans.vlan_id') + ->orderBy('vlan_vlan', $sort['vlan']); + } + + if (isset($sort['interface'])) { + $query->leftJoin('ports', 'ports_fdb.port_id', 'ports.port_id') + ->orderBy('ports.ifDescr', $sort['interface']); + } + + if (isset($sort['description'])) { + $query->leftJoin('ports', 'ports_fdb.port_id', 'ports.port_id') + ->orderBy('ports.ifDescr', $sort['description']); + } + + return $query; + } + + public function formatItem($fdb_entry) + { + $ip_info = $this->findIps($fdb_entry->mac_address); + + $item = [ + 'device' => $fdb_entry->device ? Url::deviceLink($fdb_entry->device) : '', + 'mac_address' => Rewrite::readableMac($fdb_entry->mac_address), + 'ipv4_address' => $ip_info['ips']->implode(', '), + 'interface' => '', + 'vlan' => $fdb_entry->vlan ? $fdb_entry->vlan->vlan_vlan : '', + 'description' => '', + 'dnsname' => $ip_info['dns'], + ]; + + if ($fdb_entry->port) { + $item['interface'] = Url::portLink($fdb_entry->port, $fdb_entry->port->getShortLabel()); + $item['description'] = $fdb_entry->port->ifAlias; + if ($fdb_entry->port->ifInErrors > 0 || $fdb_entry->port->ifOutErrors > 0) { + $item['interface'] .= ' ' . Url::portLink($fdb_entry->port, ''); + } + if ($this->getMacCount($fdb_entry->port) == 1) { + // only one mac on this port, likely the endpoint + $item['interface'] .= ' '; + } + } + + return $item; + } + + /** + * @param string $ip + * @return Builder + */ + protected function findMacs($ip) + { + $port_id = \Request::get('port_id'); + $device_id = \Request::get('device_id'); + + return Ipv4Mac::where('ipv4_address', 'like', "%$ip%") + ->when($device_id, function ($query) use ($device_id) { + $query->where('device_id', $device_id); + }) + ->when($port_id, function ($query) use ($port_id) { + $query->where('port_id', $port_id); + }) + ->pluck('mac_address'); + } + + /** + * @param string $vlan + * @return Builder + */ + protected function findVlans($vlan) + { + $port_id = \Request::get('port_id'); + $device_id = \Request::get('device_id'); + + return Vlan::where('vlan_vlan', $vlan) + ->when($device_id, function ($query) use ($device_id) { + $query->where('device_id', $device_id); + }) + ->when($port_id, function ($query) use ($port_id) { + $query->whereIn('device_id', function ($query) use ($port_id) { + $query->select('device_id')->from('ports')->where('port_id', $port_id); + }); + }) + ->pluck('vlan_id'); + } + + /** + * @param string $ifAlias + * @return Builder + */ + protected function findPorts($ifAlias) + { + $port_id = \Request::get('port_id'); + $device_id = \Request::get('device_id'); + + return Port::where('ifAlias', 'like', "%$ifAlias%") + ->when($device_id, function ($query) use ($device_id) { + $query->where('device_id', $device_id); + }) + ->when($port_id, function ($query) use ($port_id) { + $query->where('port_id', $port_id); + }) + ->pluck('port_id'); + } + + /** + * @param string $mac_address + * @return \Illuminate\Support\Collection + */ + protected function findIps($mac_address) + { + if (!isset($this->ipCache[$mac_address])) { + $ips = Ipv4Mac::where('mac_address', $mac_address) + ->groupBy('ipv4_address') + ->pluck('ipv4_address'); + + $dns = 'N/A'; + + // only fetch DNS if the column is visible + if (\Request::get('dns') == 'true') { + // don't try too many dns queries, this is the slowest part + foreach ($ips->take(3) as $ip) { + $hostname = gethostbyaddr($ip); + if (!IP::isValid($hostname)) { + $dns = $hostname; + break; + } + } + } + + $this->ipCache[$mac_address] = [ + 'ips' => $ips, + 'dns' => $dns, + ]; + } + + return $this->ipCache[$mac_address]; + } + + /** + * @param Port $port + * @return int + */ + protected function getMacCount($port) + { + if (!isset($this->macCountCache[$port->port_id])) { + $this->macCountCache[$port->port_id] = $port->fdbEntries()->count(); + } + + return $this->macCountCache[$port->port_id]; + } +} diff --git a/app/Http/Controllers/Table/TableController.php b/app/Http/Controllers/Table/TableController.php index ed8116b7ce..7ee2d6738d 100644 --- a/app/Http/Controllers/Table/TableController.php +++ b/app/Http/Controllers/Table/TableController.php @@ -52,11 +52,7 @@ abstract class TableController extends PaginatedAjaxController $this->search($request->get('searchPhrase'), $query, $this->searchFields($request)); $this->filter($request, $query, $this->filterFields($request)); - - $sort = $request->get('sort', $this->default_sort); - foreach ($sort as $column => $direction) { - $query->orderBy($column, $direction); - } + $this->sort($request, $query); $limit = $request->get('rowCount', 25); $page = $request->get('current', 1); diff --git a/app/Models/Ipv4Mac.php b/app/Models/Ipv4Mac.php new file mode 100644 index 0000000000..80137901ef --- /dev/null +++ b/app/Models/Ipv4Mac.php @@ -0,0 +1,11 @@ +morphMany(Eventlog::class, 'events', 'type', 'reference'); } + public function fdbEntries() + { + return $this->hasMany('App\Models\PortsFdb', 'port_id', 'port_id'); + } + public function users() { // FIXME does not include global read diff --git a/app/Models/PortsFdb.php b/app/Models/PortsFdb.php new file mode 100644 index 0000000000..bbec391230 --- /dev/null +++ b/app/Models/PortsFdb.php @@ -0,0 +1,32 @@ +hasPortAccess($query, $user); + } + + // ---- Define Relationships ---- + + public function device() + { + return $this->belongsTo('App\Models\Device', 'device_id', 'device_id'); + } + + public function port() + { + return $this->belongsTo('App\Models\Port', 'port_id', 'port_id'); + } + + public function vlan() + { + return $this->belongsTo('App\Models\Vlan', 'vlan_id', 'vlan_id'); + } +} diff --git a/app/Models/PortsNac.php b/app/Models/PortsNac.php index a58d7cc4a6..1243f2d302 100644 --- a/app/Models/PortsNac.php +++ b/app/Models/PortsNac.php @@ -49,7 +49,6 @@ class PortsNac extends BaseModel 'time_elapsed', ]; - public function scopeHasAccess($query, User $user) { return $this->hasPortAccess($query, $user); diff --git a/app/Models/Vlan.php b/app/Models/Vlan.php new file mode 100644 index 0000000000..6668c08adc --- /dev/null +++ b/app/Models/Vlan.php @@ -0,0 +1,10 @@ +hasGlobalRead()) { - $sql .= ' LEFT JOIN `devices_perms` AS `DP` USING (`device_id`)'; - $where .= ' AND `DP`.`user_id`=?'; - $param[] = LegacyAuth::id(); -} - -if (is_numeric($vars['device_id'])) { - $where .= ' AND `F`.`device_id`=?'; - $param[] = $vars['device_id']; -} - -if (is_numeric($vars['port_id'])) { - $where .= ' AND `F`.`port_id`=?'; - $param[] = $vars['port_id']; -} - -if (isset($vars['searchPhrase']) && !empty($vars['searchPhrase'])) { - $search = mres(trim($vars['searchPhrase'])); - $mac_search = '%'.str_replace(array(':', ' ', '-', '.', '0x'), '', $search).'%'; - - if (isset($vars['searchby']) && $vars['searchby'] == 'vlan') { - $where .= ' AND `V`.`vlan_vlan` = ?'; - $param[] = (int)$search; - } elseif (isset($vars['searchby']) && $vars['searchby'] == 'ip') { - $ip = $vars['searchPhrase']; - $ip_search = '%'.mres(trim($ip)).'%'; - $sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)"; - $where .= ' AND `M`.`ipv4_address` LIKE ?'; - $param[] = $ip_search; - } elseif (isset($vars['searchby']) && $vars['searchby'] == 'dnsname') { - $ip = gethostbyname($vars['searchPhrase']); - $ip_search = '%'.mres(trim($ip)).'%'; - $sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)"; - $where .= ' AND `M`.`ipv4_address` LIKE ?'; - $param[] = $ip_search; - } elseif (isset($vars['searchby']) && $vars['searchby'] == 'description') { - $desc_search = '%' . $search . '%'; - $where .= ' AND `P`.`ifAlias` LIKE ?'; - $param[] = $desc_search; - } elseif (isset($vars['searchby']) && $vars['searchby'] == 'mac') { - $where .= ' AND `F`.`mac_address` LIKE ?'; - $param[] = $mac_search; - } else { - $sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)"; - $where .= ' AND (`V`.`vlan_vlan` = ? OR `F`.`mac_address` LIKE ? OR `P`.`ifAlias` LIKE ? OR `M`.`ipv4_address` LIKE ?)'; - $param[] = (int)$search; - $param[] = $mac_search; - $param[] = '%' . $search . '%'; - $param[] = '%' . gethostbyname(trim($vars['searchPhrase'])) . '%'; - } -} - -$total = (int)dbFetchCell("SELECT COUNT(*) $sql $where", $param); - -// Don't use ipv4_mac in count it will inflate the rows unless we aggregate it -// Except for ip search. -if (empty($vars['searchPhrase']) || isset($vars['searchby']) && $vars['searchby'] != 'ip' && $vars['searchby'] != 'dnsname') { - $sql .= " LEFT JOIN `ipv4_mac` AS `M` USING (`mac_address`)"; -} -$sql .= $where; -$sql .= " GROUP BY `device_id`, `port_id`, `mac_address`, `vlan`"; - -// Get most likely endpoint port_id, used to add a visual marker for this element -// in the list -if (isset($vars['searchby']) && !empty($vars['searchPhrase']) && $vars['searchby'] != 'vlan') { - $countsql .= " ORDER BY `C`.`portCount` ASC LIMIT 1"; - foreach (dbFetchRows($select . $sql . $countsql, $param) as $entry) { - $endpoint_portid = $entry['port_id']; - } -} - -if (!isset($sort) || empty($sort)) { - $sort = '`C`.`portCount` ASC'; -} -$sql .= " ORDER BY $sort"; - -if (isset($current)) { - $limit_low = (($current * $rowCount) - ($rowCount)); - $limit_high = $rowCount; -} - -if ($rowCount != -1) { - $sql .= " LIMIT $limit_low,$limit_high"; -} - -$response = array(); -foreach (dbFetchRows($select . $sql, $param) as $entry) { - $entry = cleanPort($entry); - if (!$ignore) { - if ($entry['ifInErrors'] > 0 || $entry['ifOutErrors'] > 0) { - $error_img = generate_port_link( - $entry, - "", - 'port_errors' - ); - } else { - $error_img = ''; - } - if ($entry['port_id'] == $endpoint_portid) { - $endpoint_img = ""; - $dnsname = gethostbyaddr(reset(explode(',', $entry['ipv4_address']))) ?: 'N/A'; - } else { - $endpoint_img = ''; - $dnsname = "N/A"; - } - - $response[] = array( - 'device' => generate_device_link(device_by_id_cache($entry['device_id'])), - 'mac_address' => formatMac($entry['mac_address']), - 'ipv4_address' => $entry['ipv4_address'], - 'interface' => generate_port_link($entry, makeshortif(fixifname($entry['label']))).' '.$error_img.' '.$endpoint_img, - 'vlan' => $entry['vlan'], - 'description' => $entry['ifAlias'], - 'dnsname' => $dnsname, - ); - }//end if - - unset($ignore); -}//end foreach - -$output = array( - 'current' => $current, - 'rowCount' => $rowCount, - 'rows' => $response, - 'total' => $total, -); -echo _json_encode($output); diff --git a/html/pages/device/port/fdb.inc.php b/html/pages/device/port/fdb.inc.php index c175e14eff..4df7ede518 100644 --- a/html/pages/device/port/fdb.inc.php +++ b/html/pages/device/port/fdb.inc.php @@ -4,10 +4,11 @@ $no_refresh = true; - - + + - + +
MAC addressIPv4 AddressMAC AddressIPv4 Address PortVlanVlanDNS Name
@@ -19,11 +20,11 @@ var grid = $("#port-fdb").bootgrid({ post: function () { return { - id: "fdb-search", - port_id: "" + port_id: "", + dns: $("#port-fdb").bootgrid("getColumnSettings")[4].visible }; }, - url: "ajax_table.php" + url: "ajax/table/fdb-tables" }); diff --git a/html/pages/device/ports/fdb.inc.php b/html/pages/device/ports/fdb.inc.php index 1c192d1483..b7f5f7ec70 100644 --- a/html/pages/device/ports/fdb.inc.php +++ b/html/pages/device/ports/fdb.inc.php @@ -4,12 +4,12 @@ $no_refresh = true; - - + + - - + +
MAC addressIPv4 AddressMAC AddressIPv4 Address Port DescriptionVlanDNS NameVlanDNS Name
@@ -21,11 +21,11 @@ var grid = $("#ports-fdb").bootgrid({ post: function () { return { - id: "fdb-search", - device_id: "" + device_id: "", + dns: $("#ports-fdb").bootgrid("getColumnSettings")[5].visible }; }, - url: "ajax_table.php" + url: "ajax/table/fdb-tables" }); diff --git a/html/pages/search/fdb.inc.php b/html/pages/search/fdb.inc.php index 0e4fef9a30..5f9d041568 100644 --- a/html/pages/search/fdb.inc.php +++ b/html/pages/search/fdb.inc.php @@ -6,12 +6,12 @@ Device - MAC Address - IPv4 Address + MAC Address + IPv4 Address Port - Vlan + Vlan Description - DNS Name + DNS Name @@ -114,13 +114,13 @@ echo '"'.$vars['searchPhrase'].'"+'; post: function () { return { - id: "fdb-search", device_id: '', searchby: '', - searchPhrase: '' + searchPhrase: '', + dns: $("#fdb-search").bootgrid("getColumnSettings")[6].visible }; }, - url: "ajax_table.php" + url: "ajax/table/fdb-tables" }); diff --git a/includes/rewrites.php b/includes/rewrites.php index 0a3d2896fa..0b15382519 100644 --- a/includes/rewrites.php +++ b/includes/rewrites.php @@ -36,9 +36,7 @@ function rewrite_location($location) function formatMac($mac) { - $mac = preg_replace('/(..)(..)(..)(..)(..)(..)/', '\\1:\\2:\\3:\\4:\\5:\\6', $mac); - - return $mac; + return \LibreNMS\Util\Rewrite::readableMac($mac); } diff --git a/routes/web.php b/routes/web.php index 50869e5038..d05b97309b 100644 --- a/routes/web.php +++ b/routes/web.php @@ -62,9 +62,10 @@ Route::group(['middleware' => ['auth', '2fa'], 'guard' => 'auth'], function () { Route::group(['prefix' => 'table', 'namespace' => 'Table'], function () { Route::post('customers', 'CustomersController'); Route::post('eventlog', 'EventlogController'); + Route::post('fdb-tables', 'FdbTablesController'); + Route::post('graylog', 'GraylogController'); Route::post('location', 'LocationController'); Route::post('port-nac', 'PortNacController'); - Route::post('graylog', 'GraylogController'); Route::post('syslog', 'SyslogController'); });