From 64c4650801b338e85aeb8cccf0df26e5b075de4a Mon Sep 17 00:00:00 2001
From: Andy Norwood <2754635+bonzo81@users.noreply.github.com>
Date: Fri, 21 Jul 2023 17:30:13 +0100
Subject: [PATCH] Add FDB table vendor search drop down (#15072)
* Add vendor search and related functions
* Add OUIDB cache for vendor lookup
* Add vendor drop down
* appy style CI changes
* Apply style CI and lint changes
* more styleCI changes
* update type hinting
* Edit mac_oui cache lock name and function
* Update MAC OUI message during daily
* Use DB for vendor lookup
* New vendor_oui table migration
* New MAC OUI to database function
* Update readbleOUI to use DB rather than cache
* Make StyleCI changes
* styleCI tweak
* Remove lock release to allow refresh timer
* change migration name to match table
* add schema dump
* update schema
* styleCI tweak
---
LibreNMS/Util/Rewrite.php | 19 ++--
.../Controllers/Table/FdbTablesController.php | 40 +++++++
daily.php | 4 +-
daily.sh | 2 +-
...3_06_02_230406_create_vendor_oui_table.php | 32 ++++++
includes/functions.php | 27 +++--
includes/html/pages/search/fdb.inc.php | 8 ++
misc/db_schema.yaml | 105 ++++++++++--------
8 files changed, 167 insertions(+), 70 deletions(-)
mode change 100644 => 100755 app/Http/Controllers/Table/FdbTablesController.php
create mode 100644 database/migrations/2023_06_02_230406_create_vendor_oui_table.php
mode change 100644 => 100755 includes/functions.php
mode change 100644 => 100755 includes/html/pages/search/fdb.inc.php
diff --git a/LibreNMS/Util/Rewrite.php b/LibreNMS/Util/Rewrite.php
index 57623e59fa..1f6ff52d10 100644
--- a/LibreNMS/Util/Rewrite.php
+++ b/LibreNMS/Util/Rewrite.php
@@ -26,7 +26,7 @@
namespace LibreNMS\Util;
use App\Models\Device;
-use Cache;
+use Illuminate\Support\Facades\DB;
use LibreNMS\Config;
class Rewrite
@@ -148,20 +148,23 @@ class Rewrite
}
/**
- * Extract the OUI and match it against cached values
+ * Extract the OUI and match it against database values
*
* @param string $mac
- * @return string
+ * @return string|null
*/
public static function readableOUI($mac)
{
- $cached = Cache::get('OUIDB-' . substr($mac, 0, 6), '');
- if ($cached == 'IEEE Registration Authority') {
- // Then we may have a shorter prefix, so let's try them one ater the other, ordered by probability
- return Cache::get('OUIDB-' . substr($mac, 0, 9)) ?: Cache::get('OUIDB-' . substr($mac, 0, 7));
+ $oui = substr($mac, 0, 6);
+
+ $result = DB::table('vendor_ouis')->where('oui', $oui)->value('vendor');
+
+ if ($result === 'IEEE Registration Authority') {
+ // Then we may have a shorter prefix, so let's try them one after the other, ordered by probability
+ $result = DB::table('vendor_ouis')->whereIn('oui', [substr($mac, 0, 9), substr($mac, 0, 7)])->value('vendor');
}
- return $cached;
+ return $result ?: '';
}
/**
diff --git a/app/Http/Controllers/Table/FdbTablesController.php b/app/Http/Controllers/Table/FdbTablesController.php
old mode 100644
new mode 100755
index 1abda47b30..c7fdd74ad4
--- a/app/Http/Controllers/Table/FdbTablesController.php
+++ b/app/Http/Controllers/Table/FdbTablesController.php
@@ -31,6 +31,7 @@ use App\Models\PortsFdb;
use App\Models\Vlan;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
use LibreNMS\Util\IP;
use LibreNMS\Util\Rewrite;
use LibreNMS\Util\Url;
@@ -91,6 +92,10 @@ class FdbTablesController extends TableController
return $query->whereIn('ports_fdb.mac_address', $this->findMacs($search));
case 'description':
return $query->whereIntegerInRaw('ports_fdb.port_id', $this->findPorts($search));
+ case 'vendor':
+ $vendor_ouis = $this->ouisFromVendor($search);
+
+ return $this->findPortsByOui($vendor_ouis, $query);
default:
return $query->where(function ($query) use ($search, $mac_search) {
$query->where('ports_fdb.mac_address', 'like', $mac_search)
@@ -296,4 +301,39 @@ class FdbTablesController extends TableController
return $this->macCountCache[$port->port_id];
}
+
+ /**
+ * Get the OUI list for a specific vendor
+ *
+ * @param string $vendor
+ * @return array
+ */
+ protected function ouisFromVendor($vendor)
+ {
+ $matching_ouis = DB::table('vendor_ouis')
+ ->where('vendor', 'LIKE', '%' . $vendor . '%')
+ ->pluck('oui')
+ ->toArray();
+
+ return $matching_ouis;
+ }
+
+ /**
+ * Get all port ids from vendor OUIs
+ *
+ * @param array $vendor_ouis
+ * @return Builder
+ */
+ protected function findPortsByOui($vendor_ouis, $query)
+ {
+ $condition = '';
+ foreach ($vendor_ouis as $oui) {
+ $clean_oui = str_replace(':', '', $oui);
+ $condition .= " ports_fdb.mac_address LIKE '$clean_oui%' OR";
+ }
+ $condition = rtrim($condition, ' OR');
+ $query->whereRaw($condition);
+
+ return $query; // Return the query builder instance
+ }
}
diff --git a/daily.php b/daily.php
index 56bd29c327..5b80606775 100644
--- a/daily.php
+++ b/daily.php
@@ -359,9 +359,9 @@ if ($options['f'] === 'peeringdb') {
}
if ($options['f'] === 'mac_oui') {
- $lock = Cache::lock('macouidb', 86000);
+ $lock = Cache::lock('vendor_oui_db', 86000);
if ($lock->get()) {
- $res = cache_mac_oui();
+ $res = mac_oui_to_database();
$lock->release();
exit($res);
}
diff --git a/daily.sh b/daily.sh
index 9db3f6bb11..4503e0a83e 100755
--- a/daily.sh
+++ b/daily.sh
@@ -341,7 +341,7 @@ main () {
# and clean up the db.
status_run 'Updating SQL-Schema' 'php includes/sql-schema/update.php'
status_run 'Cleaning up DB' "'$DAILY_SCRIPT' cleanup"
- status_run 'Caching Mac OUI data' "$DAILY_SCRIPT mac_oui"
+ status_run 'Updating Mac OUI data' "$DAILY_SCRIPT mac_oui"
;;
post-pull)
# re-check dependencies after pull with the new code
diff --git a/database/migrations/2023_06_02_230406_create_vendor_oui_table.php b/database/migrations/2023_06_02_230406_create_vendor_oui_table.php
new file mode 100644
index 0000000000..ea7b288ac3
--- /dev/null
+++ b/database/migrations/2023_06_02_230406_create_vendor_oui_table.php
@@ -0,0 +1,32 @@
+id();
+ $table->string('vendor');
+ $table->string('oui');
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ *
+ * @return void
+ */
+ public function down(): void
+ {
+ Schema::dropIfExists('vendor_ouis');
+ }
+};
diff --git a/includes/functions.php b/includes/functions.php
old mode 100644
new mode 100755
index 9d105a9a9f..7662c757f6
--- a/includes/functions.php
+++ b/includes/functions.php
@@ -1166,13 +1166,12 @@ function q_bridge_bits2indices($hex_data)
/**
* Function to generate Mac OUI Cache
*/
-function cache_mac_oui()
+function mac_oui_to_database()
{
- // timers:
+ // Refresh timer
$mac_oui_refresh_int_min = 86400 * rand(7, 11); // 7 days + a random number between 0 and 4 days
- $mac_oui_cache_time = 1296000; // we keep data during 15 days maximum
- $lock = Cache::lock('macouidb-refresh', $mac_oui_refresh_int_min); //We want to refresh after at least $mac_oui_refresh_int_min
+ $lock = Cache::lock('vendor_oui_db_refresh', $mac_oui_refresh_int_min); // We want to refresh after at least $mac_oui_refresh_int_min
if (Config::get('mac_oui.enabled') !== true) {
echo 'Mac OUI integration disabled' . PHP_EOL;
@@ -1181,7 +1180,7 @@ function cache_mac_oui()
}
if ($lock->get()) {
- echo 'Caching Mac OUI' . PHP_EOL;
+ echo 'Storing Mac OUI in the database' . PHP_EOL;
try {
$mac_oui_url = 'https://gitlab.com/wireshark/wireshark/-/raw/master/manuf';
//$mac_oui_url_mirror = 'https://raw.githubusercontent.com/wireshark/wireshark/master/manuf';
@@ -1190,12 +1189,15 @@ function cache_mac_oui()
$get = \LibreNMS\Util\Http::client()->get($mac_oui_url);
echo ' -> Processing CSV ...' . PHP_EOL;
$csv_data = $get->body();
+
+ // Process each line of the CSV data
foreach (explode("\n", $csv_data) as $csv_line) {
unset($oui);
$entry = str_getcsv($csv_line, "\t");
$length = strlen($entry[0]);
$prefix = strtolower(str_replace(':', '', $entry[0]));
+ $vendor = $entry[2];
if (is_array($entry) && count($entry) >= 3 && $length == 8) {
// We have a standard OUI xx:xx:xx
@@ -1208,18 +1210,23 @@ function cache_mac_oui()
$oui = substr($prefix, 0, 9);
}
}
+
if (isset($oui)) {
- echo "Adding $oui, $entry[2]" . PHP_EOL;
- $key = 'OUIDB-' . $oui;
- Cache::put($key, $entry[2], $mac_oui_cache_time);
+ // Store the OUI for the vendor in the database
+ DB::table('vendor_ouis')->insert([
+ 'vendor' => $vendor,
+ 'oui' => $oui,
+ ]);
+
+ echo "Adding $oui for $vendor" . PHP_EOL;
}
}
} catch (Exception $e) {
- echo 'Error processing Mac OUI :' . PHP_EOL;
+ echo 'Error processing Mac OUI:' . PHP_EOL;
echo 'Exception: ' . get_class($e) . PHP_EOL;
echo $e->getMessage() . PHP_EOL;
- $lock->release(); // we did not succeed so we'll try again next time
+ $lock->release(); // We did not succeed, so we'll try again next time
return 1;
}
diff --git a/includes/html/pages/search/fdb.inc.php b/includes/html/pages/search/fdb.inc.php
old mode 100644
new mode 100755
index 6cbbef0eb1..d303dc52f4
--- a/includes/html/pages/search/fdb.inc.php
+++ b/includes/html/pages/search/fdb.inc.php
@@ -95,6 +95,14 @@ if ($vars['searchby'] == 'description') {
?>
">Description"+
+ ""+
"