Rewrite agent packages parsing code (#13840)

* Rewrite agent packages parsing code
but less crazy

* fixes

* remove commented code

* alphabetical

* Less code duplication

* fix style

app/Models/Package.php

@@ -25,8 +25,29 @@
 
 namespace App\Models;
 
-class Package extends DeviceRelatedModel
+use LibreNMS\Interfaces\Models\Keyable;
+
+class Package extends DeviceRelatedModel implements Keyable
 {
     public $timestamps = false;
     protected $primaryKey = 'pkg_id';
+    protected $fillable = [
+        'name',
+        'manager',
+        'status',
+        'version',
+        'build',
+        'arch',
+        'size',
+    ];
+
+    public function getCompositeKey()
+    {
+        return "$this->manager-$this->name-$this->arch";
+    }
+
+    public function __toString()
+    {
+        return $this->name . ' (' . $this->arch . ') version ' . $this->version . ($this->build ? "-$this->build" : '');
+    }
 }

app/Observers/PackageObserver.php

@@ -0,0 +1,70 @@
+<?php
+
+namespace App\Observers;
+
+use App\Models\Package;
+use Log;
+
+class PackageObserver
+{
+    /**
+     * Handle the Package "created" event.
+     *
+     * @param  \App\Models\Package  $package
+     * @return void
+     */
+    public function created(Package $package)
+    {
+        Log::event('Package installed: ' . $package, $package->device_id, 'package', 3);
+        Log::info("+ $package");
+    }
+
+    /**
+     * Handle the Package "updated" event.
+     *
+     * @param  \App\Models\Package  $package
+     * @return void
+     */
+    public function updated(Package $package)
+    {
+        if ($package->getOriginal('version') !== $package->version || $package->getOriginal('build') !== $package->build) {
+            $message = $package . ' from ' . $package->getOriginal('version') . ($package->getOriginal('build') ? '-' . $package->getOriginal('build') : '');
+            Log::event('Package updated: ' . $message, $package->device_id, 'package', 3);
+            Log::info("u $message");
+        }
+    }
+
+    /**
+     * Handle the Package "deleted" event.
+     *
+     * @param  \App\Models\Package  $package
+     * @return void
+     */
+    public function deleted(Package $package)
+    {
+        Log::event('Package removed: ' . $package, $package->device_id, 'package', 3);
+        Log::info("- $package");
+    }
+
+    /**
+     * Handle the Package "restored" event.
+     *
+     * @param  \App\Models\Package  $package
+     * @return void
+     */
+    public function restored(Package $package)
+    {
+        //
+    }
+
+    /**
+     * Handle the Package "force deleted" event.
+     *
+     * @param  \App\Models\Package  $package
+     * @return void
+     */
+    public function forceDeleted(Package $package)
+    {
+        //
+    }
+}

app/Providers/AppServiceProvider.php

@@ -132,9 +132,10 @@ class AppServiceProvider extends ServiceProvider
     private function bootObservers()
     {
         \App\Models\Device::observe(\App\Observers\DeviceObserver::class);
+        \App\Models\Package::observe(\App\Observers\PackageObserver::class);
         \App\Models\Service::observe(\App\Observers\ServiceObserver::class);
-        \App\Models\User::observe(\App\Observers\UserObserver::class);
         \App\Models\Stp::observe(\App\Observers\StpObserver::class);
+        \App\Models\User::observe(\App\Observers\UserObserver::class);
     }
 
     private function bootCustomValidators()

includes/polling/unix-agent/packages.inc.php

@@ -1,162 +1,77 @@
 <?php
 
+use App\Models\Package;
+
 $pkgs_id = [];
 $pkgs_db_id = [];
 
-// RPM
-if (! empty($agent_data['rpm'])) {
-    echo "\nRPM Packages: ";
-    // Build array of existing packages
-    $manager = 'rpm';
+$managers = [
+    'rpm' => [
+        'name' => 'RPM',
+        'process' => function ($line) {
+            [$name, $version, $build, $arch, $size] = explode(' ', $line);
 
-    $pkgs_db_db = dbFetchRows('SELECT * FROM `packages` WHERE `device_id` = ?', [$device['device_id']]);
-    foreach ($pkgs_db_db as $pkg_db) {
-        $pkgs_db[$pkg_db['manager']][$pkg_db['name']][$pkg_db['arch']][$pkg_db['version']][$pkg_db['build']]['id'] = $pkg_db['pkg_id'];
-        $pkgs_db[$pkg_db['manager']][$pkg_db['name']][$pkg_db['arch']][$pkg_db['version']][$pkg_db['build']]['status'] = $pkg_db['status'];
-        $pkgs_db[$pkg_db['manager']][$pkg_db['name']][$pkg_db['arch']][$pkg_db['version']][$pkg_db['build']]['size'] = $pkg_db['size'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['text'] = $pkg_db['manager'] . '-' . $pkg_db['name'] . '-' . $pkg_db['arch'] . '-' . $pkg_db['version'] . '-' . $pkg_db['build'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['manager'] = $pkg_db['manager'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['name'] = $pkg_db['name'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['arch'] = $pkg_db['arch'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['version'] = $pkg_db['version'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['build'] = $pkg_db['build'];
-    }
-
-    foreach (explode("\n", $agent_data['rpm']) as $package) {
-        [$name, $version, $build, $arch, $size] = explode(' ', $package);
-        $pkgs[$manager][$name][$arch][$version][$build]['manager'] = $manager;
-        $pkgs[$manager][$name][$arch][$version][$build]['name'] = $name;
-        $pkgs[$manager][$name][$arch][$version][$build]['arch'] = $arch;
-        $pkgs[$manager][$name][$arch][$version][$build]['version'] = $version;
-        $pkgs[$manager][$name][$arch][$version][$build]['build'] = $build;
-        $pkgs[$manager][$name][$arch][$version][$build]['size'] = $size;
-        $pkgs[$manager][$name][$arch][$version][$build]['status'] = '1';
-        $text = $manager . '-' . $name . '-' . $arch . '-' . $version . '-' . $build;
-        $pkgs_id[] = $pkgs[$manager][$name][$arch][$version][$build];
-    }
-}//end if
-
-// DPKG
-if (! empty($agent_data['dpkg'])) {
-    echo "\nDEB Packages: ";
-    // Build array of existing packages
-    $manager = 'deb';
-
-    $pkgs_db_db = dbFetchRows('SELECT * FROM `packages` WHERE `device_id` = ?', [$device['device_id']]);
-    foreach ($pkgs_db_db as $pkg_db) {
-        $pkgs_db[$pkg_db['manager']][$pkg_db['name']][$pkg_db['arch']][$pkg_db['version']][$pkg_db['build']]['id'] = $pkg_db['pkg_id'];
-        $pkgs_db[$pkg_db['manager']][$pkg_db['name']][$pkg_db['arch']][$pkg_db['version']][$pkg_db['build']]['status'] = $pkg_db['status'];
-        $pkgs_db[$pkg_db['manager']][$pkg_db['name']][$pkg_db['arch']][$pkg_db['version']][$pkg_db['build']]['size'] = $pkg_db['size'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['text'] = $pkg_db['manager'] . '-' . $pkg_db['name'] . '-' . $pkg_db['arch'] . '-' . $pkg_db['version'] . '-' . $pkg_db['build'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['manager'] = $pkg_db['manager'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['name'] = $pkg_db['name'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['arch'] = $pkg_db['arch'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['version'] = $pkg_db['version'];
-        $pkgs_db_id[$pkg_db['pkg_id']]['build'] = $pkg_db['build'];
-    }
-
-    foreach (explode("\n", $agent_data['dpkg']) as $package) {
-        [$name, $version, $arch, $size] = explode(' ', $package);
-        $build = '';
-        $pkgs[$manager][$name][$arch][$version][$build]['manager'] = $manager;
-        $pkgs[$manager][$name][$arch][$version][$build]['name'] = $name;
-        $pkgs[$manager][$name][$arch][$version][$build]['arch'] = $arch;
-        $pkgs[$manager][$name][$arch][$version][$build]['version'] = $version;
-        $pkgs[$manager][$name][$arch][$version][$build]['build'] = $build;
-        $pkgs[$manager][$name][$arch][$version][$build]['size'] = (cast_number($size) * 1024);
-        $pkgs[$manager][$name][$arch][$version][$build]['status'] = '1';
-        $text = $manager . '-' . $name . '-' . $arch . '-' . $version . '-' . $build;
-        $pkgs_id[] = $pkgs[$manager][$name][$arch][$version][$build];
-    }
-}//end if
-
-// This is run for all "packages" and is common to RPM/DEB/etc
-foreach ($pkgs_id as $pkg) {
-    $name = $pkg['name'];
-    $version = $pkg['version'];
-    $build = $pkg['build'];
-    $arch = $pkg['arch'];
-    $size = $pkg['size'];
-
-    // echo(str_pad($name, 20)." ".str_pad($version, 10)." ".str_pad($build, 10)." ".$arch."\n");
-    // echo($name." ");
-    if (is_array($pkgs_db[$pkg['manager']][$pkg['name']][$pkg['arch']][$pkg['version']][$pkg['build']])) {
-        // FIXME - packages_history table
-        $id = $pkgs_db[$pkg['manager']][$pkg['name']][$pkg['arch']][$pkg['version']][$pkg['build']]['id'];
-        if ($pkgs_db[$pkg['manager']][$pkg['name']][$pkg['arch']][$pkg['version']][$pkg['build']]['status'] != '1') {
-            $pkg_update['status'] = '1';
-        }
-
-        if ($pkgs_db[$pkg['manager']][$pkg['name']][$pkg['arch']][$pkg['version']][$pkg['build']]['size'] != $size) {
-            $pkg_update['size'] = $size;
-        }
-
-        if (! empty($pkg_update)) {
-            dbUpdate($pkg_update, 'packages', '`pkg_id` = ?', [$id]);
-            echo 'u';
-        } else {
-            echo '.';
-        }
-
-        unset($pkgs_db_id[$id]);
-    } else {
-        if (count($pkgs[$manager][$name][$arch], 1) > '10' || (is_countable($pkgs_db[$manager][$name][$arch]) && count($pkgs_db[$manager][$name][$arch], 1) == '0')) {
-            dbInsert(
-                [
-                    'device_id' => $device['device_id'],
-                    'name'      => $name,
-                    'manager'   => $manager,
-                    'status'    => 1,
-                    'version'   => $version,
-                    'build'     => $build,
-                    'arch'      => $arch,
-                    'size'      => $size,
-                ],
-                'packages'
-            );
-            if ($build != '') {
-                $dbuild = '-' . $build;
-            } else {
-                $dbuild = '';
-            }
-
-            echo '+' . $name . '-' . $version . $dbuild . '-' . $arch;
-            log_event('Package installed: ' . $name . ' (' . $arch . ') version ' . $version . $dbuild, $device, 'package', 3);
-        } elseif (is_countable($pkgs_db[$manager][$name][$arch]) && count($pkgs_db[$manager][$name][$arch], 1)) {
-            $pkg_c = dbFetchRow('SELECT * FROM `packages` WHERE `device_id` = ? AND `manager` = ? AND `name` = ? and `arch` = ? ORDER BY version DESC, build DESC', [$device['device_id'], $manager, $name, $arch]);
-            if ($pkg_c['build'] != '') {
-                $pkg_c_dbuild = '-' . $pkg_c['build'];
-            } else {
-                $pkg_c_dbuild = '';
-            }
-
-            echo 'U(' . $pkg_c['name'] . '-' . $pkg_c['version'] . $pkg_c_dbuild . '|' . $name . '-' . $version . $dbuild . ')';
-            $pkg_update = [
+            return new Package([
+                'manager' => 'rpm',
+                'name' => $name,
+                'arch' => $arch,
                 'version' => $version,
-                'build'   => $build,
-                'status'  => '1',
-                'size'    => $size,
-            ];
-            dbUpdate($pkg_update, 'packages', '`pkg_id` = ?', [$pkg_c['pkg_id']]);
-            log_event('Package updated: ' . $name . ' (' . $arch . ') from ' . $pkg_c['version'] . $pkg_c_dbuild . ' to ' . $version . $dbuild, $device, 'package', 3);
-            unset($pkgs_db_id[$pkg_c['pkg_id']]);
-        }//end if
-    }//end if
-    unset($pkg_update);
-}//end foreach
+                'build' => $build,
+                'size' => $size,
+                'status' => 1,
+            ]);
+        },
+    ],
+    'dpkg' => [
+        'name' => 'DEB',
+        'process' => function ($line) {
+            [$name, $version, $arch, $size] = explode(' ', $line);
 
-// Packages
-foreach ($pkgs_db_id as $id => $pkg) {
-    dbDelete('packages', '`pkg_id` =  ?', [$id]);
-    echo '-' . $pkg['text'];
-    log_event('Package removed: ' . $pkg['name'] . ' ' . $pkg['arch'] . ' ' . $pkg['version'] . ($pkg['build'] != '' ? '-' . $pkg['build'] : ''), $device, 'package', 3);
+            return new Package([
+                'manager' => 'deb',
+                'name' => $name,
+                'arch' => $arch,
+                'version' => $version,
+                'build' => '',
+                'size' => cast_number($size) * 1024,
+                'status' => 1,
+            ]);
+        },
+    ],
+];
+
+foreach ($managers as $key => $manager) {
+    if (! empty($agent_data[$key])) {
+        echo "\n{$manager['name']} Packages: \n";
+
+        /** @var \Illuminate\Support\Collection $packages */
+        $packages = DeviceCache::getPrimary()->packages->map(function (Package $package) {
+            $package->status = 0;
+
+            return $package;
+        })->keyBy->getCompositeKey();
+
+        foreach (explode("\n", trim($agent_data[$key])) as $line) {
+            /** @var \App\Models\Package $package */
+            $package = $manager['process']($line);
+            $package_key = $package->getCompositeKey();
+            if ($existing_package = $packages->get($package_key)) {
+                $existing_package->fill($package->attributesToArray());
+            } else {
+                $packages->put($package_key, $package);
+            }
+        }
+
+        break;
+    }
+}
+
+// update the database
+if (isset($packages)) {
+    DeviceCache::getPrimary()->packages()->saveMany($packages->where('status', 1));
+    $packages->where('status', 0)->each->delete();
 }
 
 echo "\n";
 
-unset($pkg);
-unset($pkgs_db_id);
-unset($pkg_c);
-unset($pkgs);
-unset($pkgs_db);
-unset($pkgs_db_db);
+unset($packages, $existing_package, $package, $managers);