feature: Improved install experience (#6915)

always check that the base sql has been imported.
async db building
After this merge, we could move install.php to the end of the install documentation.  Makes a more natural flow.

build-base.php

@@ -1,53 +1,54 @@
 #!/usr/bin/env php
 <?php
+/**
+ * build-base.php
+ *
+ * Create database structure.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2017 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
 
 if (!isset($init_modules)) {
+    $opts = getopt('dh:u:p:n:t:s:');
+
+    $db_vars = array(
+        'db_host' => 'h',
+        'db_user' => 'u',
+        'db_pass' => 'p',
+        'db_name' => 'n',
+        'db_port' => 't',
+        'db_socket' => 's',
+    );
+
+    $config = array();
+    foreach ($db_vars as $setting => $opt) {
+        if (isset($opts[$opt])) {
+            $config[$setting] = $opts[$opt];
+        }
+    }
+
     $init_modules = array();
     require __DIR__  . '/includes/init.php';
 
-    $options = getopt('d');
+    $debug = isset($opts['d']);
-    $debug = isset($options['d']);
 }
 
-if (!isset($sql_file)) {
-    $sql_file = 'build.sql';
-}
-
-$sql_fh = fopen($sql_file, 'r');
-if ($sql_fh === false) {
-    echo 'ERROR: Cannot open SQL build script '.$sql_file.PHP_EOL;
-    exit(1);
-}
-
-// only import build.sql to an empty database
-$tables = dbFetchRows("SHOW TABLES FROM {$config['db_name']}");
-if (empty($tables)) {
-    $limit = 0;
-
-    while (!feof($sql_fh)) {
-        $line = fgetss($sql_fh);
-        if (isset($_SESSION['stage'])) {
-            $limit++;
-            if (isset($_SESSION['offset']) && $limit < $_REQUEST['offset']) {
-                continue;
-            } elseif (time()-$_SESSION['last'] > 45) {
-                $_SESSION['offset'] = $limit;
-                $GLOBALS['refresh'] = '<b>Installing, please wait..</b><sub>'.date('r').'</sub><script>window.location.href = "install.php?offset='.$limit.'";</script>';
-                return;
-            } else {
-                echo 'Step #'.$limit.' ...'.PHP_EOL;
-            }
-        }
-
-        if (!empty($line)) {
-            $creation = dbQuery($line);
-            if (!$creation) {
-                echo 'WARNING: Cannot execute query ('.$line.'): '.mysqli_error($database_link)."\n";
-            }
-        }
-    }
-}
-
-fclose($sql_fh);
-
 require 'includes/sql-schema/update.php';
+
+exit($return);

html/ajax_output.php

@@ -12,16 +12,23 @@
  * the source code distribution for details.
  */
 
-$init_modules = array('web', 'auth', 'alerts');
+session_start();
-require realpath(__DIR__ . '/..') . '/includes/init.php';
+if (isset($_SESSION['stage']) && $_SESSION['stage'] == 2) {
+    $_SESSION['build-ok'] = true;
+    $init_modules = array('web', 'nodb');
+    require realpath(__DIR__ . '/..') . '/includes/init.php';
+} else {
+    $init_modules = array('web', 'auth', 'alerts');
+    require realpath(__DIR__ . '/..') . '/includes/init.php';
 
-if (!$_SESSION['authenticated']) {
+    if (!$_SESSION['authenticated']) {
-    echo "Unauthenticated\n";
+        echo "Unauthenticated\n";
-    exit;
+        exit;
+    }
 }
 
 set_debug($_REQUEST['debug']);
-$id = mres($_REQUEST['id']);
+$id = str_replace('/', '', $_REQUEST['id']);
 
 if (isset($id)) {
     require $config['install_dir'] . "/html/includes/output/$id.inc.php";

html/includes/output/db-update.inc.php

@@ -0,0 +1,67 @@
+<?php
+/**
+ * db-update.inc.php
+ *
+ * Run database update/deploy for installer
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @package    LibreNMS
+ * @link       http://librenms.org
+ * @copyright  2017 Tony Murray
+ * @author     Tony Murray <murraytony@gmail.com>
+ */
+
+if (file_exists($config['install_dir'] . '/config.php')) {
+    echo("This should only be called during install");
+    exit;
+}
+
+header("Content-type: text/plain");
+header('X-Accel-Buffering: no');
+
+$db_vars = array(
+    'dbhost' => 'h',
+    'dbuser' => 'u',
+    'dbpass' => 'p',
+    'dbname' => 'n',
+    'dbport' => 't',
+    'dbsocket' => 's',
+);
+
+$cmd = $config['install_dir'] . '/build-base.php';
+
+foreach ($db_vars as $var => $opt) {
+    if ($_SESSION[$var]) {
+        $cmd .= " -$opt {$_SESSION[$var]}";
+    }
+}
+
+echo "Starting Update...\n";
+
+if (($fp = popen($cmd . ' 2>&1', "r"))) {
+    while (!feof($fp)) {
+        $line = stream_get_line($fp, 1024, "\n");
+        echo preg_replace('/\033\[[\d;]+m/', '', $line) . PHP_EOL;
+        ob_flush();
+        flush(); // you have to flush the buffer
+    }
+
+    if (pclose($fp) === 0) {
+        echo "Database is up to date!";
+        $_SESSION['build-ok'] = true;
+    }
+}
+
+session_write_close();

html/install.php

@@ -3,7 +3,7 @@ session_start();
 if (empty($_POST) && !empty($_SESSION) && !isset($_REQUEST['stage'])) {
     $_POST = $_SESSION;
 } else {
-    $_SESSION = $_POST;
+    $_SESSION = array_replace($_SESSION, $_POST);
 }
 
 $stage = isset($_POST['stage']) ? $_POST['stage'] : 0;
@@ -64,6 +64,8 @@ if ($stage > 1) {
     $_SESSION['stage'] = $stage;
 }
 
+session_write_close();
+
 if ($stage == 4) {
     // Now check we have a username, password and email before adding new user
     if (empty($add_user) || empty($add_pass) || empty($add_email)) {
@@ -89,7 +91,6 @@ $stage_perc = $stage / $total_stages * 100;
 $complete = 1;
 
 ?>
-
 <!DOCTYPE HTML>
 <html>
 <head>
@@ -100,11 +101,9 @@ $complete = 1;
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <link href="css/bootstrap.min.css" rel="stylesheet" type="text/css" />
   <link href="<?php echo($config['stylesheet']);  ?>" rel="stylesheet" type="text/css" />
-  <link href="css/typeahead.js-bootstrap.css" rel="stylesheet" type="text/css" />
   <script src="js/jquery.min.js"></script>
   <script src="js/bootstrap.min.js"></script>
   <script src="js/bootstrap-hover-dropdown.min.js"></script>
-  <script src="js/typeahead.min.js"></script>
   <script src="js/hogan-2.0.0.js"></script>
 
 </head>
@@ -309,29 +308,7 @@ if ($ext_loaded == 'no') {
      </div>
      <div class="col-md-6">
          <h5 class="text-center">Importing MySQL DB - Do not close this page or interrupt the import</h5>
-<?php
+        <textarea readonly id="db-update" class="form-control" rows="20" placeholder="Please Wait..." style="resize:vertical;"></textarea>
-// Ok now let's set the db connection up
-    $config['db_host'] = $dbhost;
-    $config['db_user'] = $dbuser;
-    $config['db_pass'] = $dbpass;
-    $config['db_name'] = $dbname;
-    $config['db_port'] = $dbport;
-    $config['db_socket'] = $dbsocket;
-    $sql_file = '../build.sql';
-    $_SESSION['last'] = time();
-    ob_end_flush();
-    ob_start();
-if ($_SESSION['offset'] < 100 && $_REQUEST['offset'] < 94) {
-    require '../build-base.php';
-} else {
-    require '../includes/sql-schema/update.php';
-}
-    $_SESSION['out'] .= ob_get_clean();
-    ob_end_clean();
-    ob_start();
-    echo $GLOBALS['refresh'];
-    echo "<pre>".trim($_SESSION['out'])."</pre>";
-?>
      </div>
      <div class="col-md-3">
      </div>
@@ -349,12 +326,28 @@ if ($_SESSION['offset'] < 100 && $_REQUEST['offset'] < 94) {
           <input type="hidden" name="dbname" value="<?php echo $dbname; ?>">
           <input type="hidden" name="dbport" value="<?php echo $dbport; ?>">
           <input type="hidden" name="dbsocket" value="<?php echo $dbsocket; ?>">
-          <button type="submit" class="btn btn-success">Goto Add User</button>
+          <button type="submit" id="add-user-btn" class="btn btn-success" disabled>Goto Add User</button>
         </form>
       </div>
       <div class="col-md-3">
       </div>
     </div>
+    <script type="text/javascript">
+        output = document.getElementById("db-update");
+        xhr = new XMLHttpRequest();
+        xhr.open("GET", "ajax_output.php?id=db-update", true);
+        xhr.onprogress = function (e) {
+            output.innerHTML = e.currentTarget.responseText;
+            output.scrollTop = output.scrollHeight - output.clientHeight; // scrolls the output area
+        };
+        xhr.onreadystatechange = function () {
+            if (xhr.readyState === 4) {
+                console.log("Complete");
+                document.getElementById("add-user-btn").removeAttribute('disabled');
+            }
+        };
+        xhr.send();
+    </script>
 <?php
 } elseif ($stage == "5") {
 ?>

includes/dbFacile.php

@@ -55,7 +55,7 @@ function dbConnect($host = null, $user = '', $password = '', $database = '', $po
     if (!$database_db) {
         $db_create_sql = "CREATE DATABASE " . $config['db_name'] . " CHARACTER SET utf8 COLLATE utf8_unicode_ci";
         mysqli_query($database_link, $db_create_sql);
-        $database_db = mysqli_select_db($database_link, $config['db_name']);
+        $database_db = mysqli_select_db($database_link, $database);
     }
 
     if (!$database_db) {

includes/sql-schema/update.php

@@ -15,85 +15,98 @@
  * See COPYING for more details.
  */
 
-if (!isset($debug)  && php_sapi_name() == 'cli') {
+if (!isset($init_modules)  && php_sapi_name() == 'cli') {
     // Not called from within discovery, let's load up the necessary stuff.
     $init_modules = array();
     require realpath(__DIR__ . '/../..') . '/includes/init.php';
-
-    $options = getopt('d');
-    $debug = isset($options['d']);
 }
 
 set_lock('schema');
 
+$return = 0;
+
+// only import build.sql to an empty database
+$tables = dbFetchRows("SHOW TABLES FROM {$config['db_name']}");
+if (empty($tables)) {
+    echo "-- Creating base database structure\n";
+    $step = 0;
+    $sql_fh = fopen('build.sql', 'r');
+    if ($sql_fh === false) {
+        echo 'ERROR: Cannot open SQL build script '.$sql_file.PHP_EOL;
+        $return = 1;
+    }
+
+    while (!feof($sql_fh)) {
+        $line = fgetss($sql_fh);
+        echo 'Step #'.$step++.' ...'.PHP_EOL;
+
+        if (!empty($line)) {
+            $creation = dbQuery($line);
+            if (!$creation) {
+                echo 'WARNING: Cannot execute query ('.$line.'): '.mysqli_error($database_link)."\n";
+                $return = 1;
+            }
+        }
+    }
+
+    fclose($sql_fh);
+}
+
+
 d_echo("DB Schema update started....\n");
 
 if (db_schema_is_current()) {
     d_echo("DB Schema already up to date.\n");
-    release_lock('schema');
+} else {
-    return;
+    // Set Database Character set and Collation
-}
+    dbQuery('ALTER DATABASE ? CHARACTER SET utf8 COLLATE utf8_unicode_ci;', array(array($config['db_name'])));
 
-// Set Database Character set and Collation
+    $db_rev = get_db_schema();
-dbQuery('ALTER DATABASE ? CHARACTER SET utf8 COLLATE utf8_unicode_ci;', array(array($config['db_name'])));
+    $insert = ($db_rev == 0); // if $db_rev == 0, insert the first update
 
-$db_rev = get_db_schema();
+    $updating = 0;
-$insert = ($db_rev == 0); // if $db_rev == 0, insert the first update
+    foreach (get_schema_list() as $file_rev => $file) {
-
+        if ($file_rev > $db_rev) {
-$updating = 0;
+            if (!$updating) {
-$limit = 150; //magic marker far enough in the future
+                echo "-- Updating database schema\n";
-foreach (get_schema_list() as $file_rev => $file) {
-    if ($file_rev > $db_rev) {
-        if (isset($_SESSION['stage'])) {
-            $limit++;
-            if (time()-$_SESSION['last'] > 45) {
-                $_SESSION['offset'] = $limit;
-                $GLOBALS['refresh'] = '<b>Updating, please wait..</b><sub>'.date('r').'</sub><script>window.location.href = "install.php?offset='.$limit.'";</script>';
-                return;
             }
-        }
 
-        if (!$updating) {
+            printf('%03d -> %03d ...', $db_rev, $file_rev);
-            echo "-- Updating database schema\n";
-        }
 
-        printf('%03d -> %03d ...', $db_rev, $file_rev);
+            $err = 0;
+            if ($data = file_get_contents($file)) {
+                foreach (explode("\n", $data) as $line) {
+                    if (trim($line)) {
+                        d_echo("$line \n");
 
-        $err = 0;
+                        if ($line[0] != '#') {
-        if ($data = file_get_contents($file)) {
+                            if (!mysqli_query($database_link, $line)) {
-            foreach (explode("\n", $data) as $line) {
+                                $err++;
-                if (trim($line)) {
+                                d_echo(mysqli_error($database_link) . PHP_EOL);
-                    d_echo("$line \n");
+                            }
-
-                    if ($line[0] != '#') {
-                        if (!mysqli_query($database_link, $line)) {
-                            $err++;
-                            d_echo(mysqli_error($database_link) . PHP_EOL);
                         }
                     }
                 }
+
+                echo " done ($err errors).\n";
+            } else {
+                echo " Could not open file! $file\n";
+                $return = 1;
+            }//end if
+
+            $updating++;
+            $db_rev = $file_rev;
+            if ($insert) {
+                dbInsert(array('version' => $db_rev), 'dbSchema');
+                $insert = false;
+            } else {
+                dbUpdate(array('version' => $db_rev), 'dbSchema');
             }
-
-            echo " done ($err errors).\n";
-        } else {
-            echo " Could not open file! $file\n";
         }//end if
+    }//end foreach
 
-        $updating++;
+    if ($updating) {
-        $db_rev = $file_rev;
+        echo "-- Done\n";
-        if ($insert) {
-            dbInsert(array('version' => $db_rev), 'dbSchema');
-            $insert = false;
-        } else {
-            dbUpdate(array('version' => $db_rev), 'dbSchema');
-        }
-    }//end if
-}//end foreach
-
-if ($updating) {
-    echo "-- Done\n";
-    if (isset($_SESSION['stage'])) {
-        $_SESSION['build-ok'] = true;
     }
 }
 

tests/bootstrap.php

@@ -56,11 +56,7 @@ if (getenv('DBTEST')) {
     $empty_db = (dbFetchCell("SELECT count(*) FROM `information_schema`.`tables` WHERE `table_type` = 'BASE TABLE' AND `table_schema` = ?", array($config['db_name'])) == 0);
     dbQuery("SET GLOBAL sql_mode='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'");
 
-    if ($empty_db) {
+    $cmd = $config['install_dir'] . '/build-base.php';
-        $cmd = $config['install_dir'] . '/build-base.php';
-    } else {
-        $cmd = '/usr/bin/env php ' . $config['install_dir'] . '/includes/sql-schema/update.php';
-    }
     exec($cmd, $schema);
 
     register_shutdown_function(function () use ($empty_db, $sql_mode) {