GitHub Actions dev:check ci (#12392)

* Refactor test for GitHub Actions

* Checkout pull request HEAD commit instead of merge commit

* Fetch changed files from API instead of git

* HEAD commit

* Origin

* Fix env

* force full unit checks on gh actions change

* Fix DB conn

* Update test.yml

* Also set port in PDO connection

* Exit with correct exit code

* Update CiHelper.php

* Update test.yml

* Update CiHelper.php

* Update Proc.php

* Cleanups

Remove extra unit test run on php 7.4
Add names that will hopefully show in github.
Remove redundant DB env variables.

* Fix

Shorter names
and DB_TEST env is required

* Change command for information purposes

Co-authored-by: CrazyMax <crazy-max@users.noreply.github.com>
Co-authored-by: Tony Murray <murraytony@gmail.com>
Co-authored-by: Jellyfrog <Jellyfrog@users.noreply.github.com>

.github/workflows/doc.yml

@@ -45,6 +45,7 @@ jobs:
         uses: crazy-max/ghaction-github-pages@v2
         with:
           target_branch: gh-pages
+          keep_history: true
           build_dir: out
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,192 @@
+name: test
+
+on:
+  push:
+    branches:
+      - 'master'
+    paths-ignore:
+      - '**.md'
+      - '.github/workflows/doc.yml'
+      - 'doc/**'
+      - 'mkdocs.yml'
+  pull_request:
+    branches:
+      - 'master'
+    paths-ignore:
+      - '**.md'
+      - '.github/workflows/doc.yml'
+      - 'doc/**'
+      - 'mkdocs.yml'
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          -
+            php-version: 7.4
+            name: Unit
+            skip-style-check: 1
+            skip-web-check: 1
+          -
+            php-version: 7.4
+            name: Style and Web
+            skip-unit-check: 1
+          -
+            php-version: 7.3
+            name: Unit
+            skip-style-check: 1
+            skip-web-check: 1
+          -
+            php-version: 7.3
+            name: Style and Web
+            skip-unit-check: 1
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+          MYSQL_DATABASE: librenms_phpunit_78hunjuybybh
+          MYSQL_USER: librenms
+          MYSQL_PASSWORD: librenms
+        ports:
+          - 3306
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 3
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      -
+         name: Set $FILES
+         run: |
+           FILES=$(git diff --diff-filter=d --name-only origin/master | tr '\n' ' '|sed 's/,*$//g')
+           echo $FILES
+           
+           echo 'FILES<<EOF' >> $GITHUB_ENV
+           echo $FILES >> $GITHUB_ENV
+           echo 'EOF' >> $GITHUB_ENV  
+      -
+         name: Set $PATH
+         run: |
+           echo "/home/runner/.local/bin/" >> $GITHUB_PATH
+      -
+        name: set MySQL collate
+        env:
+          PORT: ${{ job.services.mysql.ports[3306] }}
+        run: | 
+          mysql -h"127.0.0.1" -P"$PORT" --user=librenms --password=librenms -e 'ALTER DATABASE librenms_phpunit_78hunjuybybh CHARACTER SET utf8 COLLATE utf8_unicode_ci;'
+      -
+        name: Set up PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php-version }}
+          tools: composer,php-cs-fixer
+      -
+        name: Get composer cache directory
+        id: composer-cache
+        run: |
+          echo "::set-output name=dir::$(composer config cache-files-dir)"
+      -
+        name: Cache composer
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-
+      -
+        name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      -
+        name: Install packages
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get install -y fping python3-pip python3-setuptools snmp
+      -
+        name: Pip install
+        run: |
+          pip3 install --upgrade pip
+          pip3 install --user snmpsim pylint python-memcached mysqlclient --upgrade
+      -
+        name: Composer validate
+        run: |
+          composer validate
+      -
+        name: Init env
+        run: |
+          cat > .env <<EOL
+          APP_KEY=base64:vHI+YHgkyCDad31iPEErGSNEOWO21wNzV+zyENKQv04=
+          APP_URL=http://127.0.0.1:8000
+          APP_ENV=testing
+          DB_DRIVER=mysql
+          DB_HOST=127.0.0.1
+          DB_PORT=${{ job.services.mysql.ports[3306] }}
+          DB_DATABASE=librenms_phpunit_78hunjuybybh
+          DB_USERNAME=librenms
+          DB_PASSWORD=librenms
+          DB_TEST_DRIVER=mysql
+          DB_TEST_HOST=127.0.0.1
+          DB_TEST_PORT=${{ job.services.mysql.ports[3306] }}
+          DB_TEST_DATABASE=librenms_phpunit_78hunjuybybh
+          DB_TEST_USERNAME=librenms
+          DB_TEST_PASSWORD=librenms
+          EOL
+      -
+        name: Composer install
+        run: |
+          composer install --prefer-dist --no-interaction --no-progress --no-suggest
+      -
+        name: Artisan dusk:update
+        if: matrix.skip-web-check != '1'
+        run: |
+          php artisan dusk:update --detect
+      -
+        name: Artisan serve
+        if: matrix.skip-web-check != '1'
+        run: |
+          php artisan serve --env=dusk.testing >/tmp/artisan-serve.log 2>&1 &
+          sleep 5
+      -
+        name: Start SNMP
+        if: matrix.skip-unit-check != '1'
+        run: |
+          ~/.local/bin/snmpsimd.py --data-dir=tests/snmpsim --agent-udpv4-endpoint=127.1.6.2:1162 --logging-method=file:/tmp/snmpsimd.log &
+      -
+        name: lnms dev:check ci
+        run: |
+          php lnms dev:check ci
+        env:
+          SKIP_STYLE_CHECK: ${{ matrix.skip-style-check }}
+          SKIP_UNIT_CHECK: ${{ matrix.skip-unit-check }}
+          SKIP_WEB_CHECK: ${{ matrix.skip-web-check }}
+      -
+        name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+      -
+        name: Dump serve logs
+        if: always() && matrix.skip-web-check != '1'
+        run: |
+          cat /tmp/artisan-serve.log
+      -
+        name: Dump snmp logs
+        if: always() && matrix.skip-unit-check != '1'
+        run: |
+          cat /tmp/snmpsimd.log
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1

.travis.yml

@@ -40,9 +40,6 @@ install:
 after_failure:
   - tail /tmp/snmpsimd.log
 
-after_success:
-  - scripts/deploy-docs.sh
-
 before_script:
   - phpenv config-rm xdebug.ini
   - test -n "$SKIP_WEB_CHECK" || php artisan serve --env=dusk.testing  2>/dev/null &

LibreNMS/Proc.php

@@ -176,7 +176,9 @@ class Proc
     public function close($command = null)
     {
         if (isset($command)) {
-            $this->sendInput($this->checkAddEOL($command));
+            if (is_resource($this->_pipes[0])) {
+                $this->sendInput($this->checkAddEOL($command));
+            }
         }
 
         $this->closePipes();

LibreNMS/Util/CiHelper.php

@@ -382,10 +382,8 @@ class CiHelper
             if (! $silence) {
                 echo "failed ($duration)\n";
             }
-            if ($quiet || $silence) {
-                echo $proc->getOutput() . PHP_EOL;
-                echo $proc->getErrorOutput() . PHP_EOL;
-            }
+            echo $proc->getOutput() . PHP_EOL;
+            echo $proc->getErrorOutput() . PHP_EOL;
         } elseif (! $silence) {
             echo "success ($duration)\n";
         }

LibreNMS/Util/FileCategorizer.php

@@ -38,6 +38,7 @@ class FileCategorizer extends Categorizer
             $this->setSkippable(function ($item) {
                 return in_array($item, [
                     '.travis.yml',
+                    '.github/workflows/test.yml',
                     'LibreNMS/Util/CiHelper.php',
                     'LibreNMS/Util/FileCategorizer.php',
                     'app/Console/Commands/DevCheckCommand.php',
@@ -65,7 +66,7 @@ class FileCategorizer extends Categorizer
             return Str::startsWith($item, 'resources/') ? $item : false;
         });
         $this->addCategory('full-checks', function ($item) {
-            return in_array($item, ['composer.lock', '.travis.yml']) ? $item : false;
+            return in_array($item, ['composer.lock', '.travis.yml', '.github/workflows/test.yml']) ? $item : false;
         });
         $this->addCategory('os-files', function ($item) {
             if (($os_name = $this->osFromFile($item)) !== null) {

LibreNMS/Util/Snmpsim.php

@@ -80,7 +80,7 @@ class Snmpsim
             } else {
                 echo "\nFailed to start Snmpsim. Scroll up for error.\n";
             }
-            exit;
+            exit(1);
         }
     }
 

README.md

@@ -1,6 +1,6 @@
 [![Scrutinizer Status](https://scrutinizer-ci.com/g/librenms/librenms/badges/build.png?b=master)](https://scrutinizer-ci.com/g/librenms/librenms/build-status/master)
 [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/librenms/librenms/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/librenms/librenms/?branch=master)
-[![Travis Status](https://travis-ci.com/librenms/librenms.svg?branch=master)](https://travis-ci.com/librenms/librenms)
+[![Test Status](https://github.com/librenms/librenms/workflows/test/badge.svg)](https://github.com/librenms/librenms/actions?query=workflow%3Atest)
 
 Introduction
 ------------

scripts/deploy-docs.sh

@@ -1,62 +0,0 @@
-#!/usr/bin/env bash
-GH_REPO="@github.com/librenms-docs/librenms-docs.github.io.git"
-FULL_REPO="https://${GH_TOKEN}$GH_REPO"
-
-if [ "$EXECUTE_BUILD_DOCS" != "true" ]; then
-    echo "Doc build skipped"
-    exit 0
-fi
-
-pip3 install --upgrade pip
-pip3 install --user --requirement <(cat <<EOF
-click==7.1.2
-future==0.18.2
-Jinja2==2.11.2
-joblib==0.17.0
-livereload==2.6.3
-lunr==0.5.8
-Markdown==3.3.2
-MarkupSafe==1.1.1
-mkdocs==1.1.2
-mkdocs-exclude==1.0.2
-mkdocs-macros-plugin==0.4.18
-mkdocs-material==6.1.0
-mkdocs-material-extensions==1.0.1
-nltk==3.5
-Pygments==2.7.1
-pymdown-extensions==8.0.1
-python-dateutil==2.8.1
-PyYAML==5.3.1
-regex==2020.10.23
-six==1.15.0
-termcolor==1.1.0
-tornado==6.0.4
-tqdm==4.50.2
-EOF
-)
-
-
-mkdir -p out
-cd out || exit 1
-
-git init
-git remote add origin "$FULL_REPO"
-git fetch
-git config user.name "librenms-docs"
-git config user.email "travis@librenms.org"
-git checkout master
-
-cd ../ || exit 1
-mkdocs build --clean
-build_result=$?
-
-# Only deploy after merging to master
-if [ "$build_result" == "0" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_BRANCH" == "master" ]; then
-    cd out/ || exit 1
-    touch .
-    git add -A .
-    git commit -m "GH-Pages update by travis after $TRAVIS_COMMIT"
-    git push -q origin master
-else
-    exit ${build_result}  # return doc build result
-fi

tests/bootstrap.php

@@ -54,7 +54,7 @@ if (getenv('DBTEST')) {
 
     // create testing table if needed
     $db_config = \config('database.connections.testing');
-    $connection = new PDO("mysql:host={$db_config['host']}", $db_config['username'], $db_config['password']);
+    $connection = new PDO("mysql:host={$db_config['host']};port={$db_config['port']}", $db_config['username'], $db_config['password']);
     $result = $connection->query("CREATE DATABASE IF NOT EXISTS {$db_config['database']} CHARACTER SET utf8 COLLATE utf8_unicode_ci");
     if ($connection->errorCode() == '42000') {
         echo implode(' ', $connection->errorInfo()) . PHP_EOL;