Deb packaging automation (#360)

Build and sanity check modern and minimal Routinator .deb packages using cargo-deb and GitHub Actions. Also test against Debian 11 (Contributed by @netravnen).
2024-05-19 06:50:04 +00:00 · 2020-09-09 15:03:30 +02:00
parent e439df87ec
commit 6b61e845a6
8 changed files with 365 additions and 24 deletions
--- a/.github/workflows/pkg.yml
+++ b/.github/workflows/pkg.yml
@ -0,0 +1,314 @@
+# GitHub Actions workflow for building and testing Routinator O/S packages.
+# Uses GitHub Actions caching to avoid rebuilding Rust cargo-deb and
+# Routinator dependencies on every run.
+#
+# Note: at the time of writing the GH cache contents expire after a
+# week if not used so the next build may be much slower as it will
+# have to re-download/build/install lots of Rust crates.
+#
+# Packages are built inside Docker containers as GH Runners have extra libraries
+# and packages installed which can cause package building to succeed but package
+# installation on a real target O/S to fail, due to being built against too
+# recent version of a package such as libssl or glibc.
+#
+# Packages are tested inside LXC/LXD containers because Docker containers don't
+# by default support init managers such as systemd but we want to test systemd
+# service unit installation and activation.
+
+name: Packaging
+on: [push, pull_request]
+
+defaults:
+  run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+jobs:
+  # Use the cargo-deb Rust crate to build a Debian package for installing
+  # Routinator. See: https://github.com/mmstick/cargo-deb
+  deb-pkg:
+    strategy:
+      matrix:
+        image: # can't use complex values here, only primitive values are allowed
+          - "ubuntu:xenial"   # ubuntu/16.04
+          - "ubuntu:bionic"   # ubuntu/18.04
+          - "ubuntu:focal"    # ubuntu/20.04
+          - "debian:stretch"  # debian/9
+          - "debian:buster"   # debian/10
+          - "debian:bullseye" # debian/11
+    env:
+      CARGO_DEB_VER: 94ba8f3
+      # A Routinator version of the form 'x.y.z-bis' denotes a dev build that is
+      # newer than the released x.y.z version but is not yet a new release.
+      NEXT_VER_LABEL: bis
+    name: deb-pkg
+    runs-on: ubuntu-latest
+    # Build on the oldest platform we are targeting in order to avoid
+    # https://github.com/rust-lang/rust/issues/57497. Specifying container
+    # causes all of the steps in this job to run inside a Docker container.
+    container: ${{ matrix.image }}
+
+    steps:
+    - name: Set vars
+      id: setvars
+      shell: bash
+      run: |
+        # Get the operating system and release name (e.g. ubuntu and xenial) from
+        # the image name (e.g. ubuntu:xenial) by extracting only the parts before
+        # and after but not including the colon:
+        echo ::set-env name=OS_NAME::${MATRIX_IMAGE%:*}
+        echo ::set-env name=OS_REL::${MATRIX_IMAGE#*:}
+      env:
+        MATRIX_IMAGE: ${{ matrix.image }}
+
+    # Git clone the Routinator code in the branch we were invoked on.
+    - name: Checkout repository
+      uses: actions/checkout@v1
+
+    # Install Rust the hard way rather than using a GH Action because the action
+    # doesn't work inside a Docker container.
+    - name: Install Rust
+      run: |
+        apt-get update
+        apt-get install -y curl
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile minimal -y
+        echo "::add-path::$HOME/.cargo/bin"
+      env:
+        DEBIAN_FRONTEND: noninteractive
+
+    - name: Install compilation and other dependencies
+      run: |
+          apt-get install -y build-essential jq lintian pkg-config
+      env:
+        DEBIAN_FRONTEND: noninteractive
+
+    # Speed up Routinator Rust builds by caching unchanged built dependencies.
+    # See: https://github.com/actions/cache/blob/master/examples.md#rust---cargo
+    - name: Cache Dot Cargo
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ matrix.image }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+    # Speed up cargo-deb installation by only re-downloading and re-building its
+    # dependent crates if we change the version of cargo-deb that we are using.
+    - name: Cache Cargo Deb binary
+      id: cache-cargo-deb
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin/cargo-deb
+        key: ${{ matrix.image }}-cargo-deb-${{ env.CARGO_DEB_VER }}
+
+    # Only install cargo-deb if not already fetched from the cache.
+    - name: Install Cargo Deb
+      if: steps.cache-cargo-deb.outputs.cache-hit != 'true'
+      run: |
+        cargo install --git https://github.com/mmstick/cargo-deb.git --rev $CARGO_DEB_VER cargo-deb
+
+    # Instruct cargo-deb to build the Debian package using the config section
+    # in Cargo.toml for the specified "variant".
+    - name: Create the DEB package
+      run: |
+        # Packages for different distributions (e.g. Stretch, Buster) of the same
+        # O/S (e.g. Debian) when served from a single package repository MUST have
+        # unique package_ver_architecture triples. Cargo deb can vary the name based
+        # on the 'variant' config section in use, but doesn't do so according to
+        # Debian policy (as it modifies the package name, not the package version).
+        #   Format: package_ver_architecture
+        #   Where ver has format: [epoch:]upstream_version[-debian_revision]
+        #   And debian_version should be of the form: 1<xxx>
+        #   Where it is common to set <xxx> to the O/S name.
+        # See:
+        #   - https://unix.stackexchange.com/a/190899
+        #   - https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
+        # Therefore we generate the version ourselves.
+        #
+        # In addition, Semantic Versioning and Debian version policy cannot
+        # express a pre-release label in the same way. For example 0.8.0-rc.1
+        # is a valid Cargo.toml [package].version value but when used as a
+        # Debian package version 0.8.0-rc.1 would be considered _NEWER_ than
+        # the final 0.8.0 release. To express this in a Debian compatible way we
+        # must replace the dash '-' with a tilda '~'.
+        #
+        # Finally, sometimes we want a version to be NEWER than the latest
+        # release but without having to decide what higher semver number to bump
+        # to. In this case we do NOT want dash '-' to become '~' because `-`
+        # is treated as higher and tilda is treated as lower.
+        ROUTINATOR_VER=$(cargo read-manifest | jq -r '.version' | tr '-' '~')
+        DEB_ROUTINATOR_VER=$(echo $ROUTINATOR_VER | sed -e "s/~$NEXT_VER_LABEL/-$NEXT_VER_LABEL/")
+
+        case ${OS_REL} in
+          xenial|bionic|stretch) VARIANT_NAME="minimal" ;;
+          *)                     VARIANT_NAME="" ;;
+        esac
+
+        case ${{ github.event_name }} in
+          pull_request) MAINTAINER="${{ github.actor }} <unknown@email.address>" ;;
+          push)         MAINTAINER="${{ github.event.pusher.name }} <${{ github.event.pusher.email }}>" ;;
+          *)            echo 2>&1 "ERROR: Unexpected GitHub Actions event"; exit 1 ;;
+        esac
+
+        # Generate the RFC 5322 format date by hand instead of using date --rfc-email
+        # because that option doesn't exist on Ubuntu 16.04 and Debian 9
+        RFC5322_TS=$(LC_TIME=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S %z')
+
+        # Generate the changelog file that Debian packages are required to have.
+        # See: https://www.debian.org/doc/manuals/maint-guide/dreq.en.html#changelog
+        echo "routinator (${DEB_ROUTINATOR_VER}) unstable; urgency=medium" >debian/changelog
+        echo "  * See: https://github.com/NLnetLabs/routinator/releases/tag/v${ROUTINATOR_VER}" >>debian/changelog
+        echo " -- maintainer ${MAINTAINER}  ${RFC5322_TS}" >>debian/changelog
+        DEB_VER="${DEB_ROUTINATOR_VER}-1${OS_REL}"
+
+        if [[ "${VARIANT_NAME}" == "" ]]; then
+          cargo deb --deb-version ${DEB_VER} -v
+        else
+          cargo deb --deb-version ${DEB_VER} --variant ${VARIANT_NAME} -v
+        fi
+
+    # See what Lintian thinks of our package.
+    - name: Verify the DEB package
+      run: |
+        lintian -v target/debian/*.deb
+
+    # Upload the produced DEB package. The artifact will be available
+    # via the GH Actions job summary and build log pages, but only to
+    # users logged in to GH with sufficient rights in this project. The
+    # uploaded artifact is also downloaded by the next job (see below)
+    # to sanity check that it can be installed and results in a working
+    # Routinator installation.
+    - name: Upload DEB package
+      uses: actions/upload-artifact@v2
+      with:
+        name: ${{ env.OS_NAME }}_${{ env.OS_REL }}
+        path: target/debian/*.deb
+
+  # Download and sanity check on target operating systems the packages created
+  # by previous jobs (see above). Don't test on GH runners as they come with
+  # lots of software and libraries pre-installed and thus are not representative
+  # of the actual deployment targets, nor do GH runners support all targets that
+  # we want to test. Don't test in Docker containers as they do not support
+  # systemd.
+  deb-pkg-test:
+    name: deb-pkg-test
+    needs: deb-pkg
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image: # can't use complex values here, only primitive values are allowed
+          - "ubuntu:xenial"   # ubuntu/16.04
+          - "ubuntu:bionic"   # ubuntu/18.04
+          - "ubuntu:focal"    # ubuntu/20.04
+          - "debian:stretch"  # debian/9
+          - "debian:buster"   # debian/10
+          - "debian:bullseye" # debian/11
+    steps:
+    # Set some environment variables that will be available to "run" steps below
+    # in this job, and some output variables that will be available in GH Action
+    # step definitions below.
+    - name: Set vars
+      id: setvars
+      shell: bash
+      run: |
+        # Get the operating system and release name (e.g. ubuntu and xenial) from
+        # the image name (e.g. ubuntu:xenial) by extracting only the parts before
+        # and after but not including the colon:
+        OS_NAME=${MATRIX_IMAGE%:*}
+        OS_REL=${MATRIX_IMAGE#*:}
+
+        echo ::set-env name=OS_NAME::${OS_NAME}
+        echo ::set-env name=OS_REL::${OS_REL}
+        echo ::set-env name=LXC_IMAGE::images:${OS_NAME}/${OS_REL}/cloud
+      env:
+        MATRIX_IMAGE: ${{ matrix.image }}
+
+    - name: Download DEB package
+      uses: actions/download-artifact@v2
+      with:
+        name: ${{ env.OS_NAME }}_${{ env.OS_REL }}
+
+    - name: Add current user to LXD group
+      run: |
+        sudo usermod --append --groups lxd $(whoami)
+
+    - name: Initialize LXD
+      run: |
+        sudo lxd init --auto
+
+    - name: Check LXD configuration
+      run: |
+        sg lxd -c "lxc info"
+
+    - name: Launch LXC container
+      run: |
+        # security.nesting=true is needed to avoid error "Failed to set up mount
+        # namespacing: Permission denied" in a Debian 10 container.
+        sg lxd -c "lxc launch ${LXC_IMAGE} -c security.nesting=true testcon"
+
+    # Run apt-get update and install man and sudo support (missing in some LXC/LXD
+    # O/S images) but first wait for cloud-init to finish otherwise the network
+    # isn't yet ready. Don't use cloud-init status --wait as that isn't supported
+    # on older O/S's like Ubuntu 16.04 and Debian 9. Use the sudo package provided
+    # configuration files otherwise when using sudo we get an error that the root
+    # user isn't allowed to use sudo.
+    - name: Prepare container
+      shell: bash
+      run: |
+        echo "Waiting for cloud-init.."
+        while ! sudo lxc exec testcon -- ls -la /var/lib/cloud/data/result.json; do
+          sleep 1s
+        done
+        sg lxd -c "lxc exec testcon -- apt-get update"
+        sg lxd -c "lxc exec testcon -- apt-get install -y -o Dpkg::Options::=\"--force-confnew\" man sudo"
+
+    - name: Copy DEB into LXC container
+      run: |
+        DEB_FILE=$(ls -1 *.deb)
+        sg lxd -c "lxc file push ${DEB_FILE} testcon/tmp/"
+        echo ::set-env name=DEB_FILE::${DEB_FILE}
+
+    - name: Install new DEB package
+      run: |
+        sg lxd -c "lxc exec testcon -- apt-get -y install /tmp/${DEB_FILE}"
+
+    - name: Test installed packages
+      run: |
+        echo -e "\nROUTINATOR VERSION:"
+        sg lxd -c "lxc exec testcon -- routinator --version"
+
+        echo -e "\nROUTINATOR CONF:"
+        sg lxd -c "lxc exec testcon -- cat /etc/routinator/routinator.conf"
+
+        echo -e "\nROUTINATOR DATA DIR:"
+        sg lxd -c "lxc exec testcon -- ls -la /var/lib/routinator"
+
+        echo -e "\nROUTINATOR SERVICE STATUS BEFORE ENABLE:"
+        sg lxd -c "lxc exec testcon -- systemctl status routinator || true"
+
+        echo -e "\nINIT ROUTINATOR:"
+        sg lxd -c "lxc exec testcon -- sudo -u routinator routinator --config /etc/routinator/routinator.conf init --accept-arin-rpa"
+
+        echo -e "\nENABLE ROUTINATOR SERVICE:"
+        sg lxd -c "lxc exec testcon -- systemctl enable routinator"
+
+        echo -e "\nROUTINATOR SERVICE STATUS AFTER ENABLE:"
+        sg lxd -c "lxc exec testcon -- systemctl status routinator || true"
+
+        echo -e "\nSTART ROUTINATOR SERVICE:"
+        sg lxd -c "lxc exec testcon -- systemctl start routinator"
+        
+        echo -e "\nROUTINATOR SERVICE STATUS AFTER START:"
+        sleep 1s
+        sg lxd -c "lxc exec testcon -- systemctl status routinator"
+
+        echo -e "\nROUTINATOR MAN PAGE:"
+        sg lxd -c "lxc exec testcon -- man -P cat routinator"
+
+        echo -e "\nROUTINATOR TALS DIR:"
+        sg lxd -c "lxc exec testcon -- ls -la /var/lib/routinator/tals/"
+
+        echo -e "\nROUTINATOR RPKI CACHE DIR (first 20 lines of ls output only):"
+        sg lxd -c "lxc exec testcon -- ls -ltR /var/lib/routinator/rpki-cache/ | head -n 20"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -1,4 +1,5 @@
 [package]
+# Note: some of these values are also used when building Debian packages below.
 name = "routinator"
 version = "0.7.1-bis"
 edition = "2018"
@ -49,7 +50,7 @@ rustc_version   = "0.2.3"

 [features]
 default = ["rustls-tls", "socks"]
-extra-debug = [ "rpki/extra-debug" ]
+extra-debug = ["rpki/extra-debug"]
 socks = [ "reqwest/socks" ]
 rta = []
 native-tls = [ "reqwest/default-tls", "tls" ]
@ -57,6 +58,7 @@ rustls-tls = [ "reqwest/rustls-tls", "tls" ]
 tls = []

 [package.metadata.deb]
+name = "routinator"
 maintainer = "The NLnet Labs RPKI Team <rpki@nlnetlabs.nl>"
 license-file = ["LICENSE", "0"]
 extended-description = """\
@ -65,7 +67,7 @@ statements about the association of Internet routing resources. \
 In particular, it allows the holder of an IP address prefix to publish which \
 AS number will be the origin of BGP route announcements for it. \
 Routinator is a RPKI relying party software written in Rust. """
-depends = "$auto, rsync"
+depends = "$auto, rsync, adduser"
 section = "net"
 priority = "optional"
 assets = [
@ -73,20 +75,13 @@ assets = [
    ["README.md", "usr/share/doc/routinator/", "644"],
    ["doc/misc.md", "usr/share/doc/routinator/misc.md", "644"],
    ["doc/routinator.1", "usr/share/man/man1/routinator.1", "644"],
-    ["etc/routinator.service", "lib/systemd/system/routinator.service", "644"],
-    ["etc/routinator.conf.system-service", "etc/routinator/routinator.conf", "644"]
+    ["etc/routinator.conf.system-service", "etc/routinator/routinator.conf", "644"],
+    ["debian/service.preset", "/lib/systemd/system-preset/50-routinator.preset", "644"]
 ]
 maintainer-scripts = "debian"
-conf-files = [
-    "etc/routinator/routinator.conf"
-]
+changelog = "debian/changelog" # this will be generated by the pkg workflow
+copyright = "Copyright (c) 2019, NLnet Labs. All rights reserved."
+conf-files = ["/etc/routinator/routinator.conf"]
+systemd-units = { unit-name = "routinator", enable = false }

 [package.metadata.deb.variants.minimal]
-assets = [
-    ["target/release/routinator", "usr/bin/", "755"],
-    ["README.md", "usr/share/doc/routinator/", "644"],
-    ["doc/misc.md", "usr/share/doc/routinator/misc.md", "644"],
-    ["doc/routinator.1", "usr/share/man/man1/routinator.1", "644"],
-    ["etc/routinator.service.minimal", "lib/systemd/system/routinator.service", "644"],
-    ["etc/routinator.conf.system-service", "etc/routinator/routinator.conf", "644"]
-]
--- a/debian/postinst
+++ b/debian/postinst
@ -0,0 +1,17 @@
+#!/bin/sh -e
+
+ROUTINATOR_HOME="/var/lib/routinator/"
+ROUTINATOR_USER="routinator"
+
+create_user() {
+    if id ${ROUTINATOR_USER} > /dev/null 2>&1; then return; fi
+    adduser --system --home "${ROUTINATOR_HOME}" --group ${ROUTINATOR_USER}
+}
+
+case "$1" in
+configure)
+    create_user
+    ;;
+esac
+
+#DEBHELPER#
--- a/debian/postrm
+++ b/debian/postrm
@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+ROUTINATOR_ETC="/etc/routinator"
+
+case "$1" in
+purge)
+    # Per https://www.debian.org/doc/debian-policy/ch-files.html#behavior
+    # "configuration files must be preserved when the package is removed, and
+    #  only deleted when the package is purged."
+    if [ -d ${ROUTINATOR_ETC} ]; then
+        rm -R ${ROUTINATOR_ETC}
+    fi
+    ;;
+esac
+
+#DEBHELPER#
--- a/debian/routinator-minimal.routinator.service
+++ b/debian/routinator-minimal.routinator.service
@ -5,8 +5,7 @@ After=network.target

 [Service]
 ExecStart=/usr/bin/routinator --config=/etc/routinator/routinator.conf --syslog server
+User=routinator

 [Install]
-WantedBy=multi-user.target
-
-
+WantedBy=multi-user.target
--- a/debian/routinator.routinator.service
+++ b/debian/routinator.routinator.service
--- a/debian/service.preset
+++ b/debian/service.preset
@ -0,0 +1 @@
+disable routinator.service
--- a/etc/routinator.conf.system-service
+++ b/etc/routinator.conf.system-service
@ -2,8 +2,8 @@
 # ========================================================
 #
 # This configuration assumes that the TALs are placed in
-# /etc/routinator/tals and the repository is maintained in
-# /var/lib/routinator/repository.
+# /var/lib/routinator/tals and the repository cache is maintained in
+# /var/lib/routinator/rpki-cache.
 #
 # It will start Routinator with an RTR server listening on port 3323 and
 # an HTTP server listening on port 8323. Both are limited to localhost by
@ -17,8 +17,7 @@
 # example, see etc/routinator.conf.example in the source distribution or
 # consult the manual page.

-repository-dir = "/var/lib/routinator"
-tal-dir = "/etc/routinator/tals"
+repository-dir = "/var/lib/routinator/rpki-cache"
+tal-dir = "/var/lib/routinator/tals"
 rtr-listen = ["127.0.0.1:3323"]
-http-listen = ["127.0.0.1:8323"]
-
+http-listen = ["127.0.0.1:8323"]