From 6b61e845a62cad33ade07ce8880a8b82eca9dccf Mon Sep 17 00:00:00 2001 From: ximon18 <3304436+ximon18@users.noreply.github.com> Date: Wed, 9 Sep 2020 15:03:30 +0200 Subject: [PATCH] Deb packaging automation (#360) Build and sanity check modern and minimal Routinator .deb packages using cargo-deb and GitHub Actions. Also test against Debian 11 (Contributed by @netravnen). --- .github/workflows/pkg.yml | 314 ++++++++++++++++++ Cargo.toml | 25 +- debian/postinst | 17 + debian/postrm | 16 + .../routinator-minimal.routinator.service | 5 +- .../routinator.routinator.service | 0 debian/service.preset | 1 + etc/routinator.conf.system-service | 11 +- 8 files changed, 365 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/pkg.yml create mode 100644 debian/postinst create mode 100644 debian/postrm rename etc/routinator.service.minimal => debian/routinator-minimal.routinator.service (82%) rename etc/routinator.service => debian/routinator.routinator.service (100%) create mode 100644 debian/service.preset diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml new file mode 100644 index 0000000..82f4804 --- /dev/null +++ b/.github/workflows/pkg.yml @@ -0,0 +1,314 @@ +# GitHub Actions workflow for building and testing Routinator O/S packages. +# Uses GitHub Actions caching to avoid rebuilding Rust cargo-deb and +# Routinator dependencies on every run. +# +# Note: at the time of writing the GH cache contents expire after a +# week if not used so the next build may be much slower as it will +# have to re-download/build/install lots of Rust crates. +# +# Packages are built inside Docker containers as GH Runners have extra libraries +# and packages installed which can cause package building to succeed but package +# installation on a real target O/S to fail, due to being built against too +# recent version of a package such as libssl or glibc. +# +# Packages are tested inside LXC/LXD containers because Docker containers don't +# by default support init managers such as systemd but we want to test systemd +# service unit installation and activation. + +name: Packaging +on: [push, pull_request] + +defaults: + run: + # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell + shell: bash --noprofile --norc -eo pipefail -x {0} + +jobs: + # Use the cargo-deb Rust crate to build a Debian package for installing + # Routinator. See: https://github.com/mmstick/cargo-deb + deb-pkg: + strategy: + matrix: + image: # can't use complex values here, only primitive values are allowed + - "ubuntu:xenial" # ubuntu/16.04 + - "ubuntu:bionic" # ubuntu/18.04 + - "ubuntu:focal" # ubuntu/20.04 + - "debian:stretch" # debian/9 + - "debian:buster" # debian/10 + - "debian:bullseye" # debian/11 + env: + CARGO_DEB_VER: 94ba8f3 + # A Routinator version of the form 'x.y.z-bis' denotes a dev build that is + # newer than the released x.y.z version but is not yet a new release. + NEXT_VER_LABEL: bis + name: deb-pkg + runs-on: ubuntu-latest + # Build on the oldest platform we are targeting in order to avoid + # https://github.com/rust-lang/rust/issues/57497. Specifying container + # causes all of the steps in this job to run inside a Docker container. + container: ${{ matrix.image }} + + steps: + - name: Set vars + id: setvars + shell: bash + run: | + # Get the operating system and release name (e.g. ubuntu and xenial) from + # the image name (e.g. ubuntu:xenial) by extracting only the parts before + # and after but not including the colon: + echo ::set-env name=OS_NAME::${MATRIX_IMAGE%:*} + echo ::set-env name=OS_REL::${MATRIX_IMAGE#*:} + env: + MATRIX_IMAGE: ${{ matrix.image }} + + # Git clone the Routinator code in the branch we were invoked on. + - name: Checkout repository + uses: actions/checkout@v1 + + # Install Rust the hard way rather than using a GH Action because the action + # doesn't work inside a Docker container. + - name: Install Rust + run: | + apt-get update + apt-get install -y curl + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile minimal -y + echo "::add-path::$HOME/.cargo/bin" + env: + DEBIAN_FRONTEND: noninteractive + + - name: Install compilation and other dependencies + run: | + apt-get install -y build-essential jq lintian pkg-config + env: + DEBIAN_FRONTEND: noninteractive + + # Speed up Routinator Rust builds by caching unchanged built dependencies. + # See: https://github.com/actions/cache/blob/master/examples.md#rust---cargo + - name: Cache Dot Cargo + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ matrix.image }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + # Speed up cargo-deb installation by only re-downloading and re-building its + # dependent crates if we change the version of cargo-deb that we are using. + - name: Cache Cargo Deb binary + id: cache-cargo-deb + uses: actions/cache@v2 + with: + path: ~/.cargo/bin/cargo-deb + key: ${{ matrix.image }}-cargo-deb-${{ env.CARGO_DEB_VER }} + + # Only install cargo-deb if not already fetched from the cache. + - name: Install Cargo Deb + if: steps.cache-cargo-deb.outputs.cache-hit != 'true' + run: | + cargo install --git https://github.com/mmstick/cargo-deb.git --rev $CARGO_DEB_VER cargo-deb + + # Instruct cargo-deb to build the Debian package using the config section + # in Cargo.toml for the specified "variant". + - name: Create the DEB package + run: | + # Packages for different distributions (e.g. Stretch, Buster) of the same + # O/S (e.g. Debian) when served from a single package repository MUST have + # unique package_ver_architecture triples. Cargo deb can vary the name based + # on the 'variant' config section in use, but doesn't do so according to + # Debian policy (as it modifies the package name, not the package version). + # Format: package_ver_architecture + # Where ver has format: [epoch:]upstream_version[-debian_revision] + # And debian_version should be of the form: 1 + # Where it is common to set to the O/S name. + # See: + # - https://unix.stackexchange.com/a/190899 + # - https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + # Therefore we generate the version ourselves. + # + # In addition, Semantic Versioning and Debian version policy cannot + # express a pre-release label in the same way. For example 0.8.0-rc.1 + # is a valid Cargo.toml [package].version value but when used as a + # Debian package version 0.8.0-rc.1 would be considered _NEWER_ than + # the final 0.8.0 release. To express this in a Debian compatible way we + # must replace the dash '-' with a tilda '~'. + # + # Finally, sometimes we want a version to be NEWER than the latest + # release but without having to decide what higher semver number to bump + # to. In this case we do NOT want dash '-' to become '~' because `-` + # is treated as higher and tilda is treated as lower. + ROUTINATOR_VER=$(cargo read-manifest | jq -r '.version' | tr '-' '~') + DEB_ROUTINATOR_VER=$(echo $ROUTINATOR_VER | sed -e "s/~$NEXT_VER_LABEL/-$NEXT_VER_LABEL/") + + case ${OS_REL} in + xenial|bionic|stretch) VARIANT_NAME="minimal" ;; + *) VARIANT_NAME="" ;; + esac + + case ${{ github.event_name }} in + pull_request) MAINTAINER="${{ github.actor }} " ;; + push) MAINTAINER="${{ github.event.pusher.name }} <${{ github.event.pusher.email }}>" ;; + *) echo 2>&1 "ERROR: Unexpected GitHub Actions event"; exit 1 ;; + esac + + # Generate the RFC 5322 format date by hand instead of using date --rfc-email + # because that option doesn't exist on Ubuntu 16.04 and Debian 9 + RFC5322_TS=$(LC_TIME=en_US.UTF-8 date +'%a, %d %b %Y %H:%M:%S %z') + + # Generate the changelog file that Debian packages are required to have. + # See: https://www.debian.org/doc/manuals/maint-guide/dreq.en.html#changelog + echo "routinator (${DEB_ROUTINATOR_VER}) unstable; urgency=medium" >debian/changelog + echo " * See: https://github.com/NLnetLabs/routinator/releases/tag/v${ROUTINATOR_VER}" >>debian/changelog + echo " -- maintainer ${MAINTAINER} ${RFC5322_TS}" >>debian/changelog + DEB_VER="${DEB_ROUTINATOR_VER}-1${OS_REL}" + + if [[ "${VARIANT_NAME}" == "" ]]; then + cargo deb --deb-version ${DEB_VER} -v + else + cargo deb --deb-version ${DEB_VER} --variant ${VARIANT_NAME} -v + fi + + # See what Lintian thinks of our package. + - name: Verify the DEB package + run: | + lintian -v target/debian/*.deb + + # Upload the produced DEB package. The artifact will be available + # via the GH Actions job summary and build log pages, but only to + # users logged in to GH with sufficient rights in this project. The + # uploaded artifact is also downloaded by the next job (see below) + # to sanity check that it can be installed and results in a working + # Routinator installation. + - name: Upload DEB package + uses: actions/upload-artifact@v2 + with: + name: ${{ env.OS_NAME }}_${{ env.OS_REL }} + path: target/debian/*.deb + + # Download and sanity check on target operating systems the packages created + # by previous jobs (see above). Don't test on GH runners as they come with + # lots of software and libraries pre-installed and thus are not representative + # of the actual deployment targets, nor do GH runners support all targets that + # we want to test. Don't test in Docker containers as they do not support + # systemd. + deb-pkg-test: + name: deb-pkg-test + needs: deb-pkg + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: # can't use complex values here, only primitive values are allowed + - "ubuntu:xenial" # ubuntu/16.04 + - "ubuntu:bionic" # ubuntu/18.04 + - "ubuntu:focal" # ubuntu/20.04 + - "debian:stretch" # debian/9 + - "debian:buster" # debian/10 + - "debian:bullseye" # debian/11 + steps: + # Set some environment variables that will be available to "run" steps below + # in this job, and some output variables that will be available in GH Action + # step definitions below. + - name: Set vars + id: setvars + shell: bash + run: | + # Get the operating system and release name (e.g. ubuntu and xenial) from + # the image name (e.g. ubuntu:xenial) by extracting only the parts before + # and after but not including the colon: + OS_NAME=${MATRIX_IMAGE%:*} + OS_REL=${MATRIX_IMAGE#*:} + + echo ::set-env name=OS_NAME::${OS_NAME} + echo ::set-env name=OS_REL::${OS_REL} + echo ::set-env name=LXC_IMAGE::images:${OS_NAME}/${OS_REL}/cloud + env: + MATRIX_IMAGE: ${{ matrix.image }} + + - name: Download DEB package + uses: actions/download-artifact@v2 + with: + name: ${{ env.OS_NAME }}_${{ env.OS_REL }} + + - name: Add current user to LXD group + run: | + sudo usermod --append --groups lxd $(whoami) + + - name: Initialize LXD + run: | + sudo lxd init --auto + + - name: Check LXD configuration + run: | + sg lxd -c "lxc info" + + - name: Launch LXC container + run: | + # security.nesting=true is needed to avoid error "Failed to set up mount + # namespacing: Permission denied" in a Debian 10 container. + sg lxd -c "lxc launch ${LXC_IMAGE} -c security.nesting=true testcon" + + # Run apt-get update and install man and sudo support (missing in some LXC/LXD + # O/S images) but first wait for cloud-init to finish otherwise the network + # isn't yet ready. Don't use cloud-init status --wait as that isn't supported + # on older O/S's like Ubuntu 16.04 and Debian 9. Use the sudo package provided + # configuration files otherwise when using sudo we get an error that the root + # user isn't allowed to use sudo. + - name: Prepare container + shell: bash + run: | + echo "Waiting for cloud-init.." + while ! sudo lxc exec testcon -- ls -la /var/lib/cloud/data/result.json; do + sleep 1s + done + sg lxd -c "lxc exec testcon -- apt-get update" + sg lxd -c "lxc exec testcon -- apt-get install -y -o Dpkg::Options::=\"--force-confnew\" man sudo" + + - name: Copy DEB into LXC container + run: | + DEB_FILE=$(ls -1 *.deb) + sg lxd -c "lxc file push ${DEB_FILE} testcon/tmp/" + echo ::set-env name=DEB_FILE::${DEB_FILE} + + - name: Install new DEB package + run: | + sg lxd -c "lxc exec testcon -- apt-get -y install /tmp/${DEB_FILE}" + + - name: Test installed packages + run: | + echo -e "\nROUTINATOR VERSION:" + sg lxd -c "lxc exec testcon -- routinator --version" + + echo -e "\nROUTINATOR CONF:" + sg lxd -c "lxc exec testcon -- cat /etc/routinator/routinator.conf" + + echo -e "\nROUTINATOR DATA DIR:" + sg lxd -c "lxc exec testcon -- ls -la /var/lib/routinator" + + echo -e "\nROUTINATOR SERVICE STATUS BEFORE ENABLE:" + sg lxd -c "lxc exec testcon -- systemctl status routinator || true" + + echo -e "\nINIT ROUTINATOR:" + sg lxd -c "lxc exec testcon -- sudo -u routinator routinator --config /etc/routinator/routinator.conf init --accept-arin-rpa" + + echo -e "\nENABLE ROUTINATOR SERVICE:" + sg lxd -c "lxc exec testcon -- systemctl enable routinator" + + echo -e "\nROUTINATOR SERVICE STATUS AFTER ENABLE:" + sg lxd -c "lxc exec testcon -- systemctl status routinator || true" + + echo -e "\nSTART ROUTINATOR SERVICE:" + sg lxd -c "lxc exec testcon -- systemctl start routinator" + + echo -e "\nROUTINATOR SERVICE STATUS AFTER START:" + sleep 1s + sg lxd -c "lxc exec testcon -- systemctl status routinator" + + echo -e "\nROUTINATOR MAN PAGE:" + sg lxd -c "lxc exec testcon -- man -P cat routinator" + + echo -e "\nROUTINATOR TALS DIR:" + sg lxd -c "lxc exec testcon -- ls -la /var/lib/routinator/tals/" + + echo -e "\nROUTINATOR RPKI CACHE DIR (first 20 lines of ls output only):" + sg lxd -c "lxc exec testcon -- ls -ltR /var/lib/routinator/rpki-cache/ | head -n 20" diff --git a/Cargo.toml b/Cargo.toml index 795c813..20a0f5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [package] +# Note: some of these values are also used when building Debian packages below. name = "routinator" version = "0.7.1-bis" edition = "2018" @@ -49,7 +50,7 @@ rustc_version = "0.2.3" [features] default = ["rustls-tls", "socks"] -extra-debug = [ "rpki/extra-debug" ] +extra-debug = ["rpki/extra-debug"] socks = [ "reqwest/socks" ] rta = [] native-tls = [ "reqwest/default-tls", "tls" ] @@ -57,6 +58,7 @@ rustls-tls = [ "reqwest/rustls-tls", "tls" ] tls = [] [package.metadata.deb] +name = "routinator" maintainer = "The NLnet Labs RPKI Team " license-file = ["LICENSE", "0"] extended-description = """\ @@ -65,7 +67,7 @@ statements about the association of Internet routing resources. \ In particular, it allows the holder of an IP address prefix to publish which \ AS number will be the origin of BGP route announcements for it. \ Routinator is a RPKI relying party software written in Rust. """ -depends = "$auto, rsync" +depends = "$auto, rsync, adduser" section = "net" priority = "optional" assets = [ @@ -73,20 +75,13 @@ assets = [ ["README.md", "usr/share/doc/routinator/", "644"], ["doc/misc.md", "usr/share/doc/routinator/misc.md", "644"], ["doc/routinator.1", "usr/share/man/man1/routinator.1", "644"], - ["etc/routinator.service", "lib/systemd/system/routinator.service", "644"], - ["etc/routinator.conf.system-service", "etc/routinator/routinator.conf", "644"] + ["etc/routinator.conf.system-service", "etc/routinator/routinator.conf", "644"], + ["debian/service.preset", "/lib/systemd/system-preset/50-routinator.preset", "644"] ] maintainer-scripts = "debian" -conf-files = [ - "etc/routinator/routinator.conf" -] +changelog = "debian/changelog" # this will be generated by the pkg workflow +copyright = "Copyright (c) 2019, NLnet Labs. All rights reserved." +conf-files = ["/etc/routinator/routinator.conf"] +systemd-units = { unit-name = "routinator", enable = false } [package.metadata.deb.variants.minimal] -assets = [ - ["target/release/routinator", "usr/bin/", "755"], - ["README.md", "usr/share/doc/routinator/", "644"], - ["doc/misc.md", "usr/share/doc/routinator/misc.md", "644"], - ["doc/routinator.1", "usr/share/man/man1/routinator.1", "644"], - ["etc/routinator.service.minimal", "lib/systemd/system/routinator.service", "644"], - ["etc/routinator.conf.system-service", "etc/routinator/routinator.conf", "644"] -] diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 0000000..321fd4f --- /dev/null +++ b/debian/postinst @@ -0,0 +1,17 @@ +#!/bin/sh -e + +ROUTINATOR_HOME="/var/lib/routinator/" +ROUTINATOR_USER="routinator" + +create_user() { + if id ${ROUTINATOR_USER} > /dev/null 2>&1; then return; fi + adduser --system --home "${ROUTINATOR_HOME}" --group ${ROUTINATOR_USER} +} + +case "$1" in +configure) + create_user + ;; +esac + +#DEBHELPER# \ No newline at end of file diff --git a/debian/postrm b/debian/postrm new file mode 100644 index 0000000..0fdb754 --- /dev/null +++ b/debian/postrm @@ -0,0 +1,16 @@ +#!/bin/sh -e + +ROUTINATOR_ETC="/etc/routinator" + +case "$1" in +purge) + # Per https://www.debian.org/doc/debian-policy/ch-files.html#behavior + # "configuration files must be preserved when the package is removed, and + # only deleted when the package is purged." + if [ -d ${ROUTINATOR_ETC} ]; then + rm -R ${ROUTINATOR_ETC} + fi + ;; +esac + +#DEBHELPER# \ No newline at end of file diff --git a/etc/routinator.service.minimal b/debian/routinator-minimal.routinator.service similarity index 82% rename from etc/routinator.service.minimal rename to debian/routinator-minimal.routinator.service index 457cdb0..21104f1 100644 --- a/etc/routinator.service.minimal +++ b/debian/routinator-minimal.routinator.service @@ -5,8 +5,7 @@ After=network.target [Service] ExecStart=/usr/bin/routinator --config=/etc/routinator/routinator.conf --syslog server +User=routinator [Install] -WantedBy=multi-user.target - - +WantedBy=multi-user.target \ No newline at end of file diff --git a/etc/routinator.service b/debian/routinator.routinator.service similarity index 100% rename from etc/routinator.service rename to debian/routinator.routinator.service diff --git a/debian/service.preset b/debian/service.preset new file mode 100644 index 0000000..08322c8 --- /dev/null +++ b/debian/service.preset @@ -0,0 +1 @@ +disable routinator.service diff --git a/etc/routinator.conf.system-service b/etc/routinator.conf.system-service index c0915f9..fcfda98 100644 --- a/etc/routinator.conf.system-service +++ b/etc/routinator.conf.system-service @@ -2,8 +2,8 @@ # ======================================================== # # This configuration assumes that the TALs are placed in -# /etc/routinator/tals and the repository is maintained in -# /var/lib/routinator/repository. +# /var/lib/routinator/tals and the repository cache is maintained in +# /var/lib/routinator/rpki-cache. # # It will start Routinator with an RTR server listening on port 3323 and # an HTTP server listening on port 8323. Both are limited to localhost by @@ -17,8 +17,7 @@ # example, see etc/routinator.conf.example in the source distribution or # consult the manual page. -repository-dir = "/var/lib/routinator" -tal-dir = "/etc/routinator/tals" +repository-dir = "/var/lib/routinator/rpki-cache" +tal-dir = "/var/lib/routinator/tals" rtr-listen = ["127.0.0.1:3323"] -http-listen = ["127.0.0.1:8323"] - +http-listen = ["127.0.0.1:8323"] \ No newline at end of file