diff --git a/.vscode/settings.json b/.vscode/settings.json index 3edcedc..e4829b0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,6 +23,9 @@ "[just]": { "editor.defaultFormatter": "TobiasHochguertel.just-formatter" }, + "[robotframework]": { + "editor.defaultFormatter": "d-biehl.robotcode", + }, "python-envs.defaultEnvManager": "ms-python.python:system", "coverage-gutters.showGutterCoverage": true, "coverage-gutters.showLineCoverage": true, diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 99be25a..360c0d7 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -2,7 +2,7 @@ "version": "2.0.0", "tasks": [ { - "label": "packtly_builder_tooling", + "label": "container: tooling just recipe", "type": "shell", "command": "just", "args": [ @@ -15,15 +15,7 @@ "problemMatcher": [] }, { - "label": "run robot tests", - "type": "shell", - "command": "${workspaceFolder}/test/robot/run.sh", - "problemMatcher": [ - "$python" - ] - }, - { - "label": "packtly_build_full", + "label": "container: tooling full", "type": "shell", "command": "/opt/venv/bin/python", "args": [ @@ -41,7 +33,7 @@ "problemMatcher": [] }, { - "label": "packtly_upload", + "label": "container: tooling upload", "type": "shell", "command": "/opt/venv/bin/python", "args": [ @@ -59,6 +51,26 @@ "cwd": "${workspaceFolder}/packtly-builder/tooling" }, "problemMatcher": [] + }, + { + "label": "host: run robot tests", + "type": "shell", + "command": "${workspaceFolder}/test/robot/run.sh", + "problemMatcher": [ + "$python" + ] + }, + { + "label": "host: run just recipe", + "type": "shell", + "command": "just", + "args": [ + "${input:justRecipeHost}" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [] } ], "inputs": [ @@ -77,6 +89,32 @@ "test_keys" ], "default": "default" + }, + { + "id": "justRecipeHost", + "type": "pickString", + "description": "Choose a top-level just recipe to run on the host", + "options": [ + "all", + "build-base", + "build-builder", + "build-builder-multiarch", + "build-devcontainer", + "build-runtime", + "build-runtime-multiarch", + "build-tooling", + "clean", + "clean-base", + "clean-builder", + "clean-containers", + "clean-devcontainer", + "clean-runtime", + "install-robocop", + "lint-robot", + "test-tooling", + "test-tooling-keys" + ], + "default": "default" } ] } diff --git a/justfile b/justfile index f7ca203..8984bac 100644 --- a/justfile +++ b/justfile @@ -214,6 +214,20 @@ shell: _require-tools exit "$rc" fi +# --- Robot Framework linting --- + +# Install robocop linter (own pipx venv, independent of the RF test runner) +install-robocop: + #!/usr/bin/env bash + set -euo pipefail + if ! pipx list --short | grep -q 'robotframework-robocop'; then + pipx install robotframework-robocop + fi + +# Lint Robot Framework files with robocop +lint-robot: install-robocop + robocop check test/robot + # --- Pipeline --- clean: _require-tools diff --git a/test/robot/keywords/aptly.robot b/test/robot/keywords/aptly.resource similarity index 55% rename from test/robot/keywords/aptly.robot rename to test/robot/keywords/aptly.resource index 253005b..b3ad4c1 100644 --- a/test/robot/keywords/aptly.robot +++ b/test/robot/keywords/aptly.resource @@ -1,10 +1,12 @@ *** Settings *** +Documentation Keywords for generating Aptly credentials used in integration tests. Library OperatingSystem + *** Keywords *** Generate Aptly Credentials - [Arguments] - ... ${path} + [Documentation] Generate a simple aptly credentials file + [Arguments] ${path} ... ${username}=admin ... ${password}=password diff --git a/test/robot/keywords/git.robot b/test/robot/keywords/git.resource similarity index 71% rename from test/robot/keywords/git.robot rename to test/robot/keywords/git.resource index 3340f17..250834e 100644 --- a/test/robot/keywords/git.robot +++ b/test/robot/keywords/git.resource @@ -1,8 +1,11 @@ *** Settings *** +Documentation Keywords for interacting with Git repositories in integration tests. Library Process + *** Keywords *** Checkout Repository + [Documentation] Checkout a git repository to a given path. [Arguments] ${url} ${path} ${branch}=main ${result}= Run Process diff --git a/test/robot/keywords/gpg.resource b/test/robot/keywords/gpg.resource new file mode 100644 index 0000000..51412b7 --- /dev/null +++ b/test/robot/keywords/gpg.resource @@ -0,0 +1,69 @@ +*** Settings *** +Documentation Keywords for generating GPG key pairs used to sign and verify Debian packages in integration tests. +Library OperatingSystem +Library Process +Library String + + +*** Variables *** +${GEN_GPG_SCRIPT} ${EMPTY} +${GPG_NAME} packtly-test +${GPG_EMAIL} test@packtly.local +${GPG_PASS} test1234 + + +*** Keywords *** +Get Gen GPG Script + [Documentation] Return the absolute path to the gen_gpg helper script in the repository. + ${result}= Run Process git rev-parse --show-toplevel + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + ${topdir}= Strip String ${result.stdout} + RETURN ${topdir}/packtly-builder/containers/scripts/gen_gpg + +Generate GPG Keys + [Documentation] Generate a GPG key pair inside the build container and export its files. + ... The public and private keys are written to ${keys_dir}. + [Arguments] ${keys_dir} + ... ${image} + + VAR ${gpg_out} ${keys_dir}/_gpg_out + ${gen_gpg}= Get Gen GPG Script + + Prepare GPG Key Directories ${keys_dir} ${gpg_out} + Run GPG Generator ${gen_gpg} ${gpg_out} ${image} + Export GPG Keys ${gpg_out} ${keys_dir} + Remove Directory ${gpg_out} recursive=True + +Prepare GPG Key Directories + [Documentation] Create the public, private, and temporary GPG key directories. + [Arguments] ${keys_dir} ${gpg_out} + + Create Directory ${keys_dir}/public + Create Directory ${keys_dir}/private + Create Directory ${gpg_out} + +Run GPG Generator + [Documentation] Run the GPG generation helper in the specified build container. + [Arguments] ${gen_gpg} ${gpg_out} ${image} + + ${result}= Run Process + ... podman run --rm + ... --entrypoint /usr/local/bin/gen_gpg + ... -v ${gen_gpg}:/usr/local/bin/gen_gpg:Z,ro + ... -v ${gpg_out}:/opt/keys/gpg:Z + ... ${image} + ... --name ${GPG_NAME} + ... --email ${GPG_EMAIL} + ... --pass ${GPG_PASS} + + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 ${result.stderr} + +Export GPG Keys + [Documentation] Move generated GPG key material from the temporary directory to ${keys_dir}. + [Arguments] ${gpg_out} ${keys_dir} + + Move File ${gpg_out}/repo_signing.key ${keys_dir}/public/repo_signing.key + Move File ${gpg_out}/repo_signing_private.key ${keys_dir}/private/repo_signing_private.key + Move File ${gpg_out}/repo_signing_private_pass ${keys_dir}/private/repo_signing_private_pass diff --git a/test/robot/keywords/gpg.robot b/test/robot/keywords/gpg.robot deleted file mode 100644 index 74c339a..0000000 --- a/test/robot/keywords/gpg.robot +++ /dev/null @@ -1,48 +0,0 @@ -*** Settings *** -Library Process -Library String -Library OperatingSystem - -*** Variables *** -${GEN_GPG_SCRIPT} ${EMPTY} - -*** Keywords *** -Get Gen GPG Script - ${result}= Run Process git rev-parse --show-toplevel - Should Be Equal As Integers ${result.rc} 0 ${result.stderr} - ${topdir}= Strip String ${result.stdout} - RETURN ${topdir}/packtly-builder/containers/scripts/gen_gpg - -Generate GPG Keys - [Arguments] - ... ${keys_dir} - ... ${image} - ... ${name}=packtly-test - ... ${email}=test@packtly.local - ... ${pass}=test1234 - - ${gpg_out}= Set Variable ${keys_dir}/_gpg_out - ${gen_gpg}= Get Gen GPG Script - - Create Directory ${keys_dir}/public - Create Directory ${keys_dir}/private - Create Directory ${gpg_out} - - ${result}= Run Process - ... podman run --rm - ... --entrypoint /usr/local/bin/gen_gpg - ... -v ${gen_gpg}:/usr/local/bin/gen_gpg:Z,ro - ... -v ${gpg_out}:/opt/keys/gpg:Z - ... ${image} - ... --name ${name} - ... --email ${email} - ... --pass ${pass} - ... stdout=PIPE stderr=PIPE - - Should Be Equal As Integers ${result.rc} 0 ${result.stderr} - - Move File ${gpg_out}/repo_signing.key ${keys_dir}/public/repo_signing.key - Move File ${gpg_out}/repo_signing_private.key ${keys_dir}/private/repo_signing_private.key - Move File ${gpg_out}/repo_signing_private_pass ${keys_dir}/private/repo_signing_private_pass - - Remove Directory ${gpg_out} recursive=True diff --git a/test/robot/suites/debhello_quilt.robot b/test/robot/suites/debhello_quilt.robot index 9bbcd1c..08df645 100644 --- a/test/robot/suites/debhello_quilt.robot +++ b/test/robot/suites/debhello_quilt.robot @@ -1,65 +1,108 @@ *** Settings *** +Documentation Integration tests for building and verifying a debhello quilt-format Debian package. +... The package is built with packtly-builder. Library OperatingSystem Library Process -Resource ../keywords/git.robot -Resource ../keywords/gpg.robot -Resource ../keywords/aptly.robot +Resource ../keywords/git.resource +Resource ../keywords/gpg.resource +Resource ../keywords/aptly.resource Suite Setup Prepare Workspace -Suite Teardown Run Keyword If '${WORKSPACE}' != '' Remove Directory ${WORKSPACE} recursive=True +Suite Teardown Cleanup Workspace *** Variables *** ${FIXTURES_DIR} ${CURDIR}/../../fixtures ${DEBHELLO_DIR} ${FIXTURES_DIR}/debhello-quilt ${CONTAINER_IMAGE} packtly-builder:latest -${APTLY_CREDENTIALS} ${EMPTY} -${WORKSPACE} ${EMPTY} -${KEYS_DIR} ${EMPTY} +${DEBIAN_IMAGE} debian:trixie +${WORKSPACE} ${TEMPDIR}/debhello-quilt-build +${KEYS_DIR} ${WORKSPACE}/keys +${APTLY_CREDENTIALS} ${WORKSPACE}/aptly-credentials + + +*** Test Cases *** +Debhello Quilt Build + [Documentation] Test building debhello package with packtly-builder + [Tags] build + Run Debhello Quilt Build + +Binary Packages Are Valid + [Documentation] Verify .changes and .deb artifacts were produced and are structurally valid. + [Tags] build binary + Verify Build Artifacts ${WORKSPACE} ${CONTAINER_IMAGE} + +Changes File Is Signed + [Documentation] Verify the .changes file carries a valid GPG signature. + [Tags] build signature + Verify Changes Signature ${WORKSPACE} ${KEYS_DIR} ${CONTAINER_IMAGE} + +Binary Package Is Installable + [Documentation] Verify each produced .deb can be installed with dpkg on a clean Debian image. + [Tags] install binary + Verify Package Is Installable ${WORKSPACE} ${DEBIAN_IMAGE} + +Source Package Is Valid + [Documentation] Verify the .dsc and referenced source tarballs pass dpkg-source integrity check. + [Tags] build source + Verify Source Package Artifacts ${WORKSPACE} ${CONTAINER_IMAGE} + +DSC File Is Signed + [Documentation] Verify the .dsc file carries a valid GPG signature. + [Tags] build signature source + Verify DSC Signature ${WORKSPACE} ${KEYS_DIR} ${CONTAINER_IMAGE} + +Source Package Is Installable + [Documentation] Verify the source package (.dsc + tarballs) can be extracted by dpkg-source. + ... The check runs in a clean Debian image. + [Tags] install source + Verify Source Package Artifacts ${WORKSPACE} ${DEBIAN_IMAGE} + *** Keywords *** +Cleanup Workspace + [Documentation] Cleanup the temporary workspace after the debhello-quilt build test. + IF '${WORKSPACE}' != '' + Remove Directory ${WORKSPACE} recursive=True + END + Prepare Workspace - Set Suite Variable ${WORKSPACE} ${TEMPDIR}/debhello-quilt-build - Set Suite Variable ${KEYS_DIR} ${WORKSPACE}/keys - Set Suite Variable ${APTLY_CREDENTIALS} ${WORKSPACE}/aptly-credentials + [Documentation] Prepare a temporary workspace for the debhello-quilt build test. + Run Process git clean -fdx cwd=${FIXTURES_DIR} Create Directory ${WORKSPACE} Create Directory ${WORKSPACE}/logs Generate GPG Keys ${KEYS_DIR} ${CONTAINER_IMAGE} Generate Aptly Credentials ${APTLY_CREDENTIALS} + Copy Directory ${DEBHELLO_DIR} ${WORKSPACE}/debhello-quilt Run Debhello Quilt Build - [Arguments] - ... ${workspace} - ... ${keys_dir} - ... ${aptly_credentials} - ... ${image} - ... ${dist}=trixie-apollo + [Documentation] packtly-builder build of the debhello-quilt package inside a container. + [Arguments] ${dist}=trixie-apollo ... ${component}=main ${result}= Run Process ... podman run --rm - ... -v ${workspace}:/workspace:Z - ... -v ${keys_dir}/public/repo_signing.key:/opt/keys/gpg/repo_signing.key:Z,ro - ... -v ${keys_dir}/private/repo_signing_private.key:/opt/keys/gpg/repo_signing_private.key:Z,ro - ... -v ${keys_dir}/private/repo_signing_private_pass:/opt/keys/gpg/repo_signing_private_pass:Z,ro - ... -v ${aptly_credentials}:/run/secrets/aptly-credentials:Z,ro - ... -v ${workspace}/logs:/logs:Z + ... -v ${WORKSPACE}:/workspace:Z + ... -v ${KEYS_DIR}/public/repo_signing.key:/opt/keys/gpg/repo_signing.key:Z,ro + ... -v ${KEYS_DIR}/private/repo_signing_private.key:/opt/keys/gpg/repo_signing_private.key:Z,ro + ... -v ${KEYS_DIR}/private/repo_signing_private_pass:/opt/keys/gpg/repo_signing_private_pass:Z,ro + ... -v ${APTLY_CREDENTIALS}:/run/secrets/aptly-credentials:Z,ro + ... -v ${WORKSPACE}/logs:/logs:Z ... -e APTLYHOST\=http://localhost:8080 ... --network\=host - ... ${image} + ... ${CONTAINER_IMAGE} ... /workspace/debhello-quilt ... --log-file /logs/build.log ... --dist ${dist} ... --component ${component} ... --build-mode full - Log File ${workspace}/logs/build.log - Copy File ${workspace}/logs/build.log ${CURDIR}/../results/debhello_build.log + Log File ${WORKSPACE}/logs/build.log + Copy File ${WORKSPACE}/logs/build.log ${CURDIR}/../results/debhello_build.log Should Be Equal As Integers ${result.rc} 0 ... Build failed with rc ${result.rc}. See stdout/stderr and build.log above. - Verify Build Artifacts [Documentation] Verify a successful build produced valid .changes and .deb artifacts. [Arguments] ${workspace} ${image} @@ -84,14 +127,16 @@ Verify Build Artifacts END Verify Changes Signature + [Documentation] Verify the .changes file carries a valid GPG signature. [Arguments] ${workspace} ${keys_dir} ${image} ${changes}= List Files In Directory ${workspace} *.changes absolute=True Length Should Be ${changes} 1 - ${changes_file}= Set Variable ${changes[0].rsplit('/',1)[1]} - ${cmd}= Set Variable - ... gpg --batch --import /tmp/repo_signing.key >/dev/null 2>&1 && gpg --status-fd\=1 --verify /workspace/${changes_file} + VAR ${changes_file} ${changes[0].rsplit('/',1)[1]} + ${cmd}= Catenate SEPARATOR=${SPACE} + ... gpg --batch --import /tmp/repo_signing.key >/dev/null 2>&1 + ... && gpg --status-fd\=1 --verify /workspace/${changes_file} ${result}= Run Process ... podman run --rm ... -v ${workspace}:/workspace:Z,ro @@ -107,20 +152,47 @@ Verify Changes Signature Should Contain ${result.stdout} [GNUPG:] GOODSIG Should Contain ${result.stdout} [GNUPG:] VALIDSIG +Verify Package Is Installable + [Documentation] Verify each produced .deb can be installed with dpkg inside the build container. + [Arguments] ${workspace} ${image} + + ${debs}= List Files In Directory ${workspace} *.deb absolute=True + Should Not Be Empty ${debs} No .deb packages found to install + + FOR ${deb} IN @{debs} + VAR ${deb_name} ${deb.rsplit('/', 1)[1]} + ${result}= Run Process + ... podman run --rm + ... --entrypoint sh + ... -v ${workspace}:/workspace:Z,ro + ... ${image} + ... -c dpkg --install --force-depends /workspace/${deb_name} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + ... dpkg --install failed for ${deb_name}: ${result.stderr} + END + Verify Source Package Artifacts - [Documentation] Verify source package artifacts (.dsc + referenced tarballs) exist and pass dpkg-source integrity check. + [Documentation] Verify source package artifacts exist and pass dpkg-source integrity check. + ... Artifacts include the .dsc file and its referenced tarballs. [Arguments] ${workspace} ${image} ${dscs}= List Files In Directory ${workspace} *.dsc absolute=True Should Not Be Empty ${dscs} No .dsc file produced - source package missing - ${dsc_file}= Set Variable ${dscs[0].rsplit('/', 1)[1]} + VAR ${dsc_file} ${dscs[0].rsplit('/', 1)[1]} + ${cmd}= Catenate SEPARATOR=${SPACE} + ... command -v dpkg-source || apt-get update -qq + ... && apt-get install -y --no-install-recommends dpkg-dev + ... && dpkg-source -x /workspace/${dsc_file} /tmp/src-check + ... && rm -rf /tmp/src-check ${result}= Run Process ... podman run --rm ... --entrypoint sh ... -v ${workspace}:/workspace:Z,ro ... ${image} - ... -c dpkg-source -x /workspace/${dsc_file} /tmp/src-check && rm -rf /tmp/src-check + ... -c ${cmd} Log ${result.stdout} Log ${result.stderr} @@ -128,14 +200,16 @@ Verify Source Package Artifacts ... dpkg-source -x failed for ${dsc_file}: ${result.stderr} Verify DSC Signature + [Documentation] Verify the .dsc file carries a valid GPG signature. [Arguments] ${workspace} ${keys_dir} ${image} ${dscs}= List Files In Directory ${workspace} *.dsc absolute=True Length Should Be ${dscs} 1 - ${dsc_file}= Set Variable ${dscs[0].rsplit('/', 1)[1]} - ${cmd}= Set Variable - ... gpg --batch --import /tmp/repo_signing.key >/dev/null 2>&1 && gpg --status-fd\=1 --verify /workspace/${dsc_file} + VAR ${dsc_file} ${dscs[0].rsplit('/', 1)[1]} + ${cmd}= Catenate SEPARATOR=${SPACE} + ... gpg --batch --import /tmp/repo_signing.key >/dev/null 2>&1 + ... && gpg --status-fd\=1 --verify /workspace/${dsc_file} ${result}= Run Process ... podman run --rm ... -v ${workspace}:/workspace:Z,ro @@ -150,36 +224,3 @@ Verify DSC Signature Should Be Equal As Integers ${result.rc} 0 GPG verification of .dsc failed! Check logs. Should Contain ${result.stdout} [GNUPG:] GOODSIG Should Contain ${result.stdout} [GNUPG:] VALIDSIG - - -*** Test Cases *** - -Debhello Quilt Build - [Documentation] Test building debhello package with packtly-builder - [Tags] debhello build - Copy Directory ${FIXTURES_DIR}/debhello-quilt ${WORKSPACE}/debhello-quilt - Run Debhello Quilt Build - ... ${WORKSPACE} - ... ${KEYS_DIR} - ... ${APTLY_CREDENTIALS} - ... image=${CONTAINER_IMAGE} - -Binary Packages Are Valid - [Documentation] Verify .changes and .deb artifacts were produced and are structurally valid. - [Tags] debhello build binary - Verify Build Artifacts ${WORKSPACE} ${CONTAINER_IMAGE} - -Changes File Is Signed - [Documentation] Verify the .changes file carries a valid GPG signature. - [Tags] debhello build signature - Verify Changes Signature ${WORKSPACE} ${KEYS_DIR} ${CONTAINER_IMAGE} - -Source Package Is Valid - [Documentation] Verify the .dsc and referenced source tarballs pass dpkg-source integrity check. - [Tags] debhello build source - Verify Source Package Artifacts ${WORKSPACE} ${CONTAINER_IMAGE} - -DSC File Is Signed - [Documentation] Verify the .dsc file carries a valid GPG signature. - [Tags] debhello build signature source - Verify DSC Signature ${WORKSPACE} ${KEYS_DIR} ${CONTAINER_IMAGE}