From 9e73a94fe66f39e764de7e5b7fc2aad1914c1dcb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 20 Sep 2026 00:16:49 +0200 Subject: [PATCH 1/2] nginx-systemd units: fix service start and socket removal, so CI runs test_nginx_dispatch test_nginx_dispatch (and all generic backend tests for the 2 nginx-proxied REST backends) were silently skipped in all Linux CI jobs, because the REST servers behind nginx never worked there: - borgstore@.service had ExecStart=/usr/bin/borgstore-server-rest, but "pip install" as root puts the script into /usr/local/bin on Debian/Ubuntu, so the service failed with status=203/EXEC in a restart loop and the first request hung until the client timed out. Give the executable without a path, systemd then finds it in /usr/local/bin or /usr/bin. - both units declared RuntimeDirectory=borgstore. /run/borgstore/ is shared by the sockets of all instances, but systemd removes a runtime directory when the unit declaring it stops or fails - together with the sockets of all other instances (nginx: "connect() to unix:/run/borgstore/repo1.sock failed (2: No such file or directory)"). Also, starting a service chown'ed the directory recursively to borgstore:borgstore, so the sockets lost SocketGroup=www-data. systemd creates the parent directory of a ListenStream= socket by itself, so just do not declare a RuntimeDirectory. CI: - create and destroy a store in both repos via nginx at the end of the setup step, so a broken setup fails the job instead of skipping the tests. - do not add www-data to the borgstore group: not needed, the sockets now keep their www-data group. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 18 +++++++++++++----- contrib/server/nginx-systemd/README.md | 9 +++++++-- .../server/nginx-systemd/borgstore@.service | 13 +++++++------ contrib/server/nginx-systemd/borgstore@.socket | 12 +++++------- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 216473c..ba7dcec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,16 +152,13 @@ jobs: # Create borgstore system user (matches contrib/server/nginx-systemd/borgstore@.service) sudo useradd --system --home /srv/borgstore --shell /usr/sbin/nologin borgstore || true - # Add the web server user (www-data) to the borgstore group so it can connect - # to the unix sockets (which are owned by borgstore:borgstore or borgstore:www-data). - sudo usermod -aG borgstore www-data || true # Install borgstore[rest] to the system Python so the borgstore service user can run it. # Must NOT use -e (editable): the service runs with ProtectHome=true, which makes # /home inaccessible. An editable install leaves a .pth pointing to the workspace # under /home/runner/work/, so the import would fail at service start. A regular - # install copies the package into system site-packages (/usr/lib/python3/...) which - # is always accessible. + # install copies the package to /usr/local/lib/python3.*/dist-packages and the + # borgstore-server-rest script to /usr/local/bin, which are always accessible. sudo pip3 install --break-system-packages ".[rest]" # Running pip3 as root may have changed ownership of src, so we fix that. @@ -200,6 +197,17 @@ jobs: sudo nginx -t sudo systemctl restart nginx + # Create and destroy a store in both repos via nginx. If that does not work, fail here: + # the tests would just silently skip the nginx-proxied REST backends. + rest() { + curl -fsS --max-time 30 -o /dev/null -w "$1 $2: HTTP %{http_code}\n" -X "$1" \ + -H "Accept: application/vnd.x.borgstore.rest.v1" "http://testuser:testpass@localhost$2" + } + for repo in repo1 repo2; do + rest POST "/repos/$repo/?cmd=create" + rest DELETE "/repos/$repo/?cmd=destroy" + done + # Export REST test URLs for tox via GITHUB_ENV echo "BORGSTORE_TEST_REST1_URL=http://testuser:testpass@localhost/repos/repo1/" >> $GITHUB_ENV echo "BORGSTORE_TEST_REST2_URL=http://testuser:testpass@localhost/repos/repo2/" >> $GITHUB_ENV diff --git a/contrib/server/nginx-systemd/README.md b/contrib/server/nginx-systemd/README.md index 3e4d52f..ba38804 100644 --- a/contrib/server/nginx-systemd/README.md +++ b/contrib/server/nginx-systemd/README.md @@ -86,6 +86,11 @@ nginx -t && nginx -s reload - The borgstore process is started on the first connection and stays running while connections are open. Add `TimeoutStopSec=` to the service unit to shut it down after a period of inactivity. -- The socket file at `/run/borgstore/.sock` is recreated automatically - after a reboot by systemd (`RuntimeDirectory=borgstore` in the service unit). +- The socket file at `/run/borgstore/.sock` (and the `/run/borgstore/` + directory, if missing) is recreated automatically after a reboot by systemd + when it starts the enabled socket unit. +- The service unit runs `borgstore-server-rest` without an absolute path, so + systemd finds it in `/usr/local/bin` (`pip install "borgstore[rest]"` as root) + or `/usr/bin` (distribution package). For an install in a virtualenv, put the + absolute path into `ExecStart=`. - TLS is handled entirely by nginx; the borgstore process never sees HTTPS. diff --git a/contrib/server/nginx-systemd/borgstore@.service b/contrib/server/nginx-systemd/borgstore@.service index 9ee0b03..b85fcd6 100644 --- a/contrib/server/nginx-systemd/borgstore@.service +++ b/contrib/server/nginx-systemd/borgstore@.service @@ -27,7 +27,10 @@ Group=borgstore # Minimum required: BORGSTORE_BACKEND, BORGSTORE_USERNAME, BORGSTORE_PASSWORD EnvironmentFile=/etc/borgstore/%i.env -ExecStart=/usr/bin/borgstore-server-rest \ +# No absolute path here: systemd looks the executable up in its search path, +# which covers /usr/local/bin ("pip install" as root) as well as /usr/bin (a +# distribution package). Use an absolute path for an install in a virtualenv. +ExecStart=borgstore-server-rest \ --backend ${BORGSTORE_BACKEND} \ --username ${BORGSTORE_USERNAME} \ --password ${BORGSTORE_PASSWORD} \ @@ -48,11 +51,9 @@ ProtectSystem=strict ProtectHome=true ReadWritePaths=/srv/borgstore PrivateTmp=true -# Keep /run/borgstore/ alive for the lifetime of the service. -# The socket unit also declares RuntimeDirectory=borgstore, which creates -# the directory (with correct ownership) before the socket is bound. -RuntimeDirectory=borgstore -RuntimeDirectoryMode=0755 +# No RuntimeDirectory=borgstore here: /run/borgstore/ is shared by the sockets +# of all instances, and systemd removes a runtime directory (with all the +# sockets in it) as soon as the one instance declaring it stops or fails. [Install] # Not enabled directly; the .socket unit triggers this. diff --git a/contrib/server/nginx-systemd/borgstore@.socket b/contrib/server/nginx-systemd/borgstore@.socket index 2465d2b..99b815e 100644 --- a/contrib/server/nginx-systemd/borgstore@.socket +++ b/contrib/server/nginx-systemd/borgstore@.socket @@ -4,8 +4,8 @@ # encodes the repo name, and nginx proxies to it directly. # # Socket path: /run/borgstore/%i.sock -# RuntimeDirectory= below ensures /run/borgstore/ exists before the socket -# is created (required because the socket unit starts before the service unit). +# systemd creates the missing parent directory /run/borgstore/ (root-owned, +# mode 0755, see DirectoryMode=) when it binds the socket. # # Enable for a repo: # systemctl enable --now borgstore@repo1.socket @@ -22,11 +22,9 @@ SocketUser=borgstore SocketGroup=www-data SocketMode=0660 Accept=false -# Create /run/borgstore/ before binding the socket. -# This must be here (not only in the service unit) because the socket -# unit starts before the service unit activates. -RuntimeDirectory=borgstore -RuntimeDirectoryMode=0755 +# No RuntimeDirectory=borgstore here: /run/borgstore/ is shared by the sockets +# of all instances, and systemd removes a runtime directory (with all the +# sockets in it) as soon as the one instance declaring it stops. [Install] WantedBy=sockets.target From 0484ef4e3e6b53aace6f3860313c21bfcd9566a1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 20 Sep 2026 00:23:32 +0200 Subject: [PATCH 2/2] CI: install blake3 for the nginx-proxied REST servers They run in the system Python and compute the hashes server-side, so test_hash_blake3 failed on the rest1/rest2 backends as soon as these backends were not skipped any more. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba7dcec..afae53b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,13 +153,14 @@ jobs: # Create borgstore system user (matches contrib/server/nginx-systemd/borgstore@.service) sudo useradd --system --home /srv/borgstore --shell /usr/sbin/nologin borgstore || true - # Install borgstore[rest] to the system Python so the borgstore service user can run it. + # Install borgstore[rest,blake3] to the system Python so the borgstore service user can run it + # (blake3: the server computes the hashes, test_hash_blake3 also runs on these backends). # Must NOT use -e (editable): the service runs with ProtectHome=true, which makes # /home inaccessible. An editable install leaves a .pth pointing to the workspace # under /home/runner/work/, so the import would fail at service start. A regular # install copies the package to /usr/local/lib/python3.*/dist-packages and the # borgstore-server-rest script to /usr/local/bin, which are always accessible. - sudo pip3 install --break-system-packages ".[rest]" + sudo pip3 install --break-system-packages ".[rest,blake3]" # Running pip3 as root may have changed ownership of src, so we fix that. sudo chown -R $USER:$USER src/