nginx-systemd units: fix service start and socket removal, so CI runs test_nginx_dispatch

.github/workflows/ci.yml

@@ -152,17 +152,15 @@ jobs:
 
         # Create borgstore system user (matches contrib/server/nginx-systemd/borgstore@.service)
         sudo useradd --system --home /srv/borgstore --shell /usr/sbin/nologin borgstore || true
-        # Add the web server user (www-data) to the borgstore group so it can connect
-        # to the unix sockets (which are owned by borgstore:borgstore or borgstore:www-data).
-        sudo usermod -aG borgstore www-data || true
 
-        # Install borgstore[rest] to the system Python so the borgstore service user can run it.
+        # Install borgstore[rest,blake3] to the system Python so the borgstore service user can run it
+        # (blake3: the server computes the hashes, test_hash_blake3 also runs on these backends).
         # Must NOT use -e (editable): the service runs with ProtectHome=true, which makes
         # /home inaccessible. An editable install leaves a .pth pointing to the workspace
         # under /home/runner/work/, so the import would fail at service start. A regular
-        # install copies the package into system site-packages (/usr/lib/python3/...) which
-        # is always accessible.
-        sudo pip3 install --break-system-packages ".[rest]"
+        # install copies the package to /usr/local/lib/python3.*/dist-packages and the
+        # borgstore-server-rest script to /usr/local/bin, which are always accessible.
+        sudo pip3 install --break-system-packages ".[rest,blake3]"
 
         # Running pip3 as root may have changed ownership of src, so we fix that.
         sudo chown -R $USER:$USER src/
@@ -200,6 +198,17 @@ jobs:
         sudo nginx -t
         sudo systemctl restart nginx
 
+        # Create and destroy a store in both repos via nginx. If that does not work, fail here:
+        # the tests would just silently skip the nginx-proxied REST backends.
+        rest() {
+          curl -fsS --max-time 30 -o /dev/null -w "$1 $2: HTTP %{http_code}\n" -X "$1" \
+            -H "Accept: application/vnd.x.borgstore.rest.v1" "http://testuser:testpass@localhost$2"
+        }
+        for repo in repo1 repo2; do
+          rest POST "/repos/$repo/?cmd=create"
+          rest DELETE "/repos/$repo/?cmd=destroy"
+        done
+
         # Export REST test URLs for tox via GITHUB_ENV
         echo "BORGSTORE_TEST_REST1_URL=http://testuser:testpass@localhost/repos/repo1/" >> $GITHUB_ENV
         echo "BORGSTORE_TEST_REST2_URL=http://testuser:testpass@localhost/repos/repo2/" >> $GITHUB_ENV

contrib/server/nginx-systemd/README.md

@@ -86,6 +86,11 @@ nginx -t && nginx -s reload
 - The borgstore process is started on the first connection and stays running
   while connections are open. Add `TimeoutStopSec=` to the service unit to
   shut it down after a period of inactivity.
-- The socket file at `/run/borgstore/<name>.sock` is recreated automatically
-  after a reboot by systemd (`RuntimeDirectory=borgstore` in the service unit).
+- The socket file at `/run/borgstore/<name>.sock` (and the `/run/borgstore/`
+  directory, if missing) is recreated automatically after a reboot by systemd
+  when it starts the enabled socket unit.
+- The service unit runs `borgstore-server-rest` without an absolute path, so
+  systemd finds it in `/usr/local/bin` (`pip install "borgstore[rest]"` as root)
+  or `/usr/bin` (distribution package). For an install in a virtualenv, put the
+  absolute path into `ExecStart=`.
 - TLS is handled entirely by nginx; the borgstore process never sees HTTPS.

contrib/server/nginx-systemd/borgstore@.service

@@ -27,7 +27,10 @@ Group=borgstore
 # Minimum required: BORGSTORE_BACKEND, BORGSTORE_USERNAME, BORGSTORE_PASSWORD
 EnvironmentFile=/etc/borgstore/%i.env
 
-ExecStart=/usr/bin/borgstore-server-rest \
+# No absolute path here: systemd looks the executable up in its search path,
+# which covers /usr/local/bin ("pip install" as root) as well as /usr/bin (a
+# distribution package). Use an absolute path for an install in a virtualenv.
+ExecStart=borgstore-server-rest \
     --backend ${BORGSTORE_BACKEND} \
     --username ${BORGSTORE_USERNAME} \
     --password ${BORGSTORE_PASSWORD} \
@@ -48,11 +51,9 @@ ProtectSystem=strict
 ProtectHome=true
 ReadWritePaths=/srv/borgstore
 PrivateTmp=true
-# Keep /run/borgstore/ alive for the lifetime of the service.
-# The socket unit also declares RuntimeDirectory=borgstore, which creates
-# the directory (with correct ownership) before the socket is bound.
-RuntimeDirectory=borgstore
-RuntimeDirectoryMode=0755
+# No RuntimeDirectory=borgstore here: /run/borgstore/ is shared by the sockets
+# of all instances, and systemd removes a runtime directory (with all the
+# sockets in it) as soon as the one instance declaring it stops or fails.
 
 [Install]
 # Not enabled directly; the .socket unit triggers this.

contrib/server/nginx-systemd/borgstore@.socket

@@ -4,8 +4,8 @@
 # encodes the repo name, and nginx proxies to it directly.
 #
 # Socket path: /run/borgstore/%i.sock
-# RuntimeDirectory= below ensures /run/borgstore/ exists before the socket
-# is created (required because the socket unit starts before the service unit).
+# systemd creates the missing parent directory /run/borgstore/ (root-owned,
+# mode 0755, see DirectoryMode=) when it binds the socket.
 #
 # Enable for a repo:
 #   systemctl enable --now borgstore@repo1.socket
@@ -22,11 +22,9 @@ SocketUser=borgstore
 SocketGroup=www-data
 SocketMode=0660
 Accept=false
-# Create /run/borgstore/ before binding the socket.
-# This must be here (not only in the service unit) because the socket
-# unit starts before the service unit activates.
-RuntimeDirectory=borgstore
-RuntimeDirectoryMode=0755
+# No RuntimeDirectory=borgstore here: /run/borgstore/ is shared by the sockets
+# of all instances, and systemd removes a runtime directory (with all the
+# sockets in it) as soon as the one instance declaring it stops.
 
 [Install]
 WantedBy=sockets.target