feat: the RTE talks to the app over a WebSocket, without database credentials

.env

@@ -36,8 +36,8 @@ LOG_CONSOLE_LEVEL=info
 # Password of the MySQL root user, for MySQL itself: AnsibleForms does not connect as root
 MYSQLDB_ROOT_PASSWORD=AnsibleForms!123
 
-# The user AnsibleForms (the app and the RTE) connects with. MySQL creates it on its first
-# start with all rights on the AnsibleForms schema and none elsewhere. Not root.
+# The user the app connects with (the RTE holds no database credentials). MySQL creates it on its
+# first start with all rights on the AnsibleForms schema and none elsewhere. Not root.
 MYSQLDB_USER=ansibleforms
 MYSQLDB_PASSWORD=AnsibleForms!123
 
@@ -172,12 +172,26 @@ ADMIN_PASSWORD=AnsibleForms!123
 # REST_ALLOWED_HOSTS=
 # REST_DENIED_HOSTS=
 
+# How long a call of the fn.fnRest* helper functions may wait for an answer, in seconds
+# Default: 60
+# REST_TIMEOUT_SECONDS=60
+
+# Skip the certificate check of the fn.fnRest* helper functions, for self-signed endpoints
+# (0=check, 1=skip). Before 7.3 they always skipped it.
+# Default: 0
+# REST_IGNORE_CERTS=0
+
+# How long a jq filter (fnJq, the jq of the REST, file and credential functions and of the
+# database queries) may run before it is stopped, in seconds
+# Default: 30
+# JQ_TIMEOUT_SECONDS=30
+
 # =============================================================================
 # RTE (RUNTIME ENVIRONMENT)
 # =============================================================================
 
-# Token the app sends with every call to the RTE, at least 16 characters. Both containers
-# read it from here, and data/seed.yaml hands it to the app's runner on every start.
+# Token the app dials the RTE with over its WebSocket, at least 16 characters. Both containers
+# read it from here: data/seed.yaml puts it in the credential of the app's rte runner on every start.
 RTE_TOKEN=AnsibleForms!RTE!Token!123
 
 # =============================================================================

.github/workflows/ci.yml

@@ -77,7 +77,7 @@ jobs:
           runner="$(curl -fsSk "${auth[@]}" "$URL/api/v2/runner" | jq -c '.records[] | select(.type == "rte" and .is_default == 1)')"
           [ -n "$runner" ] || { echo "::error::data/seed.yaml registered no default rte runner."; exit 1; }
           echo "Runner: $runner"
-          # the RTE starts once the app is healthy, so give it a moment to listen
+          # the RTE starts beside the app, and the app dials it once both are up, so give it a moment
           check=""
           for i in $(seq 1 30); do
             check="$(curl -fsSk "${auth[@]}" -X POST "$URL/api/v2/runner/$(jq -r .id <<<"$runner")/check" || true)"

.gitignore

@@ -17,6 +17,9 @@ data/forms_backups/
 data/forms_staging/
 data/uploads/
 data/python/
+# The RTE's job spool and the files cache of RTE_FILES=transfer (created in either mode)
+data/rte-spool/
+data/rte-files/
 data/ansibleForms.lock
 
 # Editor and OS files

README.md

@@ -49,7 +49,7 @@ Everything under `data/` is mounted into the containers and survives a restart o
 
 ## Database
 
-The app and the RTE connect to MySQL as `ansibleforms` (`MYSQLDB_USER` in `.env`), with all rights on the `AnsibleForms`
+The app connects to MySQL as `ansibleforms` (`MYSQLDB_USER` in `.env`), with all rights on the `AnsibleForms`
 schema and none elsewhere: a hole in the app then stays a hole in its own data. Root (`MYSQLDB_ROOT_PASSWORD`) is for MySQL.
 
 MySQL creates that user on its first start only. An install that has connected as root so far keeps its database, so
@@ -64,7 +64,8 @@ docker compose exec mysqldb mysql -uroot -p -e "CREATE USER 'ansibleforms'@'%' I
 ## Running playbooks
 
 AnsibleForms 7 runs no playbook in the app: the `rte` container runs them, from
-`ghcr.io/ansibleforms/ansibleforms-rte-full:7`. The app reaches it on `http://rte:8000` with `RTE_TOKEN` from `.env`.
+`ghcr.io/ansibleforms/ansibleforms-rte-full:7`. The app dials it on `http://rte:8000` with `RTE_TOKEN` from `.env` and keeps
+one WebSocket open (`RTE_MODE=inbound`): the RTE gets no database credentials, and is not published on the host.
 
 `data/seed.yaml` registers it as the default runner on every start, so it is read-only under Connections > Runners.
 Ansible settings, roles and collections under `data/ansible/` are mounted into the RTE, not the app. When your

data/seed.yaml

@@ -1,12 +1,21 @@
 # Config seed, applied by the app at every start (CONFIG_SEED_PATH in docker-compose.yml).
-# It registers the rte container as the default runner, so playbook forms run there. Objects
-# declared here are read-only in the interface; see https://ansibleforms.com/seed.
+# It registers the rte container as the default runner, so playbook forms run there, signing in
+# with the credential below. The app dials the RTE (RTE_MODE=inbound) over one WebSocket: the RTE
+# holds no database credentials. Objects declared here are read-only in the interface; see
+# https://ansibleforms.com/seed.
 version: 1
+credentials:
+  items:
+    - name: rte
+      credential_type: rte
+      description: The RTE_TOKEN of the rte container of docker-compose.yml, and its address
+      url: http://rte:8000
+      password: ${RTE_TOKEN}
 runners:
   items:
     - name: rte
       type: rte
+      rte_mode: inbound
       description: The rte container of docker-compose.yml
-      uri: http://rte:8000
-      token: ${RTE_TOKEN}
+      credential: rte
       is_default: true

docker-compose.yml

@@ -79,37 +79,35 @@ services:
       # - ./data/assets/images:/app/dist/views/img          # 6 custom images (login_background_%theme%.jpg and logo_%theme%.svg | 2 for each theme = light, dark and color)
   # AnsibleForms RTE (runtime environment): runs the playbooks. AnsibleForms 7 runs none in
   # the app itself, so this is where ansible, the Python libraries and the collections live.
-  # The app reaches it on http://rte:8000 with RTE_TOKEN; it is not published on the host.
+  # The app dials it on http://rte:8000 with RTE_TOKEN (RTE_MODE=inbound) and keeps one WebSocket
+  # open : the RTE holds no database credentials and is not published on the host.
   rte:
-    # Only start once the app is healthy: it creates the schema on the first start, and an
-    # RTE that finds no schema stops working without exiting, so a restart would not help
-    depends_on:
-      mysqldb:
-        condition: service_healthy
-      app:
-        condition: service_healthy
     image: ghcr.io/ansibleforms/ansibleforms-rte-full:7
     container_name: af_rte
     restart: unless-stopped
-    # Healthy once the RTE takes connections on its port (its API needs the token)
+    # Healthy once the RTE takes connections on its port (its socket needs the token)
     healthcheck:
       test: ["CMD", "node", "./healthcheck.js"]
       interval: 10s
       timeout: 5s
       retries: 30
       start_period: 30s
-    # Load extra environment variables from file (the database, ENCRYPTION_SECRET, RTE_TOKEN)
-    env_file:
-      - ./.env
-    # Set environment variables
+    # Only what the RTE needs : not the .env, which holds the database's passwords
     environment:
-      - DB_HOST=mysqldb
-      - DB_USER=$MYSQLDB_USER
-      - DB_PASSWORD=$MYSQLDB_PASSWORD
-      - DB_PORT=$MYSQLDB_DOCKER_PORT
+      # The app dials it (data/seed.yaml declares its runner, rte_mode inbound)
+      - NODE_ENV=production
+      - RTE_MODE=inbound
+      - RTE_TOKEN=$RTE_TOKEN
+      # Its files are the app's own (./data, mounted below) : nothing to send with each job
+      - RTE_FILES=local
       - PORT=8000
       # Only the app calls it, inside the compose network
       - HTTPS=0
+      - LOG_LEVEL=$LOG_LEVEL
+      - LOG_CONSOLE_LEVEL=$LOG_CONSOLE_LEVEL
+      - LOG_TZ=${LOG_TZ:-UTC}
+      # The playbook output it stores per stream, in bytes (read by the RTE, not the app)
+      - PROCESS_MAX_BUFFER=${PROCESS_MAX_BUFFER:-52428800}
       # The libraries the ansibleforms_pip_install playbook installs into the persistent folder
       - PYTHONPATH=/app/dist/persistent/python/lib
     volumes: