Scw runner lifecycle improvements

runner/ansible/README.md

@@ -0,0 +1,56 @@
+# RISC-V Runner Ansible Roles & Playbooks
+
+This directory contains Ansible playbooks and roles for configuring Scaleway RISC-V baremetal runners.
+
+## Directory Structure
+
+```text
+runner/ansible/
+├── ansible.cfg            # Ansible configuration (pipelining, 300s ControlPersist)
+├── site.yml               # Main entrypoint playbook
+├── group_vars/
+│   └── all.yml            # Central versions, checksums, kernel modules, & default variables
+└── roles/
+    ├── kernelspace/       # Kernel toolchain, compilation, sysctl, & module loading
+    └── userspace/         # containerd, node_exporter, prometheus-agent, k8s binaries, & watchdog
+```
+
+## Extra Variables
+
+The following variables can be passed at runtime via extra-vars (`-e`):
+
+| Variable | Description | Default |
+| :--- | :--- | :--- |
+| `cockpit_metrics_push_url` | Scaleway Cockpit metrics remote write URL | `""` |
+| `cockpit_metrics_token` | Scaleway Cockpit metrics push authorization secret | `""` |
+| `github_probe_token` | Bearer token for `github-probe` health checks | `""` |
+
+## Usage via `scw.py` (Recommended)
+
+`scw.py` automatically discovers target baremetal IPs, retrieves telemetry tokens, and invokes Ansible securely:
+
+```bash
+# Full runner setup
+.venv/bin/python3 scripts/scw.py runner setup riscv-runner-1
+
+# Kernelspace-only setup
+.venv/bin/python3 scripts/scw.py runner setup --kernelspace-only riscv-runner-1
+
+# Userspace-only setup
+.venv/bin/python3 scripts/scw.py runner setup --userspace-only riscv-runner-1
+```
+
+## Stand-Alone CLI Usage
+
+To run playbooks manually against a baremetal runner without `scw.py`:
+
+```bash
+.venv/bin/ansible-playbook \
+  -i "62.210.163.200," \
+  -u ubuntu \
+  --private-key ~/.ssh/id_scw \
+  -e "cockpit_metrics_push_url=https://..." \
+  -e "cockpit_metrics_token=..." \
+  -e "github_probe_token=..." \
+  runner/ansible/site.yml
+```

runner/ansible/ansible.cfg

@@ -0,0 +1,16 @@
+[defaults]
+roles_path = roles
+host_key_checking = False
+retry_files_enabled = False
+stdout_callback = yaml
+bin_ansible_callbacks = True
+
+[privilege_escalation]
+become = True
+become_method = sudo
+become_user = root
+become_ask_pass = False
+
+[ssh_connection]
+pipelining = True
+ssh_args = -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlMaster=auto -o ControlPersist=300s

runner/ansible/group_vars/all.yml

@@ -0,0 +1,49 @@
+---
+# Software Versions & URLs
+node_exporter_version: "1.11.1"
+node_exporter_checksum: "sha256:8d73447c47488a94f7eba467838c815ea7dceb449c75b1b8e91fa6dc3e0e364e"
+prometheus_version: "3.11.3"
+prometheus_checksum: "sha256:bd6978937d64f4afa82919e0c4b3b83ace50808b953ab6174e480ca7dda2ba9a"
+crictl_version: "v1.35.0"
+cni_plugins_version: "v1.4.0"
+kubernetes_version: "1.35.0"
+kubernetes_gitlab_artifacts_url: "https://gitlab.com/riseproject/risc-v-runner/kubernetes/-/jobs/13257210986/artifacts/download"
+
+# Kernel & Module Configuration
+kernel_version_suffix: "-scw1"
+modules_to_build:
+  - net/netfilter/ipset
+  - fs/erofs
+  - drivers/md
+
+modules_to_load_k8s:
+  - overlay
+  - br_netfilter
+  - nf_conntrack
+  - tun
+
+modules_to_load_users:
+  - ip_set
+  - ip_set_bitmap_ip
+  - ip_set_bitmap_ipmac
+  - ip_set_bitmap_port
+  - ip_set_hash_ip
+  - ip_set_hash_ipmac
+  - ip_set_hash_ipmark
+  - ip_set_hash_ipport
+  - ip_set_hash_ipportip
+  - ip_set_hash_ipportnet
+  - ip_set_hash_mac
+  - ip_set_hash_net
+  - ip_set_hash_netiface
+  - ip_set_hash_netnet
+  - ip_set_hash_netport
+  - ip_set_hash_netportnet
+  - ip_set_list_set
+  - erofs
+  - dm_verity
+
+# Cockpit & Probe Configuration
+cockpit_metrics_push_url: ""
+cockpit_metrics_token: ""
+github_probe_token: ""

runner/ansible/roles/kernelspace/tasks/main.yml

@@ -0,0 +1,156 @@
+---
+- name: Install kernel build dependencies
+  ansible.builtin.apt:
+    name:
+      - build-essential
+      - libelf-dev
+      - libssl-dev
+      - bc
+      - bison
+      - flex
+      - gcc-14
+      - ipset
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Set gcc-14 as default gcc
+  ansible.builtin.alternatives:
+    name: gcc
+    link: /usr/bin/gcc
+    path: /usr/bin/gcc-14
+    priority: 100
+
+- name: Check if kernel modules are already built and verified
+  ansible.builtin.stat:
+    path: "/lib/modules/{{ ansible_facts['kernel'] }}/kernel/net/netfilter/ipset/ip_set.ko"
+  register: ip_set_ko
+
+- name: Build and install kernel modules
+  when: not ip_set_ko.stat.exists
+  block:
+    - name: Seed kernel config from /proc/config.gz
+      ansible.builtin.shell: |
+        zcat /proc/config.gz > .config
+      args:
+        chdir: "/usr/lib/modules/{{ ansible_facts['kernel'] }}/source"
+        creates: "/usr/lib/modules/{{ ansible_facts['kernel'] }}/source/.config"
+
+    - name: Configure kernel options for ipset, erofs, and dm_verity
+      ansible.builtin.shell: |
+        sudo scripts/config -m IP_SET_HASH_NET
+        sudo scripts/config -m IP_SET_HASH_IPPORT
+        sudo scripts/config -m IP_SET_HASH_IPPORTIP
+        sudo scripts/config -m IP_SET_HASH_IPPORTNET
+        sudo scripts/config -m IP_SET_HASH_NETPORT
+        sudo scripts/config -m IP_SET_HASH_NETIFACE
+        sudo scripts/config -m IP_SET_HASH_NETNET
+        sudo scripts/config -m IP_SET_HASH_NETPORTNET
+        sudo scripts/config -m IP_SET_HASH_IPMARK
+        sudo scripts/config -m IP_SET_HASH_IPMAC
+        sudo scripts/config -m IP_SET_HASH_MAC
+        sudo scripts/config -m IP_SET_BITMAP_IP
+        sudo scripts/config -m IP_SET_BITMAP_IPMAC
+        sudo scripts/config -m IP_SET_BITMAP_PORT
+        sudo scripts/config -m IP_SET_LIST_SET
+        sudo scripts/config -m EROFS_FS
+        sudo scripts/config -e EROFS_FS_ZIP
+        sudo scripts/config -e EROFS_FS_ZIP_LZMA
+        sudo scripts/config -e EROFS_FS_ZIP_DEFLATE
+        sudo scripts/config -e EROFS_FS_POSIX_ACL
+        sudo scripts/config -e EROFS_FS_XATTR
+        sudo scripts/config -m DM_VERITY
+        sudo scripts/config -e DM_VERITY_VERIFY_ROOTHASH_SIG
+        sudo scripts/config -e BLK_DEV_DM
+        sudo scripts/config -e MD
+        sudo scripts/config -m CRYPTO_SHA256
+        sudo scripts/config --set-str LOCALVERSION "{{ kernel_version_suffix }}"
+        sudo scripts/config -d LOCALVERSION_AUTO
+      args:
+        chdir: "/usr/lib/modules/{{ ansible_facts['kernel'] }}/source"
+
+    - name: Prepare kernel build tree
+      ansible.builtin.shell: |
+        sudo make ARCH=riscv olddefconfig
+        sudo make ARCH=riscv prepare
+        sudo make ARCH=riscv modules_prepare
+        sudo cp /lib/modules/{{ ansible_facts['kernel'] }}/build/Module.symvers .
+      args:
+        chdir: "/usr/lib/modules/{{ ansible_facts['kernel'] }}/source"
+
+    - name: Build kernel modules
+      ansible.builtin.shell: |
+        sudo make ARCH=riscv \
+             KCFLAGS="-mno-relax -fno-asynchronous-unwind-tables -fno-unwind-tables -g0" \
+             KAFLAGS="-mno-relax" \
+             M="{{ modules_to_build | join(' ') }}" modules -j$(nproc)
+      args:
+        chdir: "/usr/lib/modules/{{ ansible_facts['kernel'] }}/source"
+
+    - name: Verify and install kernel modules
+      ansible.builtin.shell: |
+        set -e
+        for m in {{ modules_to_build | join(' ') }}; do
+            sudo mkdir -p /lib/modules/{{ ansible_facts['kernel'] }}/kernel/${m}
+            for ko in ${m}/*.ko; do
+                modinfo "$ko" | grep '^vermagic:' | grep -q "{{ ansible_facts['kernel'] }} SMP preempt mod_unload riscv" || {
+                    echo "FAIL vermagic mismatch: $ko" >&2
+                    exit 1
+                }
+                READELF="$(command -v riscv64-linux-gnu-readelf || command -v readelf)"
+                if "$READELF" -r "$ko" | awk '{print $3}' | sort -u | grep -qE '(R_RISCV_ALIGN|R_RISCV_32_PCREL)'; then
+                    echo "FAIL forbidden relocations: $ko" >&2
+                    exit 1
+                fi
+                sudo cp -v "$ko" /lib/modules/{{ ansible_facts['kernel'] }}/kernel/${m}/
+            done
+        done
+        sudo depmod -a
+      args:
+        chdir: "/usr/lib/modules/{{ ansible_facts['kernel'] }}/source"
+
+- name: Configure k8s modules to load at boot
+  ansible.builtin.copy:
+    dest: /etc/modules-load.d/k8s.conf
+    content: |
+      {{ modules_to_load_k8s | join('\n') }}
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Configure user modules to load at boot
+  ansible.builtin.copy:
+    dest: /etc/modules-load.d/users.conf
+    content: |
+      {{ modules_to_load_users | join('\n') }}
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Load required k8s kernel modules
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: present
+  loop: "{{ modules_to_load_k8s }}"
+
+- name: Configure modprobe blacklist for algif_aead
+  ansible.builtin.copy:
+    dest: /etc/modprobe.d/blacklist-copyfail.conf
+    content: |
+      blacklist algif_aead
+      install algif_aead /bin/true
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Configure sysctl for Kubernetes networking
+  ansible.posix.sysctl:
+    name: "{{ item.key }}"
+    value: "{{ item.value }}"
+    sysctl_file: /etc/sysctl.d/k8s.conf
+    state: present
+    reload: true
+  loop:
+    - { key: 'net.bridge.bridge-nf-call-iptables', value: '1' }
+    - { key: 'net.bridge.bridge-nf-call-ip6tables', value: '1' }
+    - { key: 'net.ipv4.ip_forward', value: '1' }

runner/ansible/roles/userspace/handlers/main.yml

@@ -0,0 +1,31 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Restart containerd
+  ansible.builtin.systemd:
+    name: containerd
+    state: restarted
+
+- name: Restart node_exporter
+  ansible.builtin.systemd:
+    name: node_exporter
+    state: restarted
+
+- name: Restart prometheus-agent
+  ansible.builtin.systemd:
+    name: prometheus-agent
+    state: restarted
+
+- name: Enable kubelet-watchdog timer
+  ansible.builtin.systemd:
+    name: kubelet-watchdog.timer
+    state: started
+    enabled: true
+
+- name: Enable containerd-watchdog timer
+  ansible.builtin.systemd:
+    name: containerd-watchdog.timer
+    state: started
+    enabled: true

runner/ansible/roles/userspace/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kernelspace