Add Keystone-on-kind integration test (forge)

.gitignore

@@ -2,5 +2,6 @@
 clouds.yml
 secure.yml
 settings.toml
+test/integration/clouds.yaml
 
 __pycache__

.zuul.yaml

@@ -11,6 +11,18 @@
     vars:
       tox_envlist: test
 
+- job:
+    name: openstack-project-manager-integration
+    # The full forge ControlPlane stack (cert-manager, MariaDB/Memcached/
+    # external-secrets/keystone/c5c3 operators, k-orc, OpenBao, flux) saturates
+    # the default node's CPU before the Envoy data-plane Pod can schedule, so
+    # Gateway/openstack-gw never gets a backing Pod and stays NoResources. The
+    # larger flavour has the CPU headroom the single-node kind cluster needs.
+    nodeset: ubuntu-noble-large
+    pre-run: playbooks/pre-integration.yml
+    run: playbooks/test-integration.yml
+    timeout: 2400
+
 - project:
     merge-mode: squash-merge
     default-branch: main
@@ -19,6 +31,7 @@
         - flake8
         - openstack-project-manager-mypy
         - openstack-project-manager-tox
+        - openstack-project-manager-integration
         - yamllint
         - python-black
 

Makefile

@@ -0,0 +1,17 @@
+CLUSTER_NAME ?= forge
+export CLUSTER_NAME
+
+.PHONY: integration integration-up integration-down
+
+# Full cycle: provision kind + Keystone, create a project, verify it, then
+# tear the cluster down (always, including on failure).
+integration:
+	test/integration/run.sh
+
+# Deploy Keystone and leave the cluster running for local debugging.
+integration-up:
+	test/integration/deploy_keystone.sh
+
+# Tear down the cluster created by integration-up.
+integration-down:
+	kind delete cluster --name $(CLUSTER_NAME)

playbooks/pre-integration.yml

@@ -0,0 +1,59 @@
+---
+- name: Pre integration play
+  hosts: all
+
+  roles:
+    - ensure-pip
+    - ensure-pipenv
+    - ensure-docker
+
+  tasks:
+    - name: Ensure required APT packages are installed
+      ansible.builtin.apt:
+        name:
+          - jq
+          - libldap2-dev
+          - libsasl2-dev
+        update_cache: true
+      become: true  # run as root
+
+    - name: Install mikefarah/yq
+      # forge's deploy-infra.sh needs mikefarah/yq (Go) for its `select(...)`,
+      # `strenv(...)` and `-i` filters. The Ubuntu `yq` APT package is the
+      # incompatible python-yq (a jq wrapper), so fetch the upstream binary.
+      ansible.builtin.get_url:
+        url: "https://github.com/mikefarah/yq/releases/download/{{ yq_version }}/yq_linux_amd64"
+        dest: /usr/local/bin/yq
+        owner: root
+        group: root
+        mode: "0755"
+      become: true
+      vars:
+        yq_version: v4.53.3
+
+    - name: Keep router advertisements honoured while IPv6 forwarding is on
+      # The OSISM Zuul nodes are IPv6-only and learn their default route via
+      # SLAAC/RA. When kind makes Docker create a dual-stack network it enables
+      # net.ipv6.conf.all.forwarding, after which the kernel (default
+      # accept_ra=1) stops honouring RAs and the node drops off the network
+      # mid-run. accept_ra=2 keeps RAs honoured while forwarding is on. The
+      # all/default keys do not retroactively apply to an already-up interface,
+      # so the live default interface is set explicitly too -- that per-interface
+      # key is what actually preserves the running node's default route.
+      ansible.builtin.copy:
+        dest: /etc/sysctl.d/99-opm-integration-accept-ra.conf
+        content: |
+          net.ipv6.conf.all.accept_ra = 2
+          net.ipv6.conf.default.accept_ra = 2
+          {% if ansible_default_ipv6.interface is defined %}
+          net.ipv6.conf.{{ ansible_default_ipv6.interface }}.accept_ra = 2
+          {% endif %}
+        owner: root
+        group: root
+        mode: "0644"
+      become: true
+
+    - name: Apply the sysctl settings
+      ansible.builtin.command: sysctl --system
+      become: true
+      changed_when: false

playbooks/test-integration.yml

@@ -0,0 +1,25 @@
+---
+- name: Run integration play
+  hosts: all
+
+  vars:
+    python_venv_dir: /tmp/venv
+
+  tasks:
+    - name: Install Python dependencies into the venv
+      ansible.builtin.pip:
+        name:
+          - tox
+          - openstacksdk
+        virtualenv: "{{ python_venv_dir }}"
+        virtualenv_command: python3 -m venv
+
+    - name: Run the integration test
+      ansible.builtin.shell: |
+        set -e -o pipefail -x
+        export PATH="{{ python_venv_dir }}/bin:${HOME}/.local/bin:${PATH}"
+        make integration
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+        executable: /bin/bash
+      changed_when: true

test/integration/README.md

@@ -0,0 +1,74 @@
+# Integration test
+
+This integration test exercises the real project-creation path of
+`openstack-project-manager` against a **live Keystone** instead of mocks.
+Keystone is deployed locally on a [kind](https://kind.sigs.k8s.io/) cluster via
+[forge](https://github.com/c5c3/forge) (its ControlPlane Quick Start), and the
+test then creates one project using the admin account and verifies it through
+the OpenStack SDK.
+
+It is deliberately a scaffold: one happy-path test. The unit suite under
+`test/unit/` keeps everything mocked; this harness is the first thing that
+talks to a real `openstack.connect()` / `keystone.projects.update()` path.
+
+## Prerequisites
+
+- A Docker-capable host with the Docker daemon running.
+- Outbound network access (to clone forge, pull operator images, and resolve
+  `keystone.127-0-0-1.nip.io` via [nip.io](https://nip.io/) to `127.0.0.1`).
+- For local runs, `openstacksdk` and `tox` must be importable in the active
+  environment (the CI job installs them into a throwaway venv). `kind` and
+  `kubectl` are installed by forge's own `make install-test-deps` into
+  `~/.local/bin`; the scripts prepend that directory to `PATH`.
+
+## Usage
+
+```bash
+# Full cycle: provision kind + Keystone, create a project, verify it, and tear
+# the cluster down on every exit path (including failure).
+make integration
+
+# Deploy Keystone and leave the cluster running for local debugging.
+make integration-up
+
+# Tear down the cluster left running by `make integration-up`.
+make integration-down
+```
+
+`make integration` only deletes a cluster it created itself. If a kind cluster
+named `forge` already exists (for example one you started with
+`make integration-up`), it is reused and left in place on exit.
+
+## Environment overrides
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `FORGE_REF` | pinned commit | forge git ref to check out (pinned for reproducibility). |
+| `FORGE_DIR` | `${TMPDIR:-/tmp}/opm-forge` | Where forge is cloned. |
+| `CLUSTER_NAME` | `forge` | kind cluster name. |
+| `KIND_HOST_PORT` | `8443` | Host port the Envoy Gateway (and Keystone) is exposed on. |
+| `NAMESPACE` | `openstack` | Namespace the ControlPlane CR is applied to. |
+| `OS_CLIENT_CONFIG_FILE` | `test/integration/clouds.yaml` | Generated clouds.yaml path. |
+| `KEEP_PROJECT` | `0` | Set to `1` to keep the created project after `verify.py` (the cluster is torn down regardless). |
+
+## What is asserted
+
+After `tox -e create` creates the project as admin, `verify.py` connects with
+the OpenStack SDK and asserts that:
+
+- the `default` domain resolves,
+- the `opm-integration-test` project exists in it, and
+- the per-project group of the same name exists.
+
+`verify.py` exits non-zero on any failed assertion. It is a standalone script,
+not a `unittest`/`pytest` test, and is not named `test_*.py`, so the fast unit
+gate (`tox -e test`) never collects it.
+
+## Credentials
+
+No real credentials are committed. `deploy_keystone.sh` reads the admin
+password from the operator-projected
+`controlplane-keystone-admin-credentials` Secret and writes an ephemeral
+`clouds.yaml` (gitignored) with `verify: false`, because the Envoy Gateway
+serves a self-signed cert. `run.sh` removes the generated `clouds.yaml` on
+exit.

test/integration/controlplane.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: c5c3.io/v1alpha1
+kind: ControlPlane
+metadata:
+  name: controlplane
+  namespace: openstack
+spec:
+  openStackRelease: "2025.2"
+  services:
+    keystone:
+      replicas: 1
+      publicEndpoint: https://keystone.127-0-0-1.nip.io:8443/v3
+      gateway:
+        parentRef:
+          name: openstack-gw
+        hostname: keystone.127-0-0-1.nip.io
+        path: /

test/integration/deploy_keystone.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+# Deploy a Keystone-only forge ControlPlane on a kind cluster and emit a
+# clouds.yaml the project-manager can authenticate against.
+#
+# Safe to run standalone (see `make integration-up`). A kind cluster of the
+# same name that already exists is reused; this script does not delete it.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Pin forge to a known-good commit; forge is a fast-moving prototyping repo.
+FORGE_REF="${FORGE_REF:-35ef73ae48d6814270af1674370d60ae410efdc7}"
+FORGE_DIR="${FORGE_DIR:-${TMPDIR:-/tmp}/opm-forge}"
+
+NAMESPACE="${NAMESPACE:-openstack}"
+KIND_HOST_PORT="${KIND_HOST_PORT:-8443}"
+CLOUDS_FILE="${OS_CLIENT_CONFIG_FILE:-${HERE}/clouds.yaml}"
+
+# Refuse to clobber an existing file outside our dedicated in-tree path. An
+# operator who exports OS_CLIENT_CONFIG_FILE to point at their real
+# ~/.config/openstack/clouds.yaml would otherwise have it truncated and
+# replaced with throwaway test admin credentials by the write in step 3.
+if [[ -e "${CLOUDS_FILE}" && "${CLOUDS_FILE}" != "${HERE}/clouds.yaml" ]]; then
+    echo "Refusing to overwrite existing ${CLOUDS_FILE}." >&2
+    echo "Unset OS_CLIENT_CONFIG_FILE or point it at a throwaway path." >&2
+    exit 1
+fi
+
+# 1. forge toolchain + infra -------------------------------------------------
+# forge's own installer pins kind/kubectl into ${HOME}/.local/bin.
+if [[ ! -d "${FORGE_DIR}/.git" ]]; then
+    git clone https://github.com/c5c3/forge.git "${FORGE_DIR}"
+else
+    git -C "${FORGE_DIR}" fetch --quiet origin
+fi
+git -C "${FORGE_DIR}" checkout --quiet "${FORGE_REF}"
+
+make -C "${FORGE_DIR}" install-test-deps
+export PATH="${HOME}/.local/bin:${PATH}"
+
+# Pin the optional forge stacks off: this Keystone-only integration test needs
+# neither Chaos Mesh nor the Prometheus stack, and they only add provisioning
+# time and load. These are forge's defaults today; set them explicitly so a
+# future forge default flip cannot silently pull them in.
+KIND_HOST_PORT="${KIND_HOST_PORT}" WITH_CONTROLPLANE=true \
+    WITH_CHAOS_MESH=false WITH_PROMETHEUS=false \
+    make -C "${FORGE_DIR}" deploy-infra
+
+# 2. Keystone-only ControlPlane ---------------------------------------------
+kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 \
+    || kubectl create namespace "${NAMESPACE}"
+kubectl apply -f "${HERE}/controlplane.yaml"
+
+# Wait for the ControlPlane's aggregate Ready condition. A bare
+# `kubectl wait --for=condition=Ready --timeout=15m` is silent for the whole
+# window, so a stuck sub-condition (the operator reconciles them in order:
+# InfrastructureReady -> KeystoneReady -> KORCReady -> AdminCredentialReady ->
+# CatalogReady) is indistinguishable from a dead hang in the CI log. Poll
+# instead and print the first not-yet-True condition each tick, so the log
+# shows which stage is pending; on timeout dump the full CR before failing
+# (the verbose pod/event dump is left to run.sh's diagnostics trap).
+cp_timeout="${CONTROLPLANE_TIMEOUT:-900}"
+cp_deadline=$(($(date +%s) + cp_timeout))
+while true; do
+    if [[ "$(kubectl get controlplane/controlplane -n "${NAMESPACE}" \
+        -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' \
+        2>/dev/null)" == "True" ]]; then
+        echo "ControlPlane/controlplane is Ready."
+        break
+    fi
+    pending="$(kubectl get controlplane/controlplane -n "${NAMESPACE}" -o json 2>/dev/null \
+        | jq -r 'first(.status.conditions[]? | select(.type != "Ready" and .status != "True")
+                 | "\(.type)=\(.status) (\(.reason)): \(.message)") // "status not reported yet"')"
+    echo "  ControlPlane/controlplane not Ready yet: ${pending}"
+    if [[ "$(date +%s)" -ge "${cp_deadline}" ]]; then
+        echo "ERROR: ControlPlane/controlplane not Ready after ${cp_timeout}s." >&2
+        kubectl get controlplane/controlplane -n "${NAMESPACE}" -o yaml || true
+        exit 1
+    fi
+    sleep 15
+done
+
+# 3. clouds.yaml from the operator-projected admin credentials --------------
+admin_password="$(kubectl get secret controlplane-keystone-admin-credentials \
+    -n "${NAMESPACE}" -o jsonpath='{.data.password}' | base64 -d)"
+
+# Serialize with a YAML library rather than a heredoc: the minted admin
+# password is arbitrary bytes, and a `"` or `\` interpolated into a quoted YAML
+# scalar would corrupt the file or silently change the value via escape
+# processing.
+umask 077
+ADMIN_PASSWORD="${admin_password}" KIND_HOST_PORT="${KIND_HOST_PORT}" \
+    CLOUDS_FILE="${CLOUDS_FILE}" python3 - <<'PY'
+import os
+
+import yaml
+
+data = {
+    "clouds": {
+        "admin": {
+            "auth": {
+                "auth_url": f"https://keystone.127-0-0-1.nip.io:{os.environ['KIND_HOST_PORT']}/v3",
+                "username": "admin",
+                "password": os.environ["ADMIN_PASSWORD"],
+                "project_name": "admin",
+                "user_domain_name": "Default",
+                "project_domain_name": "Default",
+            },
+            "identity_api_version": 3,
+            "verify": False,
+        }
+    }
+}
+with open(os.environ["CLOUDS_FILE"], "w") as f:
+    yaml.safe_dump(data, f, default_flow_style=False)
+PY
+echo "Wrote clouds.yaml to ${CLOUDS_FILE}"
+
+# 4. Ensure the roles create.py expects exist on the bare Keystone ----------
+# create.py indexes its role cache directly (CACHE_ROLES[role_name]) when it
+# assigns DEFAULT_ROLES = member, load-balancer_member, so a missing role
+# aborts the run. A bare forge Keystone ships member but not
+# load-balancer_member (Octavia provisions that one), so pre-create it here.
+# Idempotent.
+export OS_CLIENT_CONFIG_FILE="${CLOUDS_FILE}"
+python3 - <<'PY'
+import openstack
+
+cloud = openstack.connect(cloud="admin")
+for name in ("member", "load-balancer_member"):
+    if cloud.identity.find_role(name) is None:
+        cloud.identity.create_role(name=name)
+        print(f"Created role: {name}")
+    else:
+        print(f"Role already present: {name}")
+PY