From b59771814dfaf7b495437a01ad90384b4ea5b73e Mon Sep 17 00:00:00 2001 From: Beinan Date: Wed, 22 Jul 2026 21:13:01 +0000 Subject: [PATCH 1/2] test: add docker-compose test env + AI-agent harness Bring up the full control-plane + data-plane stack (MinIO, etcd, master, two workers) so the real HTTP surface can be exercised end to end. This closes the coverage gap for the record-list source selector (?source=fragments|wal|all, PR #170), which is only reachable through live etcd + object store and is otherwise #[ignore]d in CI. - test/Dockerfile: single multi-stage build (ui -> builder -> master/worker) - test/docker-compose.yml: MinIO-backed s3://lance-context DATA_DIR shared by master + workers; WAL self-merge disabled so source split is observable - test/harness/{up,down,smoke}.sh: bring up/tear down + assert source semantics over HTTP (fragments=0, wal=3, all=3 before merge; default= fragments; unknown source -> 400) - test/harness/README.md: usage + manual curl recipes for agents - .dockerignore: keep build context small Co-Authored-By: Claude Opus 4 --- .dockerignore | 16 +++++ test/Dockerfile | 70 ++++++++++++++++++ test/docker-compose.yml | 156 ++++++++++++++++++++++++++++++++++++++++ test/harness/README.md | 91 +++++++++++++++++++++++ test/harness/down.sh | 20 ++++++ test/harness/smoke.sh | 127 ++++++++++++++++++++++++++++++++ test/harness/up.sh | 36 ++++++++++ 7 files changed, 516 insertions(+) create mode 100644 .dockerignore create mode 100644 test/Dockerfile create mode 100644 test/docker-compose.yml create mode 100644 test/harness/README.md create mode 100755 test/harness/down.sh create mode 100755 test/harness/smoke.sh create mode 100755 test/harness/up.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..51e6c27 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +# Keep the Docker build context small. The Rust build happens inside the +# builder stage; host build artifacts and node_modules must not be shipped. +# +# NOTE: do not exclude `python/` — it is a Cargo workspace member, so cargo +# needs its manifest present to resolve the workspace even when we only build +# the master/server crates. +target/ +**/node_modules/ +crates/lance-context-master/ui/dist/ +.git/ +.github/ +examples/ +docs/ +specs/ +deploy/ +.claude/ diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 0000000..a37f678 --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,70 @@ +# syntax=docker/dockerfile:1 + +# Single multi-stage build for the lance-context test environment. +# +# Stages: +# ui -> builds the master's React/Vite admin UI into static assets +# builder -> compiles the Rust workspace binaries (master + server) +# master -> runtime image for lance-context-master (control-plane), bundles UI +# worker -> runtime image for lance-context-server (data-plane) +# +# Select a target with `--target master` / `--target worker`, which is what the +# accompanying docker-compose.yml does. Built once, the shared `builder` stage +# is cached across both runtime images. + +# --------------------------------------------------------------------------- +# UI assets (only needed by the master image) +# --------------------------------------------------------------------------- +FROM node:20-bookworm-slim AS ui +WORKDIR /ui +COPY crates/lance-context-master/ui/package.json crates/lance-context-master/ui/package-lock.json ./ +RUN npm ci +COPY crates/lance-context-master/ui/ ./ +RUN npm run build +# Vite emits to ./dist + +# --------------------------------------------------------------------------- +# Rust workspace build +# --------------------------------------------------------------------------- +FROM rust:1-bookworm AS builder +# Native deps mirror what CI installs: protoc for the lance protobufs, openssl +# for etcd TLS + object-store, make/perl for the openssl-sys vendored build. +RUN apt-get update && apt-get install -y --no-install-recommends \ + protobuf-compiler libssl-dev pkg-config make perl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /build +COPY . . +# Build only the two service binaries we ship (skip python/examples). +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/build/target \ + cargo build --release \ + -p lance-context-master \ + -p lance-context-server \ + && mkdir -p /out \ + && cp target/release/lance-context-master /out/ \ + && cp target/release/lance-context-server /out/ + +# --------------------------------------------------------------------------- +# Master runtime (control-plane + UI) +# --------------------------------------------------------------------------- +FROM debian:bookworm-slim AS master +RUN apt-get update && apt-get install -y --no-install-recommends \ + libssl3 ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /out/lance-context-master /usr/local/bin/lance-context-master +COPY --from=ui /ui/dist /app/ui +ENV UI_DIR=/app/ui +ENV MASTER_PORT=8090 +EXPOSE 8090 +ENTRYPOINT ["lance-context-master"] + +# --------------------------------------------------------------------------- +# Worker runtime (data-plane) +# --------------------------------------------------------------------------- +FROM debian:bookworm-slim AS worker +RUN apt-get update && apt-get install -y --no-install-recommends \ + libssl3 ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /out/lance-context-server /usr/local/bin/lance-context-server +EXPOSE 3000 +ENTRYPOINT ["lance-context-server"] diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100644 index 0000000..c3066dd --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,156 @@ +# lance-context test environment +# +# A full control-plane + data-plane stack an AI agent (or human) can bring up to +# exercise the real HTTP surface end to end — including the record-list source +# selector (?source=fragments|wal|all) that today is only covered by #[ignore]d +# Rust tests because it needs a live etcd + object store. +# +# Topology: +# minio S3-compatible object store, backs the shared DATA_DIR +# minio-init one-shot: creates the bucket +# etcd durable scheduler queue / locks for the master +# worker-0 data-plane writer (owns MemWAL shard "worker-0"), port 3001 +# worker-1 data-plane writer (owns MemWAL shard "worker-1"), port 3002 +# master control-plane admin API + UI, port 8090 +# +# All services share DATA_DIR=s3://lance-context on MinIO. Workers write rollout +# datasets there; the master discovers + reads them. +# +# Bring up: docker compose -f test/docker-compose.yml up --build --wait +# Smoke test: test/harness/smoke.sh +# Tear down: docker compose -f test/docker-compose.yml down -v + +name: lance-context-test + +x-s3-env: &s3-env + # lance object-store -> MinIO wiring + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT: http://minio:9000 + AWS_ALLOW_HTTP: "true" + DATA_DIR: s3://lance-context + +services: + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:9000/minio/health/live"] + interval: 3s + timeout: 2s + retries: 20 + volumes: + - minio-data:/data + + minio-init: + image: minio/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 minioadmin minioadmin && + mc mb --ignore-existing local/lance-context && + echo 'bucket ready' + " + + etcd: + image: quay.io/coreos/etcd:v3.5.13 + command: + - etcd + - --name=etcd0 + - --data-dir=/etcd-data + - --advertise-client-urls=http://etcd:2379 + - --listen-client-urls=http://0.0.0.0:2379 + ports: + - "2379:2379" + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 3s + timeout: 2s + retries: 20 + volumes: + - etcd-data:/etcd-data + + worker-0: + build: + context: .. + dockerfile: test/Dockerfile + target: worker + depends_on: + minio-init: + condition: service_completed_successfully + environment: + <<: *s3-env + INSTANCE_ID: worker-0 + # Keep WAL un-merged by default so the ?source=wal path has data to show. + ROLLOUT_MERGE_AFTER_GENERATIONS: "0" + command: ["--host", "0.0.0.0", "--port", "3000", "--data-dir", "s3://lance-context"] + ports: + - "3001:3000" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/api/v1/health"] + interval: 3s + timeout: 2s + retries: 20 + + worker-1: + build: + context: .. + dockerfile: test/Dockerfile + target: worker + depends_on: + minio-init: + condition: service_completed_successfully + environment: + <<: *s3-env + INSTANCE_ID: worker-1 + ROLLOUT_MERGE_AFTER_GENERATIONS: "0" + command: ["--host", "0.0.0.0", "--port", "3000", "--data-dir", "s3://lance-context"] + ports: + - "3002:3000" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/api/v1/health"] + interval: 3s + timeout: 2s + retries: 20 + + master: + build: + context: .. + dockerfile: test/Dockerfile + target: master + depends_on: + etcd: + condition: service_healthy + minio-init: + condition: service_completed_successfully + environment: + <<: *s3-env + ETCD_ENDPOINTS: http://etcd:2379 + WORKER_ENDPOINTS: http://worker-0:3000,http://worker-1:3000 + MASTER_HOST: 0.0.0.0 + MASTER_PORT: "8090" + UI_DIR: /app/ui + # Fast sweeps so agents see merge/compaction react quickly in tests. + STATS_SCAN_INTERVAL_SECS: "10" + MERGE_WAL_INTERVAL_SECS: "0" + COMPACTION_INTERVAL_SECS: "0" + ports: + - "8090:8090" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8090/metrics"] + interval: 3s + timeout: 2s + retries: 30 + +volumes: + minio-data: + etcd-data: diff --git a/test/harness/README.md b/test/harness/README.md new file mode 100644 index 0000000..a317a2b --- /dev/null +++ b/test/harness/README.md @@ -0,0 +1,91 @@ +# lance-context test environment + harness + +A Docker Compose stack that brings up the **full control-plane + data-plane** so an +AI agent (or human) can exercise the real HTTP surface end to end — the surface that +today is only covered by `#[ignore]`d Rust tests because they need a live etcd and +object store. + +## Why this exists + +The master's record-list endpoint (and its `?source=fragments|wal|all` selector shipped +in PR #170) can only be integration-tested against a running etcd + object store. Those +tests are marked `#[ignore]` and never run in CI, so the HTTP layer and UI had no +automated coverage. This harness closes that gap: `smoke.sh` makes real HTTP calls and +asserts the source-selector semantics. + +## Topology + +| Service | Role | Host port | +|------------|----------------------------------------|-----------| +| `minio` | S3-compatible object store (DATA_DIR) | 9000 (API), 9001 (console) | +| `etcd` | master scheduler queue / locks | 2379 | +| `worker-0` | data-plane writer, MemWAL shard `worker-0` | 3001 | +| `worker-1` | data-plane writer, MemWAL shard `worker-1` | 3002 | +| `master` | control-plane admin API + UI | 8090 | + +All services share `DATA_DIR=s3://lance-context` on MinIO. Workers write rollout +datasets there; the master discovers and reads them. MemWAL self-merge is disabled +(`ROLLOUT_MERGE_AFTER_GENERATIONS=0`) so appended rows stay **pending in the WAL** — +which is exactly what makes `?source=fragments` vs `wal` vs `all` observably different. + +## Prerequisites + +- Docker with Compose v2 (`docker compose`) +- `curl` and `jq` on the host (for `smoke.sh`) + +## Usage + +```bash +# Build images and bring the stack up (waits for all healthchecks): +test/harness/up.sh + +# Run the end-to-end smoke test (asserts source-selector semantics): +test/harness/smoke.sh + +# Tear down (removes volumes for a clean slate): +test/harness/down.sh +``` + +`up.sh --no-build` skips the image build if images already exist. +`down.sh --keep-volumes` retains MinIO + etcd data across runs. + +## What `smoke.sh` asserts + +1. Create a rollout store on `worker-0` and append 3 records (they land in MemWAL, + un-merged). +2. Wait for the master to discover the experiment. +3. `GET .../records?source=fragments` → **0** rows (base table only, nothing merged yet). +4. `GET .../records?source=wal` → **3** rows (the pending generations). +5. `GET .../records?source=all` → **3** rows (base ∪ WAL union). +6. Response JSON echoes the resolved `source`. +7. No `source` param defaults to `fragments`. +8. An unknown `source` value returns HTTP `400`. + +## Manual poking (for agents) + +```bash +# Create a store + append on a worker: +curl -X POST localhost:3001/api/v1/rollouts \ + -H 'Content-Type: application/json' -d '{"name":"exp1"}' +curl -X POST localhost:3001/api/v1/rollouts/exp1/records \ + -H 'Content-Type: application/json' \ + -d '{"records":[{"id":"a","rollout_id":"r","role":"assistant","content":"hi"}]}' + +# Browse via the master under each source: +curl 'localhost:8090/api/v1/experiments/exp1/records?source=fragments' | jq +curl 'localhost:8090/api/v1/experiments/exp1/records?source=wal' | jq +curl 'localhost:8090/api/v1/experiments/exp1/records?source=all' | jq + +# List experiments / open the UI: +curl localhost:8090/api/v1/experiments | jq +open http://localhost:8090 # admin UI (Fragments / WAL / All tabs) +``` + +## Notes / limitations + +- The images build the Rust workspace from source (protoc + openssl), so the first + `up.sh` is slow (a full release build). Subsequent runs reuse the BuildKit cache. +- The master's WAL-merge and compaction sweeps are disabled in this stack + (`MERGE_WAL_INTERVAL_SECS=0`, `COMPACTION_INTERVAL_SECS=0`) so WAL rows stay pending + and the source split is deterministic. Enqueue merges/compactions manually via the + `/api/v1/tasks` endpoint if you want to test the scheduler. diff --git a/test/harness/down.sh b/test/harness/down.sh new file mode 100755 index 0000000..6c4a989 --- /dev/null +++ b/test/harness/down.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tear down the lance-context test environment. +# +# Usage: test/harness/down.sh [--keep-volumes] +# +# By default removes containers AND volumes (fresh state next `up`). Pass +# --keep-volumes to retain the MinIO bucket + etcd data across runs. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${HERE}/../docker-compose.yml" + +VOL_FLAG="-v" +if [[ "${1:-}" == "--keep-volumes" ]]; then + VOL_FLAG="" +fi + +echo ">> tearing down lance-context test stack" +# shellcheck disable=SC2086 +docker compose -f "${COMPOSE_FILE}" down ${VOL_FLAG} diff --git a/test/harness/smoke.sh b/test/harness/smoke.sh new file mode 100755 index 0000000..a28c4eb --- /dev/null +++ b/test/harness/smoke.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# End-to-end smoke test for the lance-context test environment. +# +# This exercises the real HTTP surface that today is only covered by #[ignore]d +# Rust tests (they need a live etcd + object store). In particular it asserts +# the record-list SOURCE SELECTOR shipped in PR #170: +# +# GET /api/v1/experiments/{name}/records?source=fragments -> base table only +# GET /api/v1/experiments/{name}/records?source=wal -> flushed WAL only +# GET /api/v1/experiments/{name}/records?source=all -> base ∪ WAL +# +# Flow: +# 1. worker-0 creates a rollout store + appends rows (they land in MemWAL, +# un-merged because ROLLOUT_MERGE_AFTER_GENERATIONS=0). +# 2. master discovers the experiment and browses it under each source. +# 3. Assert: fragments omits the un-merged rows; wal shows exactly them; all +# is the union. +# +# Usage: test/harness/smoke.sh +# Exit code 0 == all assertions passed. +set -euo pipefail + +MASTER="${MASTER_URL:-http://localhost:8090}" +WORKER="${WORKER_URL:-http://localhost:3001}" +EXP="smoke-$(date +%s)" + +pass() { echo " PASS: $*"; } +fail() { echo " FAIL: $*" >&2; exit 1; } + +require() { + command -v "$1" >/dev/null 2>&1 || fail "missing required tool: $1" +} +require curl +require jq + +echo ">> smoke test against master=${MASTER} worker=${WORKER} experiment=${EXP}" + +# --------------------------------------------------------------------------- +# 1. Create a rollout store on the worker and append un-merged rows. +# --------------------------------------------------------------------------- +echo ">> [1] create rollout store '${EXP}' on worker" +curl -sf -X POST "${WORKER}/api/v1/rollouts" \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"${EXP}\"}" >/dev/null \ + || fail "could not create rollout store" +pass "store created" + +echo ">> [2] append 3 rollout records" +curl -sf -X POST "${WORKER}/api/v1/rollouts/${EXP}/records" \ + -H 'Content-Type: application/json' \ + -d "$(cat </dev/null \ + || fail "could not append records" +pass "3 records appended (pending in MemWAL)" + +# Give the master's stats scanner a moment to discover the new dataset. +echo ">> [3] wait for master to discover experiment '${EXP}'" +for _ in $(seq 1 30); do + if curl -sf "${MASTER}/api/v1/experiments/${EXP}?fresh=true" >/dev/null 2>&1; then + break + fi + sleep 1 +done +curl -sf "${MASTER}/api/v1/experiments/${EXP}?fresh=true" >/dev/null \ + || fail "master never discovered experiment" +pass "experiment discovered by master" + +# --------------------------------------------------------------------------- +# Helper: count records returned by a given source, echo the JSON's source. +# --------------------------------------------------------------------------- +records_json() { + local src="$1" + curl -sf "${MASTER}/api/v1/experiments/${EXP}/records?source=${src}&limit=100" +} + +echo ">> [4] assert source selector semantics" + +FRAG_JSON="$(records_json fragments)" +WAL_JSON="$(records_json wal)" +ALL_JSON="$(records_json all)" + +frag_n=$(echo "$FRAG_JSON" | jq '.records | length') +wal_n=$(echo "$WAL_JSON" | jq '.records | length') +all_n=$(echo "$ALL_JSON" | jq '.records | length') + +frag_src=$(echo "$FRAG_JSON" | jq -r '.source') +wal_src=$(echo "$WAL_JSON" | jq -r '.source') +all_src=$(echo "$ALL_JSON" | jq -r '.source') + +echo " fragments: n=${frag_n} source=${frag_src}" +echo " wal: n=${wal_n} source=${wal_src}" +echo " all: n=${all_n} source=${all_src}" + +# Response echoes the resolved source (added in #170). +[[ "$frag_src" == "fragments" ]] || fail "fragments response source != fragments" +[[ "$wal_src" == "wal" ]] || fail "wal response source != wal" +[[ "$all_src" == "all" ]] || fail "all response source != all" +pass "response echoes resolved source" + +# Un-merged rows: fragments (base only) must NOT see them; wal must; all == union. +[[ "$frag_n" -eq 0 ]] || fail "fragments should be empty before merge, got ${frag_n}" +[[ "$wal_n" -eq 3 ]] || fail "wal should show 3 pending rows, got ${wal_n}" +[[ "$all_n" -eq 3 ]] || fail "all should show 3 rows (base ∪ wal), got ${all_n}" +pass "fragments omits un-merged rows; wal shows exactly them; all is the union" + +# Default (no source param) == fragments per #170. +DEF_JSON="$(curl -sf "${MASTER}/api/v1/experiments/${EXP}/records?limit=100")" +def_src=$(echo "$DEF_JSON" | jq -r '.source') +[[ "$def_src" == "fragments" ]] || fail "default source should be fragments, got ${def_src}" +pass "default source is fragments" + +# Unknown source is rejected with 400. +code=$(curl -s -o /dev/null -w '%{http_code}' \ + "${MASTER}/api/v1/experiments/${EXP}/records?source=bogus") +[[ "$code" == "400" ]] || fail "unknown source should be 400, got ${code}" +pass "unknown source rejected with 400" + +echo "" +echo ">> ALL SMOKE ASSERTIONS PASSED" diff --git a/test/harness/up.sh b/test/harness/up.sh new file mode 100755 index 0000000..f4bd756 --- /dev/null +++ b/test/harness/up.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Bring up the lance-context test environment (MinIO + etcd + master + workers). +# +# Usage: test/harness/up.sh [--no-build] +# +# Idempotent-ish: `docker compose up` will reuse running containers. Waits for +# every service healthcheck (--wait) so callers can immediately hit the API. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${HERE}/../docker-compose.yml" + +BUILD_FLAG="--build" +if [[ "${1:-}" == "--no-build" ]]; then + BUILD_FLAG="" +fi + +echo ">> bringing up lance-context test stack (${COMPOSE_FILE})" +# shellcheck disable=SC2086 +docker compose -f "${COMPOSE_FILE}" up -d ${BUILD_FLAG} --wait + +echo ">> stack is up:" +docker compose -f "${COMPOSE_FILE}" ps + +cat <<'EOF' + +Endpoints: + master admin API + UI : http://localhost:8090 + master metrics : http://localhost:8090/metrics + worker-0 (data-plane) : http://localhost:3001 + worker-1 (data-plane) : http://localhost:3002 + MinIO S3 API : http://localhost:9000 (minioadmin/minioadmin) + MinIO console : http://localhost:9001 + +Next: test/harness/smoke.sh +EOF From 447e611e9d7f2b5c3d4c13f8d823232842f6080d Mon Sep 17 00:00:00 2001 From: Beinan Date: Wed, 22 Jul 2026 22:13:59 +0000 Subject: [PATCH 2/2] test: add containerless harness for sandboxes without a Docker runtime Some environments have a Docker daemon but a kernel that forbids unshare/netlink, so no container can start (docker run fails with "failed to register layer: unshare: operation not permitted"). Add a native-process variant of the same stack so the source-selector HTTP surface can still be validated end to end: - native-up.sh: downloads a static etcd, builds the two binaries, and runs etcd + lance-context-server + lance-context-master as host processes over a local-filesystem DATA_DIR (no MinIO). --smoke chains smoke.sh. - native-down.sh: stop processes (+ --purge to wipe state). - README: document containerless mode. Verified locally: native-up.sh --smoke passes all assertions (fragments=0, wal=3, all=3 before merge; default=fragments; unknown source -> 400). Co-Authored-By: Claude Opus 4 --- test/harness/README.md | 18 +++++ test/harness/native-down.sh | 29 ++++++++ test/harness/native-up.sh | 142 ++++++++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100755 test/harness/native-down.sh create mode 100755 test/harness/native-up.sh diff --git a/test/harness/README.md b/test/harness/README.md index a317a2b..c0f2f88 100644 --- a/test/harness/README.md +++ b/test/harness/README.md @@ -46,6 +46,24 @@ test/harness/smoke.sh test/harness/down.sh ``` +### Containerless mode (sandboxes without a working Docker runtime) + +Some environments have a Docker daemon but a locked-down kernel that forbids +`unshare`/netlink, so no container can actually start (`docker run` fails with +`failed to register layer: unshare: operation not permitted`). For those, the +same stack runs as **plain host processes** — etcd (static binary, auto-downloaded), +`lance-context-server`, and `lance-context-master`, with a local-filesystem +`DATA_DIR` instead of MinIO: + +```bash +test/harness/native-up.sh --smoke # build if needed, start stack, run smoke.sh +test/harness/native-down.sh --purge # stop everything + wipe state +``` + +State lives under `$HARNESS_DIR` (default `/tmp/lance-harness`). This mode needs +`cargo` (to build the two binaries) and network access to fetch the etcd release +on first run. It asserts the exact same `?source=` semantics as the Docker path. + `up.sh --no-build` skips the image build if images already exist. `down.sh --keep-volumes` retains MinIO + etcd data across runs. diff --git a/test/harness/native-down.sh b/test/harness/native-down.sh new file mode 100755 index 0000000..f2d6517 --- /dev/null +++ b/test/harness/native-down.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Stop the containerless harness started by native-up.sh. +# +# Usage: +# test/harness/native-down.sh # stop processes, keep data +# test/harness/native-down.sh --purge # stop + delete data/logs/etcd state +set -euo pipefail + +HARNESS_DIR="${HARNESS_DIR:-/tmp/lance-harness}" +PID_DIR="${HARNESS_DIR}/pids" + +for name in master worker-0 etcd; do + pidf="$PID_DIR/$name.pid" + if [[ -f "$pidf" ]]; then + pid="$(cat "$pidf")" + if kill -0 "$pid" 2>/dev/null; then + echo ">> stopping $name (pid $pid)" + kill "$pid" 2>/dev/null || true + fi + rm -f "$pidf" + fi +done + +if [[ "${1:-}" == "--purge" ]]; then + echo ">> purging ${HARNESS_DIR} data/logs/etcd state" + rm -rf "${HARNESS_DIR}/data" "${HARNESS_DIR}/logs" "${HARNESS_DIR}/etcd-data" +fi + +echo ">> down" diff --git a/test/harness/native-up.sh b/test/harness/native-up.sh new file mode 100755 index 0000000..4cffc34 --- /dev/null +++ b/test/harness/native-up.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Containerless test environment for sandboxes where Docker can't run. +# +# Some environments (this repo's dev sandbox included) have a Docker daemon but +# a kernel that forbids `unshare`/netlink, so no container can start. This script +# brings the SAME stack up as plain host processes instead: +# +# etcd : local static binary (downloaded on first run) on 127.0.0.1:2379 +# worker-0 : lance-context-server on 127.0.0.1:3001, shard "worker-0" +# master : lance-context-master on 127.0.0.1:8090 +# +# Storage is the local filesystem (DATA_DIR=/data), not MinIO — the +# object-store abstraction treats a plain path as file://, so master + worker +# share it exactly like they'd share an S3 prefix. +# +# Usage: +# test/harness/native-up.sh # build (if needed), start etcd+worker+master +# test/harness/native-up.sh --smoke # ...then run smoke.sh against it +# test/harness/native-down.sh # stop everything +# +# State (pids, logs, data, etcd binary) lives under $HARNESS_DIR +# (default /tmp/lance-harness). +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HARNESS_DIR="${HARNESS_DIR:-/tmp/lance-harness}" +ETCD_VERSION="${ETCD_VERSION:-v3.5.13}" +DATA_DIR="${HARNESS_DIR}/data" +LOG_DIR="${HARNESS_DIR}/logs" +PID_DIR="${HARNESS_DIR}/pids" + +MASTER_PORT="${MASTER_PORT:-8090}" +WORKER_PORT="${WORKER_PORT:-3001}" +ETCD_CLIENT_URL="${ETCD_CLIENT_URL:-http://127.0.0.1:2379}" + +mkdir -p "$DATA_DIR" "$LOG_DIR" "$PID_DIR" + +log() { echo ">> $*"; } +die() { echo "FAIL: $*" >&2; exit 1; } + +wait_http() { + local url="$1" name="$2" tries="${3:-60}" + for _ in $(seq 1 "$tries"); do + if curl -sf "$url" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + die "$name did not become healthy at $url" +} + +start_bg() { + # start_bg + local name="$1" logf="$2"; shift 2 + if [[ -f "$PID_DIR/$name.pid" ]] && kill -0 "$(cat "$PID_DIR/$name.pid")" 2>/dev/null; then + log "$name already running (pid $(cat "$PID_DIR/$name.pid"))" + return 0 + fi + log "starting $name -> $logf" + nohup "$@" >"$logf" 2>&1 & + echo $! >"$PID_DIR/$name.pid" +} + +# --------------------------------------------------------------------------- +# etcd (download static binary once) +# --------------------------------------------------------------------------- +ETCD_BIN="${HARNESS_DIR}/etcd" +if [[ ! -x "$ETCD_BIN" ]]; then + log "downloading etcd ${ETCD_VERSION}" + tarball="etcd-${ETCD_VERSION}-linux-amd64.tar.gz" + curl -sfL -o "${HARNESS_DIR}/${tarball}" \ + "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/${tarball}" \ + || die "could not download etcd" + tar xzf "${HARNESS_DIR}/${tarball}" -C "$HARNESS_DIR" + cp "${HARNESS_DIR}/etcd-${ETCD_VERSION}-linux-amd64/etcd" "$ETCD_BIN" +fi + +start_bg etcd "$LOG_DIR/etcd.log" \ + "$ETCD_BIN" --name harness --data-dir "${HARNESS_DIR}/etcd-data" \ + --advertise-client-urls "$ETCD_CLIENT_URL" \ + --listen-client-urls "$ETCD_CLIENT_URL" \ + --listen-peer-urls http://127.0.0.1:2380 \ + --initial-cluster harness=http://127.0.0.1:2380 \ + --initial-advertise-peer-urls http://127.0.0.1:2380 +# etcd has no plain HTTP health path we curl easily; give it a moment. +sleep 3 + +# --------------------------------------------------------------------------- +# Build the Rust binaries (release) if missing +# --------------------------------------------------------------------------- +export PATH="$HOME/.cargo/bin:$PATH" +MASTER_BIN="${REPO}/target/release/lance-context-master" +WORKER_BIN="${REPO}/target/release/lance-context-server" +if [[ ! -x "$MASTER_BIN" || ! -x "$WORKER_BIN" ]]; then + log "building lance-context-master + lance-context-server (release)" + (cd "$REPO" && cargo build --release \ + -p lance-context-master -p lance-context-server) \ + || die "cargo build failed" +fi + +# --------------------------------------------------------------------------- +# Build the UI (optional; master serves it when UI_DIR is set) +# --------------------------------------------------------------------------- +UI_DIST="${REPO}/crates/lance-context-master/ui/dist" +if [[ ! -f "${UI_DIST}/index.html" ]] && command -v npm >/dev/null 2>&1; then + log "building admin UI" + (cd "${REPO}/crates/lance-context-master/ui" && npm ci --silent && npm run build) || true +fi + +# --------------------------------------------------------------------------- +# worker-0 (data-plane) +# --------------------------------------------------------------------------- +start_bg worker-0 "$LOG_DIR/worker-0.log" \ + env INSTANCE_ID=worker-0 ROLLOUT_MERGE_AFTER_GENERATIONS=0 \ + "$WORKER_BIN" --host 127.0.0.1 --port "$WORKER_PORT" --data-dir "$DATA_DIR" +wait_http "http://127.0.0.1:${WORKER_PORT}/api/v1/health" worker-0 + +# --------------------------------------------------------------------------- +# master (control-plane + UI) +# --------------------------------------------------------------------------- +UI_ARG=() +[[ -f "${UI_DIST}/index.html" ]] && UI_ARG=(--ui-dir "$UI_DIST") +start_bg master "$LOG_DIR/master.log" \ + env ETCD_ENDPOINTS="$ETCD_CLIENT_URL" \ + WORKER_ENDPOINTS="http://127.0.0.1:${WORKER_PORT}" \ + STATS_SCAN_INTERVAL_SECS=10 \ + MERGE_WAL_INTERVAL_SECS=0 \ + COMPACTION_INTERVAL_SECS=0 \ + "$MASTER_BIN" --host 127.0.0.1 --port "$MASTER_PORT" --data-dir "$DATA_DIR" "${UI_ARG[@]}" +wait_http "http://127.0.0.1:${MASTER_PORT}/metrics" master + +log "stack is up:" +echo " etcd : ${ETCD_CLIENT_URL}" +echo " worker-0 : http://127.0.0.1:${WORKER_PORT}" +echo " master : http://127.0.0.1:${MASTER_PORT} (UI + admin API)" +echo " data dir : ${DATA_DIR}" +echo " logs : ${LOG_DIR}" + +if [[ "${1:-}" == "--smoke" ]]; then + log "running smoke test" + MASTER_URL="http://127.0.0.1:${MASTER_PORT}" \ + WORKER_URL="http://127.0.0.1:${WORKER_PORT}" \ + "$(dirname "${BASH_SOURCE[0]}")/smoke.sh" +fi