From af7b749a126b3ad15d16ff63e87da5c71590825b Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sat, 7 Nov 2026 12:04:40 +0100 Subject: [PATCH] feat(infra): add lightweight Docker Compose quickstart Add a root docker-compose.yml that boots PostgreSQL 16.15-alpine, a single-broker KRaft Apache Kafka 4.3.1 cluster and the GraalVM native Kates API (ghcr.io/bmscomp/kates:1-native, which bundles the kates CLI) in ~8 s cached and ~433 MiB idle RAM with no Kind or Kubernetes required. Document the 60-second Docker Compose evaluation path in README.md alongside the multi-zone Kind lab, and assert the Compose Kafka and PostgreSQL image pins in scripts/check-versions.sh. --- .github/workflows/ci.yml | 2 + README.md | 32 ++++++++++++ docker-compose.yml | 105 ++++++++++++++++++++++++++++++++++++++ scripts/check-versions.sh | 32 ++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 docker-compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6824051..22369338 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,8 @@ jobs: - 'Dockerfile.legacy-kafka' # ...and the tester's Kafka CLI and jar overrides against it. - 'tester/**' + # ...and the Docker Compose quickstart's Kafka and Postgres pins. + - 'docker-compose.yml' yaml: - 'config/**' diff --git a/README.md b/README.md index d8aeabde..13356266 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,38 @@ Most teams validate Kafka resilience by running `kafka-producer-perf-test` in on ## Quick start +Pick the setup that matches what you want to try: + +- **Docker Compose (< 1 GB RAM, no Kubernetes):** starts a single-broker KRaft Kafka cluster, PostgreSQL and the GraalVM native Kates API (with the `kates` CLI bundled inside the container) for performance benchmarks, data-integrity checks, CI quality gates and broker security audits — or points at an existing Kafka broker. +- **Multi-zone Kubernetes lab on Kind (~16 GB RAM):** builds the full stack with Strimzi, rack-aware brokers across three zones, LitmusChaos fault injection, Prometheus and twelve Grafana dashboards. + +### Try it in 60 seconds (Docker Compose) + +Needs only Docker with `docker compose` on macOS or Linux (amd64 or arm64). Download [`docker-compose.yml`](docker-compose.yml) — or run from a checkout — and start the stack: + +```bash +curl -fsSLO https://raw.githubusercontent.com/bmscomp/kates/main/docker-compose.yml +docker compose up -d --wait + +# Check health, run a 50,000-record load test, enforce a CI grade and audit the broker +docker compose exec kates kates health +docker compose exec kates kates test create --type LOAD --records 50000 --wait +docker compose exec kates kates gate --min-grade C --records 50000 +docker compose exec kates kates security audit +``` + +Once the images are cached, the stack starts in under ten seconds, idles at about 450 MB of RAM across all three containers and stays under 800 MB during benchmarks. Interactive terminal views work inside the container too (`docker compose exec kates kates kafka tui`, `docker compose exec kates kates dashboard`), or directly from a host CLI (`brew install bmscomp/tap/kates`) against `http://localhost:8080`, where API-key auth is off for local evaluation (`KATES_PORT` changes the host port). + +To test an existing broker instead, pass `KATES_KAFKA_BOOTSTRAP_SERVERS=host.docker.internal:9092` (and `KATES_REPLICATION_FACTOR=3 KATES_MIN_INSYNC_REPLICAS=2` on a multi-broker cluster). Fault injection (`kates disruption`, `kates resilience`) requires Kubernetes — use the Kind lab below. + +Tear down the containers and volumes when finished: + +```bash +docker compose down -v +``` + +### Multi-zone Kubernetes lab (Kind) + This path builds the whole stack on a local Kind cluster. You need: - a clone of this repository, for the `make` targets, which are read relative to the working directory, so run every command from the repository root; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..68585352 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,105 @@ +# Copyright 2026 Kates Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Lightweight local evaluation stack (< 1 GB RAM, no Kind or Kubernetes needed). +# Starts a single-broker KRaft Apache Kafka cluster, PostgreSQL and the +# GraalVM native Kates API (which also bundles the `kates` CLI). +# +# Quick start: +# docker compose up -d --wait +# docker compose exec kates kates health +# docker compose exec kates kates test create --type LOAD --records 50000 --wait +# docker compose exec kates kates gate --min-grade C --records 50000 +# docker compose exec kates kates security audit +# docker compose down -v + +services: + postgres: + image: postgres:16.15-alpine + command: + - postgres + - -c + - shared_buffers=32MB + - -c + - max_connections=30 + environment: + POSTGRES_DB: kates + POSTGRES_USER: kates + POSTGRES_PASSWORD: kates + healthcheck: + test: ["CMD-SHELL", "pg_isready -U kates -d kates"] + interval: 2s + timeout: 2s + retries: 15 + start_period: 2s + + kafka: + image: apache/kafka:4.3.1 + environment: + KAFKA_NODE_ID: "1" + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1" + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1" + KAFKA_DEFAULT_REPLICATION_FACTOR: "1" + KAFKA_MIN_INSYNC_REPLICAS: "1" + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: "0" + KAFKA_NUM_PARTITIONS: "3" + KAFKA_HEAP_OPTS: "-Xms192m -Xmx384m" + healthcheck: + test: ["CMD", "/opt/kafka/bin/kafka-broker-api-versions.sh", "--bootstrap-server", "localhost:9092"] + interval: 3s + timeout: 5s + retries: 20 + start_period: 5s + + kates: + image: ${KATES_IMAGE:-ghcr.io/bmscomp/kates:1-native} + command: ["-XX:MaximumHeapSizePercent=75", "-Xmx256m"] + depends_on: + postgres: + condition: service_healthy + kafka: + condition: service_healthy + ports: + - "127.0.0.1:${KATES_PORT:-8080}:8080" + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/kates + KATES_DB_USERNAME: kates + KATES_DB_PASSWORD: kates + KATES_KAFKA_BOOTSTRAP_SERVERS: ${KATES_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + KATES_KAFKA_SECURITY_PROTOCOL: ${KATES_KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + KATES_API_SECURITY_ENABLED: ${KATES_API_SECURITY_ENABLED:-false} + KATES_API_KEY: ${KATES_API_KEY:-} + KATES_CHAOS_PROVIDER: noop + KATES_CHAOS_ORPHAN_RECOVERY_ENABLED: "false" + QUARKUS_OTEL_ENABLED: "false" + # Single-broker defaults across all workload profiles (override via + # KATES_REPLICATION_FACTOR / KATES_MIN_INSYNC_REPLICAS for multi-broker targets). + KATES_TESTS_LOAD_REPLICATION_FACTOR: ${KATES_REPLICATION_FACTOR:-1} + KATES_TESTS_LOAD_MIN_INSYNC_REPLICAS: ${KATES_MIN_INSYNC_REPLICAS:-1} + KATES_TESTS_STRESS_REPLICATION_FACTOR: ${KATES_REPLICATION_FACTOR:-1} + KATES_TESTS_STRESS_MIN_INSYNC_REPLICAS: ${KATES_MIN_INSYNC_REPLICAS:-1} + KATES_TESTS_SPIKE_REPLICATION_FACTOR: ${KATES_REPLICATION_FACTOR:-1} + KATES_TESTS_SPIKE_MIN_INSYNC_REPLICAS: ${KATES_MIN_INSYNC_REPLICAS:-1} + KATES_TESTS_ENDURANCE_REPLICATION_FACTOR: ${KATES_REPLICATION_FACTOR:-1} + KATES_TESTS_ENDURANCE_MIN_INSYNC_REPLICAS: ${KATES_MIN_INSYNC_REPLICAS:-1} + KATES_TESTS_VOLUME_REPLICATION_FACTOR: ${KATES_REPLICATION_FACTOR:-1} + KATES_TESTS_VOLUME_MIN_INSYNC_REPLICAS: ${KATES_MIN_INSYNC_REPLICAS:-1} + KATES_TESTS_CAPACITY_REPLICATION_FACTOR: ${KATES_REPLICATION_FACTOR:-1} + KATES_TESTS_CAPACITY_MIN_INSYNC_REPLICAS: ${KATES_MIN_INSYNC_REPLICAS:-1} + KATES_TESTS_ROUNDTRIP_REPLICATION_FACTOR: ${KATES_REPLICATION_FACTOR:-1} + KATES_TESTS_ROUNDTRIP_MIN_INSYNC_REPLICAS: ${KATES_MIN_INSYNC_REPLICAS:-1} + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8080/q/health/ready"] + interval: 2s + timeout: 2s + retries: 15 + start_period: 2s diff --git a/scripts/check-versions.sh b/scripts/check-versions.sh index 7e3e2824..fe355724 100755 --- a/scripts/check-versions.sh +++ b/scripts/check-versions.sh @@ -705,6 +705,38 @@ else echo "OK: kates-tester ships the Kafka ${tester_kafka} CLI, and its jar overrides are written for it." fi +# The Docker Compose quickstart pins Apache Kafka and PostgreSQL to the same +# versions as the repo's Kafka pin and charts/kates/values.yaml so local +# evaluation never drifts behind the charts. +COMPOSE_YAML="docker-compose.yml" +KATES_VALUES_YAML="charts/kates/values.yaml" +compose_kafka=$(grep -Eo 'apache/kafka:[0-9]+\.[0-9]+\.[0-9]+' "$COMPOSE_YAML" | head -1 | cut -d: -f2 || true) +compose_pg=$(grep -Eo 'postgres:[0-9]+\.[0-9]+(-alpine)?' "$COMPOSE_YAML" | head -1 | cut -d: -f2 || true) +chart_pg=$(awk ' + /^postgresql:/ { in_pg = 1; next } + in_pg && /^[^[:space:]]/ { in_pg = 0 } + in_pg && /^ image:/ { in_img = 1; next } + in_img && /^ [a-z]/ { in_img = 0 } + in_img && /^ tag:/ { gsub(/^[[:space:]]*tag:[[:space:]]*"?/, ""); gsub(/"?[[:space:]]*$/, ""); print; exit } +' "$KATES_VALUES_YAML") +echo "" +echo "Docker Compose quickstart:" +printf ' %-46s %s\n' "${COMPOSE_YAML} apache/kafka:" "${compose_kafka:-}" +printf ' %-46s %s\n' "${COMPOSE_YAML} postgres:" "${compose_pg:-}" +printf ' %-46s %s\n' "${KATES_VALUES_YAML} postgresql.image.tag:" "${chart_pg:-}" +if [[ -z "$compose_kafka" || -z "$compose_pg" || -z "$chart_pg" ]]; then + echo "ERROR: could not read one of the Docker Compose or PostgreSQL image pins" >&2 + fail=1 +elif [[ "$compose_kafka" != "$env_kafka_version" ]]; then + echo "DRIFT: ${COMPOSE_YAML} runs apache/kafka:${compose_kafka}, the Kafka pin is ${env_kafka_version}." >&2 + fail=1 +elif [[ "$compose_pg" != "$chart_pg" ]]; then + echo "DRIFT: ${COMPOSE_YAML} runs postgres:${compose_pg}, ${KATES_VALUES_YAML} runs postgres:${chart_pg}." >&2 + fail=1 +else + echo "OK: ${COMPOSE_YAML} agrees with Kafka ${env_kafka_version} and PostgreSQL ${chart_pg}." +fi + check_workflows || fail=1 exit "$fail"