feat(stovepipe): add orchestrator service (#182)

Set up stovepipe orchestrator service and ensure that the stack can be
launched.

## Why?
Similar to submitqueue, we will use the gateway service for API
handling, and an orchestrator service to process work. This will give us
more flexibility in deployment.

## What?
Add separate orchestrator layer which will mainly be used to provide a
controller that can be injected via fx and deployed.

## Test Plan
 - make local-stovepipe-start
 - CI

Author: mnoah1

BUILD.bazel

@@ -6,6 +6,7 @@ load("@gazelle//:def.bzl", "gazelle")
 # gazelle:resolve go github.com/uber/submitqueue/gateway/protopb //gateway/protopb
 # gazelle:resolve go github.com/uber/submitqueue/orchestrator/protopb //orchestrator/protopb
 # gazelle:resolve go github.com/uber/submitqueue/stovepipe/gateway/protopb //stovepipe/gateway/protopb
+# gazelle:resolve go github.com/uber/submitqueue/stovepipe/orchestrator/protopb //stovepipe/orchestrator/protopb
 
 # Export marker files for test data dependencies (used by FindRepoRoot in tests)
 exports_files(

Makefile

@@ -14,6 +14,8 @@ SUBMITQUEUE_LOCAL_PROJECT = submitqueue
 
 # Stovepipe compose files
 STOVEPIPE_GATEWAY_COMPOSE_FILE = example/stovepipe/gateway/server/docker-compose.yml
+STOVEPIPE_ORCHESTRATOR_COMPOSE_FILE = example/stovepipe/orchestrator/server/docker-compose.yml
+STOVEPIPE_STACK_COMPOSE_FILE = example/stovepipe/docker-compose.yml
 
 # Fixed project name for local manual testing (tests use unique random names)
 STOVEPIPE_LOCAL_PROJECT = stovepipe
@@ -38,7 +40,7 @@ define assert_clean
 	fi
 endef
 
-.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux build-stovepipe-gateway-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-consumer integration-test-extensions integration-test-gateway integration-test-orchestrator license-fix lint lint-fmt lint-license local-clean local-gateway-start local-gateway-stop local-init-schemas local-init-stovepipe-queue-schema local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop local-stovepipe-gateway-start mocks proto query-deps query-targets run-client-gateway run-client-orchestrator run-client-stovepipe-gateway run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help
+.PHONY: build build-all-linux build-gateway-linux build-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-consumer integration-test-extensions integration-test-gateway integration-test-orchestrator license-fix lint lint-fmt lint-license local-clean local-gateway-start local-gateway-stop local-init-schemas local-init-stovepipe-queue-schema local-logs local-orchestrator-start local-orchestrator-stop local-ps local-restart local-start local-stop local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-gateway run-client-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help
 
 
 build: ## Build all services and examples
@@ -47,7 +49,7 @@ build: ## Build all services and examples
 	@echo "Build complete!"
 
 # Build Linux binaries required for Docker containers
-build-all-linux: build-gateway-linux build-orchestrator-linux build-stovepipe-gateway-linux ## Build all Linux binaries for Docker
+build-all-linux: build-gateway-linux build-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux ## Build all Linux binaries for Docker
 	@echo "All Linux binaries ready for Docker"
 
 build-gateway-linux: ## Build Gateway Linux binary for Docker
@@ -74,6 +76,14 @@ build-stovepipe-gateway-linux: ## Build Stovepipe gateway Linux binary for Docke
 	 cp -f bazel-bin/example/stovepipe/gateway/server/gateway .docker-bin/stovepipe-gateway
 	@echo "Stovepipe gateway Linux binary ready at .docker-bin/stovepipe-gateway"
 
+build-stovepipe-orchestrator-linux: ## Build Stovepipe orchestrator Linux binary for Docker
+	@echo "Building Stovepipe orchestrator Linux binary for Docker..."
+	@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/stovepipe/orchestrator/server:orchestrator
+	@mkdir -p .docker-bin
+	@cp -f bazel-bin/example/stovepipe/orchestrator/server/orchestrator_/orchestrator .docker-bin/stovepipe-orchestrator 2>/dev/null || \
+	 cp -f bazel-bin/example/stovepipe/orchestrator/server/orchestrator .docker-bin/stovepipe-orchestrator
+	@echo "Stovepipe orchestrator Linux binary ready at .docker-bin/stovepipe-orchestrator"
+
 check-gazelle: ## Check BUILD.bazel files are up to date
 	@echo "Running Gazelle to check BUILD files..."
 	@$(BAZEL) run //:gazelle
@@ -99,6 +109,7 @@ clean-proto: ## Clean generated proto files
 	@rm -rf gateway/protopb/*.pb.go
 	@rm -rf orchestrator/protopb/*.pb.go
 	@rm -rf stovepipe/gateway/protopb/*.pb.go
+	@rm -rf stovepipe/orchestrator/protopb/*.pb.go
 	@echo "Proto clean complete!"
 
 deps: tidy-go ## Download and tidy Go dependencies
@@ -266,9 +277,47 @@ local-start: build-all-linux ## Start full stack (Gateway + Orchestrator + MySQL
 local-stop: ## Stop all services (keep data)
 	@echo "Stopping all services..."
 	@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
-	@$(COMPOSE) -f $(STOVEPIPE_GATEWAY_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down
+	@$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down
 	@echo "Services stopped. Data volumes preserved."
 
+local-stovepipe-logs: ## View logs from all running Stovepipe services
+	@$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) logs -f
+
+local-stovepipe-start: build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux ## Start full Stovepipe stack (gateway + orchestrator + MySQL)
+	@echo "Starting full Stovepipe stack with compose..."
+	@$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) up -d --build --wait
+	@echo "Applying queue schema to mysql-queue (no Stovepipe app schema yet)..."
+	@$(MAKE) -s local-init-stovepipe-queue-schema
+	@echo ""
+	@echo "✅ Full Stovepipe stack is running!"
+	@echo ""
+	@echo "Expected container NAMEs (project=$(STOVEPIPE_LOCAL_PROJECT), one replica each):"
+	@echo "  $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1"
+	@echo "  $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1"
+	@echo "  $(STOVEPIPE_LOCAL_PROJECT)-gateway-service-1"
+	@echo "  $(STOVEPIPE_LOCAL_PROJECT)-orchestrator-service-1"
+	@echo ""
+	@$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) ps
+	@echo ""
+	@echo "Stovepipe gateway gRPC port:       $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+	@echo "Stovepipe orchestrator gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+	@echo "MySQL App port:                    $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+	@echo "MySQL Queue port:                  $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+
+local-stovepipe-orchestrator-start: build-stovepipe-orchestrator-linux ## Start Stovepipe orchestrator locally (orchestrator + 2 MySQL databases)
+	@echo "Starting Stovepipe orchestrator with compose..."
+	@$(COMPOSE) -f $(STOVEPIPE_ORCHESTRATOR_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) up -d --build --wait
+	@echo "Applying queue schema to mysql-queue (no Stovepipe app schema yet)..."
+	@$(MAKE) -s local-init-stovepipe-queue-schema
+	@echo ""
+	@echo "✅ Stovepipe orchestrator is running!"
+	@echo ""
+	@$(COMPOSE) -f $(STOVEPIPE_ORCHESTRATOR_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) ps
+	@echo ""
+	@echo "Stovepipe orchestrator gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+	@echo "MySQL App port:         $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+	@echo "MySQL Queue port:       $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+
 local-stovepipe-gateway-start: build-stovepipe-gateway-linux ## Start Stovepipe gateway locally (gateway + 2 MySQL databases)
 	@echo "Starting Stovepipe gateway with compose..."
 	@$(COMPOSE) -f $(STOVEPIPE_GATEWAY_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) up -d --build --wait
@@ -279,7 +328,7 @@ local-stovepipe-gateway-start: build-stovepipe-gateway-linux ## Start Stovepipe
 	@echo ""
 	@$(COMPOSE) -f $(STOVEPIPE_GATEWAY_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) ps
 	@echo ""
-	@echo "Stovepipe gateway gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-stovepipe-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
+	@echo "Stovepipe gateway gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
 	@echo "MySQL App port:    $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"
 	@echo "MySQL Queue port:  $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"
 
@@ -302,6 +351,10 @@ proto: ## Generate protobuf files from .proto definitions
 	  --go-grpc_out=stovepipe/gateway/protopb --go-grpc_opt=paths=source_relative \
 	  --yarpc-go_out=stovepipe/gateway/protopb --yarpc-go_opt=paths=source_relative \
 	  --proto_path=stovepipe/gateway/proto stovepipe/gateway/proto/gateway.proto
+	@protoc --go_out=stovepipe/orchestrator/protopb --go_opt=paths=source_relative \
+	  --go-grpc_out=stovepipe/orchestrator/protopb --go-grpc_opt=paths=source_relative \
+	  --yarpc-go_out=stovepipe/orchestrator/protopb --yarpc-go_opt=paths=source_relative \
+	  --proto_path=stovepipe/orchestrator/proto stovepipe/orchestrator/proto/orchestrator.proto
 	@echo "Protobuf files generated successfully!"
 
 # Bazel query helpers
@@ -323,6 +376,10 @@ run-client-orchestrator:
 run-client-stovepipe-gateway:
 	@$(BAZEL) run //example/stovepipe/gateway/client:gateway -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)"
 
+# Run stovepipe orchestrator client (connects to any running stovepipe orchestrator service)
+run-client-stovepipe-orchestrator:
+	@$(BAZEL) run //example/stovepipe/orchestrator/client:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8084) -message "$(or $(MESSAGE),ping)"
+
 run-queue-admin: ## Run queue-admin CLI (use ARGS to pass arguments, e.g. make run-queue-admin ARGS="list-topics")
 	@$(BAZEL) run //extension/queue/mysql/ctl -- $(ARGS)
 

example/README.md

@@ -7,6 +7,7 @@ Example gRPC servers and clients for running the submitqueue services locally.
 - **SubmitQueue Gateway** (port 8081) — entry point for land requests. Exposes `Ping` and `Land` RPCs.
 - **SubmitQueue Orchestrator** (port 8082) — coordinates the pipeline. Exposes `Ping` RPC and consumes queue messages across 9 pipeline topics.
 - **Stovepipe Gateway** (port 8083) - entry point for commit deployment verification requests. Exposes `Ping` RPC.
+- **Stovepipe Orchestrator** (port 8084) - coordinates the commit verification pipeline. Exposes `Ping` RPC.
 
 Services require MySQL (app database + queue database). Docker Compose handles this automatically.
 
@@ -28,12 +29,19 @@ example/
 │   ├── gateway/main.go             # Gateway ping client
 │   └── orchestrator/main.go        # Orchestrator ping client
 └── stovepipe/
-    └── gateway/
+    ├── docker-compose.yml          # Full stack (Stovepipe Gateway + Orchestrator + 2x MySQL)
+    ├── gateway/
+    │   ├── server/
+    │   │   ├── main.go             # Stovepipe gateway gRPC server (Docker: :8080; go run default :8083)
+    │   │   ├── Dockerfile
+    │   │   └── docker-compose.yml  # Gateway-only stack
+    │   └── client/main.go          # Stovepipe gateway ping client
+    └── orchestrator/
         ├── server/
-        │   ├── main.go             # Stovepipe gateway gRPC server (Docker: :8080; go run default :8083)
+        │   ├── main.go             # Stovepipe orchestrator gRPC server (Docker: :8080; go run default :8084)
         │   ├── Dockerfile
-        │   └── docker-compose.yml  # Gateway-only stack
-        └── client/main.go          # Stovepipe gateway ping client
+        │   └── docker-compose.yml  # Orchestrator-only stack
+        └── client/main.go          # Stovepipe orchestrator ping client
 ```
 
 ## Running
@@ -47,8 +55,10 @@ make local-start
 make local-gateway-start
 make local-orchestrator-start
 
-# Start Stovepipe gateway (Gateway + 2x MySQL)
+# Start full Stovepipe stack (Gateway + Orchestrator + MySQL)
+make local-stovepipe-start
 make local-stovepipe-gateway-start
+make local-stovepipe-orchestrator-start
 
 # View logs and status
 make local-logs
@@ -58,7 +68,7 @@ make local-ps
 make local-stop
 ```
 
-For Docker, `make build-stovepipe-gateway-linux` copies a Linux binary to `.docker-bin/stovepipe-gateway` so it does not overwrite SubmitQueue’s `.docker-bin/gateway`. Stovepipe `make local-stovepipe-gateway-start` applies **only the queue schema** on `mysql-queue` (`make local-init-stovepipe-queue-schema`); SubmitQueue storage/counter schemas on `mysql-app` are skipped until Stovepipe has its own app schema. `make local-stop` stops the SubmitQueue stack and runs `docker compose down` on the Stovepipe gateway compose file for **`STOVEPIPE_LOCAL_PROJECT`** (default `stovepipe`). SubmitQueue examples use project **`submitqueue`** by default (`make SUBMITQUEUE_LOCAL_PROJECT=myname ...`). Stovepipe containers are named like `stovepipe-mysql-app-1`, not `submitqueue-*`.
+For Docker, `make build-stovepipe-gateway-linux` copies a Linux binary to `.docker-bin/stovepipe-gateway` so it does not overwrite SubmitQueue’s `.docker-bin/gateway`. Stovepipe `make local-stovepipe-gateway-start` applies **only the queue schema** on `mysql-queue` (`make local-init-stovepipe-queue-schema`); SubmitQueue storage/counter schemas on `mysql-app` are skipped until Stovepipe has its own app schema. Compose service keys are **`gateway-service`** and **`orchestrator-service`** (same as `example/server`), so with default project **`stovepipe`** you should see **`stovepipe-gateway-service-1`** and **`stovepipe-orchestrator-service-1`** — not `stovepipe-stovepipe-*` (that pattern was from older service names; run **`make local-stop`** to run `docker compose down --remove-orphans` and drop orphans). `make local-stop` also stops the SubmitQueue stack. SubmitQueue examples use project **`submitqueue`** by default (`make SUBMITQUEUE_LOCAL_PROJECT=myname ...`). Stovepipe containers are named like `stovepipe-mysql-app-1`, not `submitqueue-*`.
 
 ### Bazel
 
@@ -67,11 +77,13 @@ For Docker, `make build-stovepipe-gateway-linux` copies a Linux binary to `.dock
 bazel build //example/server/gateway:gateway
 bazel build //example/server/orchestrator:orchestrator
 bazel build //example/stovepipe/gateway/server:gateway
+bazel build //example/stovepipe/orchestrator/server:orchestrator
 
 # Build clients
 bazel build //example/client/gateway:gateway
 bazel build //example/client/orchestrator:orchestrator
 bazel build //example/stovepipe/gateway/client:gateway
+bazel build //example/stovepipe/orchestrator/client:orchestrator
 ```
 
 ### Go
@@ -80,6 +92,7 @@ bazel build //example/stovepipe/gateway/client:gateway
 go run example/server/gateway/main.go
 go run example/server/orchestrator/main.go
 go run example/stovepipe/gateway/server/main.go
+go run example/stovepipe/orchestrator/server/main.go
 ```
 
 ## Testing with Clients
@@ -89,6 +102,7 @@ go run example/stovepipe/gateway/server/main.go
 go run example/client/gateway/main.go -addr localhost:8081 -message "hello"
 go run example/client/orchestrator/main.go -addr localhost:8082 -message "hello"
 go run example/stovepipe/gateway/client/main.go -addr localhost:8083 -message "hello"
+go run example/stovepipe/orchestrator/client/main.go -addr localhost:8084 -message "hello"
 ```
 
 Client flags:
@@ -110,16 +124,19 @@ go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest
 grpcurl -plaintext -d '{"message": "hello"}' localhost:8081 uber.submitqueue.gateway.SubmitQueueGateway/Ping
 grpcurl -plaintext -d '{"message": "hello"}' localhost:8082 uber.submitqueue.orchestrator.SubmitQueueOrchestrator/Ping
 grpcurl -plaintext -d '{"message": "hello"}' localhost:8083 uber.submitqueue.stovepipe.StovepipeGateway/Ping
+grpcurl -plaintext -d '{"message": "hello"}' localhost:8084 uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator/Ping
 
 # List services
 grpcurl -plaintext localhost:8081 list
 grpcurl -plaintext localhost:8082 list
 grpcurl -plaintext localhost:8083 list
+grpcurl -plaintext localhost:8084 list
 
 # Describe a service
 grpcurl -plaintext localhost:8081 describe uber.submitqueue.gateway.SubmitQueueGateway
 grpcurl -plaintext localhost:8082 describe uber.submitqueue.orchestrator.SubmitQueueOrchestrator
 grpcurl -plaintext localhost:8083 describe uber.submitqueue.stovepipe.StovepipeGateway
+grpcurl -plaintext localhost:8084 describe uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator
 ```
 
 ## API Reference
@@ -151,3 +168,12 @@ grpcurl -plaintext localhost:8083 describe uber.submitqueue.stovepipe.StovepipeG
 | Method | Description |
 |--------|-------------|
 | `Ping` | Health check |
+
+### Stovepipe Orchestrator
+
+**Service**: `uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator`
+**Proto**: `stovepipe/orchestrator/proto/orchestrator.proto`
+
+| Method | Description |
+|--------|-------------|
+| `Ping` | Health check |

example/stovepipe/BUILD.bazel

@@ -0,0 +1,4 @@
+exports_files(
+    ["docker-compose.yml"],
+    visibility = ["//visibility:public"],
+)