From 00ab216f40238ab4e6d1fec31f63310292e1d9bf Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:50:49 +0000 Subject: [PATCH 1/4] feat(infra): add Azure Hub deployment Provision durable Azure resources with Terraform and serialize maintenance releases around the separate setup job. Document protected state, ownership, recovery, and teardown without changing the devcontainer. Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/hub-infrastructure.yml | 37 +++ docs/README.md | 1 + hub/docs/deployment.md | 2 + infrastructure/.gitignore | 9 + infrastructure/README.md | 146 +++++++++++ infrastructure/azure/.terraform.lock.hcl | 22 ++ infrastructure/azure/backend.hcl.example | 6 + infrastructure/azure/compute.tf | 197 +++++++++++++++ infrastructure/azure/deploy.py | 235 ++++++++++++++++++ .../azure/installation.tfvars.example | 11 + infrastructure/azure/main.tf | 199 +++++++++++++++ infrastructure/azure/outputs.tf | 21 ++ infrastructure/azure/test_deploy.py | 143 +++++++++++ .../azure/tests/deployment.tftest.hcl | 147 +++++++++++ infrastructure/azure/variables.tf | 118 +++++++++ infrastructure/azure/versions.tf | 24 ++ 16 files changed, 1318 insertions(+) create mode 100644 .github/workflows/hub-infrastructure.yml create mode 100644 infrastructure/.gitignore create mode 100644 infrastructure/README.md create mode 100644 infrastructure/azure/.terraform.lock.hcl create mode 100644 infrastructure/azure/backend.hcl.example create mode 100644 infrastructure/azure/compute.tf create mode 100644 infrastructure/azure/deploy.py create mode 100644 infrastructure/azure/installation.tfvars.example create mode 100644 infrastructure/azure/main.tf create mode 100644 infrastructure/azure/outputs.tf create mode 100644 infrastructure/azure/test_deploy.py create mode 100644 infrastructure/azure/tests/deployment.tftest.hcl create mode 100644 infrastructure/azure/variables.tf create mode 100644 infrastructure/azure/versions.tf diff --git a/.github/workflows/hub-infrastructure.yml b/.github/workflows/hub-infrastructure.yml new file mode 100644 index 00000000..bca57411 --- /dev/null +++ b/.github/workflows/hub-infrastructure.yml @@ -0,0 +1,37 @@ +name: Hub infrastructure + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +defaults: + run: + working-directory: infrastructure/azure + +jobs: + terraform: + name: Terraform ${{ matrix.command }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + command: ["fmt -check -recursive", "validate", "test"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.13.4 + terraform_wrapper: false + - run: terraform init -backend=false -input=false -lockfile=readonly + - run: terraform ${{ matrix.command }} + + release: + name: Release orchestration tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - run: python3 -m unittest discover -s . -p 'test_*.py' -v diff --git a/docs/README.md b/docs/README.md index 418f7b2a..fbc8adb0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ Commits and pull requests follow the [contribution conventions](../CONTRIBUTING. ## Project guidance - [Hub documentation](../hub/docs/README.md) covers starting and checking the TypeScript API and frontend, and what is not built yet. +- [Azure Hub installation](../infrastructure/README.md) covers Terraform, release ordering, ownership, and recovery. - [Starter instructions](../starter/AGENTS.md) and [setup guide](../starter/SETUP.md) cover the copyable prototype app and skills. - [Build action setup](../CONTRIBUTING.md#replica-build-action) covers the shared action. - [Controls documentation](../hub/docs/prototype-controls.md) covers the shared engine and definition schema owned by Hub. diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index 591f04c1..e2d30d41 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -41,4 +41,6 @@ The deployment orchestrator owns serialization of the complete release, includin CI checks the image against disposable PostgreSQL, including setup, frontend/API serving, readiness, and termination. This is not a supported local-production installation. The existing devcontainer remains the development environment. +The [Azure reference installation](../../infrastructure/README.md) uses Terraform and a serialized maintenance-window release. It consumes the same image and runtime settings without adding Azure deployment logic to Hub. + An actual cloud deployment must additionally verify credentials, networking, data survival, backup and restore, domain configuration, and release ordering in an authorized environment. Never commit secrets, tenant-specific parameter files, or generated service addresses. diff --git a/infrastructure/.gitignore b/infrastructure/.gitignore new file mode 100644 index 00000000..153bcaf3 --- /dev/null +++ b/infrastructure/.gitignore @@ -0,0 +1,9 @@ +.tools/ +**/__pycache__/ +**/.terraform/ +**/*.tfstate* +**/*.tfplan +**/*.tfvars +**/*.tfvars.json +**/backend.hcl +**/crash.log diff --git a/infrastructure/README.md b/infrastructure/README.md new file mode 100644 index 00000000..349222e2 --- /dev/null +++ b/infrastructure/README.md @@ -0,0 +1,146 @@ +# Azure Hub installation + +Operators can provision a durable Hub, run setup before every release, and replace the image without replacing its database or artifacts. This reference chooses Terraform with AzureRM to follow the team's Terraform experience; it is an implementation choice, not an application dependency. The configuration is flat under [`azure/`](azure/) rather than split into one-resource modules. + +This installs the current Hub, not hosted previews, review collaboration, user admission, or a sign-in provider. A public HTTPS endpoint does not authorize users or isolate prototype previews. Confirm product access requirements before exposing sensitive data. + +## What is created + +Each installation has its own resource group tagged with its application, installation name, owner, and Terraform ownership. It contains: + +- A virtual network, delegated Container Apps and PostgreSQL subnets, and private PostgreSQL DNS. +- PostgreSQL Flexible Server, the `replica` database, enforced secure transport, and configurable point-in-time recovery retention. +- An Azure Storage account with private `artifacts` and `releases` containers, Blob versioning, and 14-day Blob and container soft deletion. Shared keys and anonymous access are disabled. The Blob endpoint is public-network reachable but requires Entra authorization; this is not a private-endpoint installation. +- One user-assigned workload identity with `Storage Blob Data Contributor` on **only** the artifacts container. An optional existing ACR receives an `AcrPull` assignment for that identity. No registry is created or owned. +- Log Analytics with 30-day retention, a Container Apps environment, a manual setup job, and, after successful setup, one combined Hub app. + +The app has external HTTPS ingress and minimum and maximum one replica **per revision**. Multiple-revision mode permits explicit deactivation before releases. Azure's single-revision mode deliberately overlaps old and new processes while readiness is checked, so it is unsuitable for this maintenance-window strategy. Do not activate old revisions alongside the combined worker. This is not a high-availability or zero-downtime installation. + +The runtime receives `RUNTIME_MODE=combined`, `HOST=0.0.0.0`, `PORT=3001`, `CLIENT_DIRECTORY=/app/dist/client`, `SHUTDOWN_TIMEOUT_MS=30000`, and `NODE_ENV=production`. Artifacts use `ARTIFACT_STORAGE_PROVIDER=azure`, the provisioned `AZURE_STORAGE_ACCOUNT`, `AZURE_STORAGE_CONTAINER=artifacts`, and the workload's `AZURE_CLIENT_ID`. `DATABASE_URL` is a Container Apps secret using `sslmode=verify-full`. Additional `runtime_secrets` are explicit, sensitive environment-name/value inputs; none are Terraform outputs. + +The setup job uses the same digest, `RUNTIME_MODE=setup`, and command `node dist/server/scripts/setup/index.js`. It has one replica, no automatic retries, and a 30-minute timeout. Hub's readiness probe is `/api/ready`; startup and liveness use `/api/health`. Azure allows 60 seconds for termination, covering the 30-second application drain and up to five seconds of telemetry flushing with platform margin. + +The reference uses the PostgreSQL administrator login for both setup and runtime so AzureRM alone can provision it without another database provider or remote SQL provisioner. Operators requiring separate migration/runtime roles must provision and test those grants before adapting this reference. Managed identity is used for Blob and ACR, not database authentication. + +## Prepare access and protected state + +Use the Terraform version constrained in [`versions.tf`](azure/versions.tf), Python 3, and a current Azure CLI with Container Apps job and revision commands. CI performs formatting, validation, mocked Terraform tests, and release-orchestration tests without cloud credentials. It does not deploy. + +An authorized administrator must first create the **separate** Terraform backend: a dedicated state resource group, Storage account, and private container. Enable TLS, Entra authorization, Blob versioning and soft deletion; restrict network access to deployment runners and grant only the deployment identity `Storage Blob Data Contributor` on the state container. These bootstrap resources are not part of this installation's state or teardown. Never store this state in the app's artifacts account. + +For automation, configure a dedicated deployment identity with a federated workload identity credential rather than a client secret. Bind its issuer to `https://token.actions.githubusercontent.com`, audience to `api://AzureADTokenExchange`, and subject to the exact protected repository/environment, such as `repo:YOUR_ORG/YOUR_REPO:environment:hub-production`. Protect that environment and deployment branch. A future release workflow needs `id-token: write` and must authenticate Azure CLI as the same identity; the infrastructure-check workflow intentionally has no such permission. + +Set `ARM_CLIENT_ID`, `ARM_TENANT_ID`, `ARM_SUBSCRIPTION_ID`, and `ARM_USE_OIDC=true` in that authorized runner. Terraform's AzureRM backend supports GitHub's OIDC request environment; other runners must supply their supported OIDC token source. For local authorized operation, use Azure CLI sign-in and set `use_oidc=false` in your private backend configuration instead. Do not copy tokens into Terraform inputs. + +The deployment identity needs resource creation/update rights for the installation, role-assignment rights for the container-scoped Blob grant and optional registry grant, and Blob data-plane rights on the installation's account for the lock blob. Resource `Contributor` alone does not grant Blob data access or role assignment. An administrator must authorize the initial resource-group creation and inherited Blob data access for bootstrap, then narrow ongoing deployment grants to that installation resource group, the existing registry, and the backend container. Precreated/imported infrastructure needs a separately reviewed initial plan rather than the empty-state bootstrap command. Do not grant the workload identity deployment or state access. + +Register `Microsoft.App`, `Microsoft.OperationalInsights`, `Microsoft.Network`, `Microsoft.Storage`, `Microsoft.DBforPostgreSQL`, and `Microsoft.ManagedIdentity` beforehand. The provider does not register services across the subscription. Allow time for newly assigned RBAC roles to propagate before setup. + +Copy [`backend.hcl.example`](azure/backend.hcl.example) to `infrastructure/azure/backend.hcl` and [`installation.tfvars.example`](azure/installation.tfvars.example) to `infrastructure/azure/installation.tfvars`. Both real files are ignored. Replace every placeholder, choose a globally unique name, and resolve an existing image to its real `registry/repository@sha256:...` digest. Use either a public image or an existing ACR; private non-ACR registry credentials and ACR's ABAC repository-permissions mode are not covered by this reference. + +Supply `postgres_admin_password` from a secret manager as `TF_VAR_postgres_admin_password`, or through the ignored variables file with mode `0600`. Supply optional `TF_VAR_runtime_secrets` as a JSON object of uppercase environment names to secret values. Never enable shell tracing while injecting secrets. Terraform's sensitive marking redacts normal CLI output but **does not encrypt state or saved plans**. Protect backend access, runner logs, environment variables, and local `.terraform/`; never publish a JSON plan or upload it as a CI artifact. + +## Bootstrap and release + +Run these commands from the repository root in an authorized environment. They change Azure resources; do not run them merely to validate this repository. + +```sh +export HUB_SUBSCRIPTION='your-subscription-id' +export HUB_IMAGE='your-registry/replica-hub@sha256:your-real-64-character-digest' +python3 infrastructure/azure/deploy.py bootstrap \ + --subscription "$HUB_SUBSCRIPTION" \ + --backend infrastructure/azure/backend.hcl \ + --variables infrastructure/azure/installation.tfvars \ + --image "$HUB_IMAGE" +``` + +Bootstrap creates infrastructure and the manual setup job with `deploy_hub=false`. No combined app exists yet. Review the configuration and a read-only Terraform plan before authorizing this command: the script prints its plans and applies them without an interactive prompt after checking for deletes/replacements. Bootstrap refuses any nonempty state so it cannot race a release on an existing installation. If initial provisioning failed partway, inspect the partial state and recover from a separately reviewed plan with `deploy_hub=false`; never use this recovery path after an app has been created. + +Use the same release command for the first installation, subsequent image releases, and infrastructure or secret changes: + +```sh +python3 infrastructure/azure/deploy.py release \ + --subscription "$HUB_SUBSCRIPTION" \ + --backend infrastructure/azure/backend.hcl \ + --variables infrastructure/azure/installation.tfvars \ + --image "$HUB_IMAGE" \ + --accept-maintenance +``` + +The operator must approve the maintenance window and inspect intended configuration changes before invoking the command. A read-only plan can be reviewed with `terraform -chdir=infrastructure/azure plan -var-file=installation.tfvars -var="image=$HUB_IMAGE" -var=deploy_hub=true` after initializing the configured backend. Do not apply that preflight plan: setup must run first. The script then: + +1. Checks the explicit subscription, Terraform state's installation, and resource group ownership. +2. Acquires an infinite lease on `releases/deployment.lock`. A competing release fails before stopping Hub. This serializes the whole release, not just individual Terraform operations; setup's database advisory lock remains a separate defense. +3. Checks that no setup execution is running, deactivates every active revision, and waits for **all** old revision replicas to disappear before setup. Users cannot use Hub during this window. +4. Uses a deliberate targeted Terraform plan/apply for the setup job and its infrastructure dependencies. It cannot update the app at this point. +5. Starts that job, checks that exact execution until it succeeds, and stops on failure or timeout. +6. Only after success, plans/applies the complete configuration with `deploy_hub=true` and the same image digest. It activates the latest revision, waits for Azure's ready-revision signal and an HTTPS `/api/ready` response, then releases the lease. + +Terraform owns the image in both resources. There is deliberately **no** `ignore_changes` or out-of-band `az containerapp update --image`: a later infrastructure apply cannot silently use a stale image. Provisioning and rollout are separated by the two phases above. Never run a full Terraform apply directly on an installed Hub, start a setup job manually, or change revisions in the portal outside this serialized procedure. Terraform state locking alone does not protect setup-to-rollout ordering. The app's `prevent_destroy` guard rejects accidental `deploy_hub=false` on later full plans. + +Plans reject resource deletion and replacement during normal releases. A database major upgrade, region move, rename, or replacement needs a reviewed migration/recovery plan, not an override of that guard. The release script deletes its sensitive saved plan on exit. It leaves Hub stopped and retains the infinite lease after any release failure; it never reactivates an older process automatically against a possibly migrated schema. + +### Recover a failed release + +Inspect the named setup execution and Container Apps/Log Analytics logs using authorized Azure access. Check that the release process has exited, every setup execution is terminal, and no old revision still has replicas. After a successful but partially deployed setup, decide whether to retry the same digest or use a compatible recovery image. Prisma migrations, queue migrations, and schedule registration are not a single atomic transaction. + +Only after those checks may an operator break the retained lease: + +```sh +az storage blob lease break --account-name YOUR_ACCOUNT \ + --container-name releases --blob-name deployment.lock \ + --auth-mode login --lease-break-period 0 --subscription "$HUB_SUBSCRIPTION" +``` + +Retry the release command. Do not break an active release's lease. A lease is an operational lock, not protection from administrators bypassing the procedure. If destruction already deleted the lock blob, complete the reviewed teardown rather than starting another release. + +Rollback is another release with a previously published compatible digest. It reruns setup, so confirm that the old image tolerates the current schema and schedules. There is no automatic database or queue downgrade. Retry and shutdown do not guarantee exactly-once jobs; business effects must tolerate at-least-once delivery. Replica disappearance is the available control-plane gate, not evidence that Azure has been proven free of all revision overlap; verify it in the target tenant before relying on worker exclusivity. + +## Hostname and TLS + +The Azure-generated app hostname already uses HTTPS. For a custom domain, first bootstrap and release using that hostname. Create the required DNS ownership TXT and CNAME/A records using the app's verification identifier and address reported by Azure. Upload or provision a certificate in **this** Container Apps environment through your authorized certificate process, then provide both `custom_hostname` and its existing `certificate_id` and run another release. Certificates, DNS zones, certificate renewals, and private keys are not created or owned by this Terraform configuration. + +Verify the custom hostname, HTTPS certificate, redirects/cookies/callback origin configuration, and readiness separately. The release's automated HTTPS check uses the Azure-generated hostname and does not prove custom-domain TLS or authenticated product flows. + +## Backups, restore, and migration + +PostgreSQL retains point-in-time backups for `backup_retention_days` (default 14). Blob/container soft deletion retains deleted objects for 14 days; versioning preserves earlier content, but versions incur storage charges and are not an independent backup. These defaults provide neither regional disaster recovery nor high availability. Managed PostgreSQL, the app's minimum replica, private networking, logs, backups, and Blob versions have ongoing costs even when traffic is low. + +Agree on acceptable data loss and recovery time with the installation owner. Schedule independent database exports and artifact copies to a separately authorized backup destination with its own retention. Run database exports from an authorized private-network host or job; PostgreSQL has no public firewall exception. Encrypt exports and never place credentials in commands, logs, or committed files. + +For recovery, stop releases and Hub, record the database recovery point and matching artifact manifest, and restore PostgreSQL to a **new** server using Azure point-in-time recovery. Restore Blob versions or independent copies to a separate private container/account. Verify database records, queue state, and artifact references together. A database restore alone cannot restore matching artifact state. + +Reconcile the restored resources with a reviewed Terraform configuration and import/state-migration plan before deployment; a restored server has different identifiers and this normal release script intentionally rejects replacements. Do not overwrite or delete the original resources until the restored installation has passed setup, integrity checks, HTTPS/readiness, representative authenticated operations, and a measured recovery exercise. Document the observed recovery time and data loss. + +To move an existing installation, pause writes and workers, export PostgreSQL, copy artifacts with an object manifest, import both into the new installation, and verify counts/checksums and a representative operation before switching traffic. Keep source backups and a rollback window. No automatic migration between local and Azure installations is performed here. + +## Guarded destruction + +Destruction removes database and artifact state. Take verified independent backups first. The default app lifecycle guard intentionally prevents a complete destroy: remove **only** the app's `prevent_destroy` block in a reviewed local change after the owner authorizes permanent deletion. Restore the guard afterward. For a bootstrap that never created the app, no guard edit is needed. + +```sh +python3 infrastructure/azure/deploy.py destroy \ + --subscription "$HUB_SUBSCRIPTION" \ + --backend infrastructure/azure/backend.hcl \ + --variables infrastructure/azure/installation.tfvars \ + --image "$HUB_IMAGE" \ + --accept-maintenance --confirm YOUR_EXACT_INSTALLATION_NAME +``` + +The script requires matching ownership tags and the exact installation name, acquires the release lock, stops Hub, checks setup executions, and applies a Terraform destroy plan. It deletes only Terraform-tracked resources; it does not issue a broad resource-group deletion command. AzureRM refuses to delete a resource group containing unmanaged resources. Review a read-only `terraform plan -destroy` with the same variables and `deploy_hub=true` before authorizing the command; the script itself does not pause for review. An existing registry, state backend, DNS, and externally supplied certificate are never independently targeted by this configuration. Azure may remove its own environment infrastructure group when the environment is deleted. + +Storage-account deletion does not preserve a usable Blob backup just because object soft deletion was enabled. Provider-retained PostgreSQL backups, soft-deleted logs, or other Azure retention may outlive resource deletion; verify the service-specific retention and billing in your tenant. The separate Terraform state backend and its historical sensitive versions remain until an administrator applies their own retention/deletion policy. + +## Credential-free checks and evidence limits + +From `infrastructure/azure/`: + +```sh +terraform init -backend=false -input=false -lockfile=readonly +terraform fmt -check -recursive +terraform validate +terraform test +python3 -m unittest discover -s . -p 'test_*.py' -v +``` + +Terraform tests use AzureRM mocks, not an Azure subscription. Orchestration tests replace command execution and HTTPS requests, proving ordering and failure gates rather than Azure behavior. Before production, an authorized tenant exercise must verify provisioning, private PostgreSQL connectivity and TLS, identity-based Blob/ACR access after RBAC propagation, failed setup blocking rollout, maintenance shutdown and revision behavior, HTTPS/readiness, persistence across replacement, custom TLS when configured, backup restoration, and owned-resource teardown. No real-cloud result is implied by these local checks. diff --git a/infrastructure/azure/.terraform.lock.hcl b/infrastructure/azure/.terraform.lock.hcl new file mode 100644 index 00000000..30c89942 --- /dev/null +++ b/infrastructure/azure/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.50.0" + constraints = "4.50.0" + hashes = [ + "h1:j2bM6QYxBw4dKDlNLR2lAWP6Dggv5lhyGU5VREOfaDw=", + "zh:11899f771a9f03c4e5d222d73ce842e930d26af8aaba88b99a2f0638265331dc", + "zh:2450a4258f40f6ba1f6705a10b2bdb6ff2067ab4c2a6938135391df03c974251", + "zh:2754e761370a18dab5748e020f595e08a7859522889e800204e7a5f65b026590", + "zh:6072e006b1c1b5c1e0a39d373bb6beebf5b4dbb619627c510b6a741521d226c6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8ddb7263f15bc16f3b85be8377c9e32ffa4db4cf809586d8010633698ff66105", + "zh:96f973bc54db728defca4817fbd9b1a412c538542e497536d1ce5f6be9946801", + "zh:aa519b7bad1e75930d84ec9ea8462ec384ed26a6ea172281cbd93244ecf8059b", + "zh:adbc238c95915d23d610990770f4b83e576311d4114b9662c4e967d0b0065b1b", + "zh:eaf94d50425ad88cdd42170e9737d8cb1a557ca3737cb905ce34e0218ec621f3", + "zh:f7aa0004e92e67fcee5d8725b878c8de7e6db6f0f766cbe2257d49a994b314f7", + "zh:fbb1195ea2784d67c690ed399b27d59d015c417dec21667f07a6aaf4099cd174", + ] +} diff --git a/infrastructure/azure/backend.hcl.example b/infrastructure/azure/backend.hcl.example new file mode 100644 index 00000000..3832c63f --- /dev/null +++ b/infrastructure/azure/backend.hcl.example @@ -0,0 +1,6 @@ +resource_group_name = "your-existing-terraform-state-group" +storage_account_name = "youruniquestateaccount" +container_name = "terraform" +key = "replica-hub.tfstate" +use_azuread_auth = true +use_oidc = true diff --git a/infrastructure/azure/compute.tf b/infrastructure/azure/compute.tf new file mode 100644 index 00000000..12cd33ca --- /dev/null +++ b/infrastructure/azure/compute.tf @@ -0,0 +1,197 @@ +resource "azurerm_container_app_job" "setup" { + name = "${var.name}-setup" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + container_app_environment_id = azurerm_container_app_environment.hub.id + workload_profile_name = "Consumption" + replica_timeout_in_seconds = 1800 + replica_retry_limit = 0 + tags = local.tags + + manual_trigger_config { + parallelism = 1 + replica_completion_count = 1 + } + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.hub.id] + } + + dynamic "registry" { + for_each = var.acr_id == null ? [] : [var.acr_id] + content { + server = split("/", var.image)[0] + identity = azurerm_user_assigned_identity.hub.id + } + } + + secret { + name = "database-url" + value = local.database_url + } + + dynamic "secret" { + for_each = local.secret_names + content { + name = "runtime-${substr(sha256(secret.value), 0, 12)}" + value = var.runtime_secrets[secret.value] + } + } + + template { + container { + name = "setup" + image = var.image + cpu = 0.5 + memory = "1Gi" + command = ["node", "dist/server/scripts/setup/index.js"] + + dynamic "env" { + for_each = merge(local.runtime_environment, { RUNTIME_MODE = "setup" }) + content { + name = env.key + value = env.value + } + } + + env { + name = "DATABASE_URL" + secret_name = "database-url" + } + + dynamic "env" { + for_each = local.secret_names + content { + name = env.value + secret_name = "runtime-${substr(sha256(env.value), 0, 12)}" + } + } + } + } + + depends_on = [ + azurerm_postgresql_flexible_server_database.hub, + azurerm_postgresql_flexible_server_configuration.tls, + azurerm_role_assignment.artifacts, + azurerm_role_assignment.image_pull, + ] +} + +resource "azurerm_container_app" "hub" { + count = var.deploy_hub ? 1 : 0 + name = "${var.name}-hub" + resource_group_name = azurerm_resource_group.hub.name + container_app_environment_id = azurerm_container_app_environment.hub.id + workload_profile_name = "Consumption" + revision_mode = "Multiple" + tags = local.tags + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.hub.id] + } + + dynamic "registry" { + for_each = var.acr_id == null ? [] : [var.acr_id] + content { + server = split("/", var.image)[0] + identity = azurerm_user_assigned_identity.hub.id + } + } + + secret { + name = "database-url" + value = local.database_url + } + + dynamic "secret" { + for_each = local.secret_names + content { + name = "runtime-${substr(sha256(secret.value), 0, 12)}" + value = var.runtime_secrets[secret.value] + } + } + + ingress { + external_enabled = true + allow_insecure_connections = false + target_port = 3001 + transport = "http" + traffic_weight { + latest_revision = true + percentage = 100 + } + } + + template { + min_replicas = 1 + max_replicas = 1 + termination_grace_period_seconds = 60 + + container { + name = "hub" + image = var.image + cpu = 0.5 + memory = "1Gi" + + dynamic "env" { + for_each = local.runtime_environment + content { + name = env.key + value = env.value + } + } + + env { + name = "DATABASE_URL" + secret_name = "database-url" + } + + dynamic "env" { + for_each = local.secret_names + content { + name = env.value + secret_name = "runtime-${substr(sha256(env.value), 0, 12)}" + } + } + + startup_probe { + transport = "HTTP" + port = 3001 + path = "/api/health" + interval_seconds = 5 + failure_count_threshold = 30 + } + readiness_probe { + transport = "HTTP" + port = 3001 + path = "/api/ready" + interval_seconds = 5 + success_count_threshold = 1 + timeout = 3 + } + liveness_probe { + transport = "HTTP" + port = 3001 + path = "/api/health" + interval_seconds = 10 + timeout = 3 + } + } + } + + lifecycle { + prevent_destroy = true + } + + depends_on = [azurerm_container_app_job.setup] +} + +resource "azurerm_container_app_custom_domain" "hub" { + count = var.deploy_hub && var.custom_hostname != null ? 1 : 0 + name = var.custom_hostname + container_app_id = azurerm_container_app.hub[0].id + container_app_environment_certificate_id = var.certificate_id + certificate_binding_type = "SniEnabled" +} diff --git a/infrastructure/azure/deploy.py b/infrastructure/azure/deploy.py new file mode 100644 index 00000000..e2f556be --- /dev/null +++ b/infrastructure/azure/deploy.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Provision once, then release Hub with an explicit maintenance window.""" + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parent +TERMINAL_EXECUTIONS = {"Succeeded", "Failed", "Stopped", "Canceled"} + + +def command(arguments, capture=False): + result = subprocess.run(arguments, check=True, text=True, capture_output=capture) + return result.stdout.strip() if capture else None + + +class Deployment: + def __init__(self, arguments): + self.args = arguments + self.installation = None + self.lease = None + self.plan_path = ROOT / ".terraform" / f"deployment-{uuid4().hex}.tfplan" + + def terraform(self, *arguments, capture=False): + return command(["terraform", f"-chdir={ROOT}", *arguments], capture=capture) + + def azure(self, *arguments): + return json.loads(command([ + "az", *arguments, "--subscription", self.args.subscription, + "--only-show-errors", "--output", "json", + ], capture=True) or "null") + + def app_arguments(self): + return ["--name", self.installation["app_name"], + "--resource-group", self.installation["resource_group"]] + + def job_arguments(self): + return ["--name", self.installation["setup_job_name"], + "--resource-group", self.installation["resource_group"]] + + def blob_arguments(self, lease=True): + return ["--account-name", self.installation["storage_account"], + "--container-name", self.installation["release_container"], + "--blob-name" if lease else "--name", self.installation["release_blob"], + "--auth-mode", "login"] + + def read_installation(self): + self.installation = json.loads(self.terraform("output", "-json", "installation", capture=True)) + if self.installation["subscription_id"] != self.args.subscription: + raise RuntimeError("The selected subscription does not match this Terraform state.") + + def verify_ownership(self): + group = self.azure("group", "show", "--name", self.installation["resource_group"]) + expected = { + "application": "replica-hub", "managed_by": "terraform", + "installation": self.installation["name"], "owner": self.installation["owner"], + } + if any(group.get("tags", {}).get(key) != value for key, value in expected.items()): + raise RuntimeError("Resource group ownership does not match this installation.") + + def acquire_lock(self): + self.lease = self.azure("storage", "blob", "lease", "acquire", *self.blob_arguments(), + "--lease-duration", "-1") + if not isinstance(self.lease, str) or not self.lease: + raise RuntimeError("Azure did not return a release lease ID.") + + def release_lock(self): + self.azure("storage", "blob", "lease", "release", *self.blob_arguments(), + "--lease-id", self.lease) + self.lease = None + + def plan(self, deploy_hub, target=None, destroy=False): + arguments = [ + "plan", "-input=false", "-lock-timeout=60s", + f"-var-file={self.args.variables}", f"-var=image={self.args.image}", + f"-var=deploy_hub={str(deploy_hub).lower()}", f"-out={self.plan_path}", + ] + if target: + arguments.append(f"-target={target}") + if destroy: + arguments.append("-destroy") + self.terraform(*arguments) + plan = json.loads(self.terraform("show", "-json", str(self.plan_path), capture=True)) + if not destroy: + changes = plan.get("resource_changes", []) + if any("delete" in change["change"]["actions"] for change in changes): + raise RuntimeError("Release plans cannot delete or replace resources. Review recovery separately.") + variables = plan["variables"] + if variables["subscription_id"]["value"] != self.args.subscription: + raise RuntimeError("The variable file selects a different subscription.") + if self.installation and variables["name"]["value"] != self.installation["name"]: + raise RuntimeError("The variable file selects a different installation.") + + def apply(self): + self.terraform("apply", "-input=false", "-lock-timeout=60s", str(self.plan_path)) + + def stop_hub(self): + if not self.installation["app_created"]: + return + revisions = self.azure("containerapp", "revision", "list", *self.app_arguments()) + # Inspect every revision: an inactive revision may still be terminating. + for revision in revisions: + if revision["properties"]["active"]: + self.azure("containerapp", "revision", "deactivate", *self.app_arguments(), + "--revision", revision["name"]) + deadline = time.monotonic() + 300 + for revision in revisions: + while self.azure("containerapp", "replica", "list", *self.app_arguments(), + "--revision", revision["name"]): + if time.monotonic() >= deadline: + raise RuntimeError("Old replicas have not stopped. Hub remains in maintenance.") + time.sleep(5) + + def ensure_no_setup(self): + executions = self.azure("containerapp", "job", "execution", "list", *self.job_arguments()) + if any(item["properties"]["status"] not in TERMINAL_EXECUTIONS for item in executions): + raise RuntimeError("A setup execution is still running; do not start another release.") + + def setup(self): + execution = self.azure("containerapp", "job", "start", *self.job_arguments()) + deadline = time.monotonic() + 1920 + while time.monotonic() < deadline: + result = self.azure("containerapp", "job", "execution", "show", + *self.job_arguments(), "--job-execution-name", execution["name"]) + status = result["properties"]["status"] + if status == "Succeeded": + return + if status in TERMINAL_EXECUTIONS: + raise RuntimeError("Setup failed. Hub remains stopped; inspect the setup execution.") + time.sleep(5) + raise RuntimeError("Setup timed out. Check or stop the execution before releasing the lock.") + + def wait_ready(self): + self.read_installation() + app = self.azure("containerapp", "show", *self.app_arguments()) + revision = app["properties"]["latestRevisionName"] + self.azure("containerapp", "revision", "activate", *self.app_arguments(), + "--revision", revision) + deadline = time.monotonic() + 300 + while time.monotonic() < deadline: + app = self.azure("containerapp", "show", *self.app_arguments()) + properties = app["properties"] + if properties.get("latestReadyRevisionName") == revision: + try: + with urllib.request.urlopen(self.installation["url"] + "/api/ready", timeout=10) as response: + if response.status == 200: + return + except (urllib.error.URLError, TimeoutError): + pass + time.sleep(5) + raise RuntimeError("Hub did not become ready. Keep the release lock and investigate.") + + def run(self): + self.terraform("init", "-input=false", f"-backend-config={self.args.backend}") + account = self.azure("account", "show") + if account["id"] != self.args.subscription: + raise RuntimeError("Azure returned a different subscription.") + if self.args.operation == "bootstrap": + resources = self.terraform("state", "list", capture=True).splitlines() + if resources: + raise RuntimeError("Infrastructure already exists. Use release; review partial bootstrap recovery separately.") + self.plan(False) + self.apply() + print("Infrastructure is ready. Hub is not running. Run release to apply setup first.") + return + + self.read_installation() + self.verify_ownership() + if self.args.operation == "destroy" and self.args.confirm != self.installation["name"]: + raise RuntimeError("Destroy requires --confirm with the exact installation name.") + self.acquire_lock() + self.read_installation() + self.verify_ownership() + self.ensure_no_setup() + self.stop_hub() + if self.args.operation == "destroy": + self.plan(self.installation["app_created"], destroy=True) + # Deleting our leased lock blob prevents new releases while Terraform destroys its container. + self.azure("storage", "blob", "delete", *self.blob_arguments(lease=False), "--lease-id", self.lease) + self.lease = None + self.apply() + print("Terraform-tracked installation resources were destroyed.") + return + + # The target deliberately excludes the app until the complete setup sequence succeeds. + self.plan(self.installation["app_created"], target="azurerm_container_app_job.setup") + self.apply() + self.setup() + self.plan(True) + self.apply() + self.wait_ready() + self.release_lock() + print("Hub is ready. The maintenance window is complete.") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=["bootstrap", "release", "destroy"]) + parser.add_argument("--subscription", required=True) + parser.add_argument("--backend", type=lambda value: str(Path(value).resolve()), required=True) + parser.add_argument("--variables", type=lambda value: str(Path(value).resolve()), required=True) + parser.add_argument("--image", required=True, help="Existing registry/repository@sha256:digest") + parser.add_argument("--accept-maintenance", action="store_true") + parser.add_argument("--confirm", help="Exact installation name; required for destruction") + arguments = parser.parse_args(argv) + if not re.fullmatch(r"[a-z0-9][a-z0-9.:/-]+@sha256:[a-f0-9]{64}", arguments.image): + parser.error("--image must name an immutable SHA-256 digest.") + if arguments.operation != "bootstrap" and not arguments.accept_maintenance: + parser.error("Releases and destruction stop Hub. Supply --accept-maintenance.") + os.umask(0o077) + deployment = Deployment(arguments) + try: + deployment.run() + except (RuntimeError, subprocess.CalledProcessError, KeyboardInterrupt) as error: + print(f"Deployment stopped: {error}", file=sys.stderr) + if deployment.lease: + print("The infinite release lease is retained. Verify all processes and setup executions " + "have stopped before an operator breaks the lease.", file=sys.stderr) + return 1 + finally: + deployment.plan_path.unlink(missing_ok=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infrastructure/azure/installation.tfvars.example b/infrastructure/azure/installation.tfvars.example new file mode 100644 index 00000000..1a6034d0 --- /dev/null +++ b/infrastructure/azure/installation.tfvars.example @@ -0,0 +1,11 @@ +subscription_id = "00000000-0000-0000-0000-000000000000" +name = "your-unique-hub" +location = "eastus2" +owner = "your-team" +image = "ghcr.io/your-org/replica-hub@sha256:REPLACE_WITH_REAL_DIGEST" + +# Supply postgres_admin_password and runtime_secrets through your secret store, +# TF_VAR_ environment variables, or an ignored, access-restricted tfvars file. +# acr_id = "/subscriptions/.../resourceGroups/.../providers/Microsoft.ContainerRegistry/registries/..." +# custom_hostname = "hub.example.com" +# certificate_id = "/subscriptions/.../resourceGroups/.../providers/Microsoft.App/managedEnvironments/.../certificates/..." diff --git a/infrastructure/azure/main.tf b/infrastructure/azure/main.tf new file mode 100644 index 00000000..fb48b891 --- /dev/null +++ b/infrastructure/azure/main.tf @@ -0,0 +1,199 @@ +locals { + tags = { + application = "replica-hub" + installation = var.name + owner = var.owner + managed_by = "terraform" + } + database_url = "postgresql://replicaadmin:${replace(urlencode(var.postgres_admin_password), "+", "%20")}@${azurerm_postgresql_flexible_server.database.fqdn}:5432/replica?sslmode=verify-full" + runtime_environment = { + NODE_ENV = "production" + RUNTIME_MODE = "combined" + HOST = "0.0.0.0" + PORT = "3001" + CLIENT_DIRECTORY = "/app/dist/client" + SHUTDOWN_TIMEOUT_MS = "30000" + ARTIFACT_STORAGE_PROVIDER = "azure" + AZURE_STORAGE_ACCOUNT = azurerm_storage_account.artifacts.name + AZURE_STORAGE_CONTAINER = azurerm_storage_container.artifacts.name + AZURE_CLIENT_ID = azurerm_user_assigned_identity.hub.client_id + } + secret_names = nonsensitive(toset(keys(var.runtime_secrets))) +} + +resource "azurerm_resource_group" "hub" { + name = "${var.name}-hub" + location = var.location + tags = local.tags +} + +resource "azurerm_virtual_network" "hub" { + name = "${var.name}-network" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + address_space = ["10.40.0.0/16"] + tags = local.tags +} + +resource "azurerm_subnet" "apps" { + name = "container-apps" + resource_group_name = azurerm_resource_group.hub.name + virtual_network_name = azurerm_virtual_network.hub.name + address_prefixes = ["10.40.0.0/23"] + delegation { + name = "container-apps" + service_delegation { + name = "Microsoft.App/environments" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_subnet" "database" { + name = "postgresql" + resource_group_name = azurerm_resource_group.hub.name + virtual_network_name = azurerm_virtual_network.hub.name + address_prefixes = ["10.40.2.0/24"] + delegation { + name = "postgresql" + service_delegation { + name = "Microsoft.DBforPostgreSQL/flexibleServers" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_private_dns_zone" "database" { + name = "${var.name}.postgres.database.azure.com" + resource_group_name = azurerm_resource_group.hub.name + tags = local.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "database" { + name = "hub" + resource_group_name = azurerm_resource_group.hub.name + private_dns_zone_name = azurerm_private_dns_zone.database.name + virtual_network_id = azurerm_virtual_network.hub.id + tags = local.tags +} + +resource "azurerm_postgresql_flexible_server" "database" { + name = "${var.name}-postgres" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + version = var.postgres_version + administrator_login = "replicaadmin" + administrator_password = var.postgres_admin_password + sku_name = var.postgres_sku + storage_mb = 32768 + backup_retention_days = var.backup_retention_days + geo_redundant_backup_enabled = false + public_network_access_enabled = false + delegated_subnet_id = azurerm_subnet.database.id + private_dns_zone_id = azurerm_private_dns_zone.database.id + tags = local.tags + depends_on = [azurerm_private_dns_zone_virtual_network_link.database] +} + +resource "azurerm_postgresql_flexible_server_configuration" "tls" { + name = "require_secure_transport" + server_id = azurerm_postgresql_flexible_server.database.id + value = "on" +} + +resource "azurerm_postgresql_flexible_server_database" "hub" { + name = "replica" + server_id = azurerm_postgresql_flexible_server.database.id + charset = "UTF8" + collation = "en_US.utf8" +} + +resource "azurerm_storage_account" "artifacts" { + name = "${replace(var.name, "-", "")}hub" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + account_tier = "Standard" + account_replication_type = "LRS" + min_tls_version = "TLS1_2" + https_traffic_only_enabled = true + allow_nested_items_to_be_public = false + shared_access_key_enabled = false + default_to_oauth_authentication = true + tags = local.tags + + blob_properties { + versioning_enabled = true + delete_retention_policy { + days = 14 + } + container_delete_retention_policy { + days = 14 + } + } +} + +resource "azurerm_storage_container" "artifacts" { + name = "artifacts" + storage_account_id = azurerm_storage_account.artifacts.id + container_access_type = "private" +} + +resource "azurerm_storage_container" "releases" { + name = "releases" + storage_account_id = azurerm_storage_account.artifacts.id + container_access_type = "private" +} + +resource "azurerm_storage_blob" "release_lock" { + name = "deployment.lock" + storage_account_name = azurerm_storage_account.artifacts.name + storage_container_name = azurerm_storage_container.releases.name + type = "Block" + source_content = "Replica Hub release lock. Do not delete while a release is running.\n" +} + +resource "azurerm_user_assigned_identity" "hub" { + name = "${var.name}-hub" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + tags = local.tags +} + +resource "azurerm_role_assignment" "artifacts" { + scope = "${azurerm_storage_account.artifacts.id}/blobServices/default/containers/${azurerm_storage_container.artifacts.name}" + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_role_assignment" "image_pull" { + count = var.acr_id == null ? 0 : 1 + scope = var.acr_id + role_definition_name = "AcrPull" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_log_analytics_workspace" "hub" { + name = "${var.name}-logs" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + sku = "PerGB2018" + retention_in_days = 30 + tags = local.tags +} + +resource "azurerm_container_app_environment" "hub" { + name = "${var.name}-environment" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + log_analytics_workspace_id = azurerm_log_analytics_workspace.hub.id + logs_destination = "log-analytics" + infrastructure_subnet_id = azurerm_subnet.apps.id + tags = local.tags + + workload_profile { + name = "Consumption" + workload_profile_type = "Consumption" + minimum_count = 0 + maximum_count = 0 + } +} diff --git a/infrastructure/azure/outputs.tf b/infrastructure/azure/outputs.tf new file mode 100644 index 00000000..a84a3dd1 --- /dev/null +++ b/infrastructure/azure/outputs.tf @@ -0,0 +1,21 @@ +output "installation" { + description = "Non-secret release coordinates; no credentials are emitted." + value = { + name = var.name + owner = var.owner + subscription_id = var.subscription_id + resource_group = azurerm_resource_group.hub.name + app_name = "${var.name}-hub" + setup_job_name = azurerm_container_app_job.setup.name + storage_account = azurerm_storage_account.artifacts.name + release_container = azurerm_storage_container.releases.name + release_blob = azurerm_storage_blob.release_lock.name + environment_id = azurerm_container_app_environment.hub.id + postgres_server_id = azurerm_postgresql_flexible_server.database.id + postgres_host = azurerm_postgresql_flexible_server.database.fqdn + managed_identity_id = azurerm_user_assigned_identity.hub.id + managed_identity_client_id = azurerm_user_assigned_identity.hub.client_id + app_created = var.deploy_hub + url = var.deploy_hub ? "https://${azurerm_container_app.hub[0].ingress[0].fqdn}" : null + } +} diff --git a/infrastructure/azure/test_deploy.py b/infrastructure/azure/test_deploy.py new file mode 100644 index 00000000..a5864c0c --- /dev/null +++ b/infrastructure/azure/test_deploy.py @@ -0,0 +1,143 @@ +import contextlib +import io +import json +import subprocess +import unittest +from unittest.mock import MagicMock, patch + +import deploy + + +IMAGE = "example.azurecr.io/hub@sha256:" + "a" * 64 + + +class AzureScenario: + def __init__(self, app_created=True, setup_status="Succeeded", lock_available=True): + self.installation = { + "name": "test-hub", "owner": "test-team", "subscription_id": "test-subscription", + "resource_group": "test-hub", "app_name": "test-hub", "setup_job_name": "test-setup", + "storage_account": "testhub", "release_container": "releases", + "release_blob": "deployment.lock", "app_created": app_created, + "url": "https://hub.example.com", + } + self.calls = [] + self.setup_status = setup_status + self.lock_available = lock_available + self.targeted = False + self.replicas_read = 0 + + def command(self, arguments, capture=False): + self.calls.append(arguments) + if arguments[0] == "terraform": + operation = arguments[2] + if operation == "output": + return json.dumps(self.installation) + if operation == "state": + return "azurerm_container_app.hub[0]" if self.installation["app_created"] else "" + if operation == "plan": + self.targeted = any(item.startswith("-target=") for item in arguments) + if operation == "show": + return json.dumps({ + "variables": { + "name": {"value": self.installation["name"]}, + "subscription_id": {"value": self.installation["subscription_id"]}, + }, + "resource_changes": [], + }) + if operation == "apply" and not self.targeted: + self.installation["app_created"] = True + return "" + operation = arguments[1:4] + if operation[:2] == ["account", "show"]: + value = {"id": self.installation["subscription_id"]} + elif operation[:2] == ["group", "show"]: + value = {"tags": { + "application": "replica-hub", "managed_by": "terraform", + "installation": self.installation["name"], "owner": self.installation["owner"], + }} + elif operation == ["storage", "blob", "lease"]: + if "acquire" in arguments: + if not self.lock_available: + raise subprocess.CalledProcessError(1, arguments) + value = "lease-id" + else: + value = None + elif operation == ["containerapp", "revision", "list"]: + value = [{"name": "old-revision", "properties": {"active": True}}] + elif operation == ["containerapp", "replica", "list"]: + self.replicas_read += 1 + value = [{"name": "old-replica"}] if self.replicas_read == 1 else [] + elif operation == ["containerapp", "job", "execution"]: + value = [] if "list" in arguments else {"properties": {"status": self.setup_status}} + elif operation == ["containerapp", "job", "start"]: + value = {"name": "setup-execution"} + elif operation[:2] == ["containerapp", "show"]: + value = {"properties": {"latestRevisionName": "new", "latestReadyRevisionName": "new"}} + elif operation[:2] == ["containerapp", "revision"]: + value = None + else: + raise AssertionError(f"Unexpected Azure command: {arguments}") + return json.dumps(value) + + def run(self, operation="release"): + response = MagicMock() + response.__enter__.return_value.status = 200 + with patch.object(deploy, "command", side_effect=self.command), \ + patch.object(deploy.time, "sleep"), \ + patch.object(deploy.urllib.request, "urlopen", return_value=response), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + return deploy.main([ + operation, "--subscription", "test-subscription", + "--backend", "backend.hcl", "--variables", "installation.tfvars", + "--image", IMAGE, "--accept-maintenance", + ]) + + +class DeploymentTests(unittest.TestCase): + def test_setup_failure_leaves_hub_stopped_and_retains_release_lock(self): + scenario = AzureScenario(setup_status="Failed") + self.assertEqual(scenario.run(), 1) + self.assertTrue(any("deactivate" in call for call in scenario.calls)) + plans = [call for call in scenario.calls if call[0] == "terraform" and call[2] == "plan"] + self.assertEqual(len(plans), 1) + self.assertIn("-target=azurerm_container_app_job.setup", plans[0]) + self.assertFalse(any("activate" in call for call in scenario.calls)) + self.assertFalse(any("release" in call for call in scenario.calls)) + + def test_release_waits_for_old_replicas_and_setup_before_rollout(self): + scenario = AzureScenario() + self.assertEqual(scenario.run(), 0) + events = scenario.calls + setup = next(index for index, call in enumerate(events) if call[1:4] == ["containerapp", "job", "start"]) + drained = [index for index, call in enumerate(events) if call[1:4] == ["containerapp", "replica", "list"]] + rollout = next(index for index, call in enumerate(events) + if call[0] == "terraform" and call[2] == "plan" and "-var=deploy_hub=true" in call + and not any(item.startswith("-target=") for item in call)) + self.assertEqual(len(drained), 2) + self.assertLess(max(drained), setup) + self.assertLess(setup, rollout) + self.assertTrue(any("release" in call for call in events)) + + def test_initial_release_runs_setup_before_creating_the_app(self): + scenario = AzureScenario(app_created=False) + self.assertEqual(scenario.run(), 0) + self.assertFalse(any("deactivate" in call for call in scenario.calls)) + plans = [call for call in scenario.calls if call[0] == "terraform" and call[2] == "plan"] + self.assertIn("-var=deploy_hub=false", plans[0]) + self.assertIn("-var=deploy_hub=true", plans[1]) + + def test_competing_release_cannot_stop_hub_or_run_setup(self): + scenario = AzureScenario(lock_available=False) + self.assertEqual(scenario.run(), 1) + self.assertFalse(any("deactivate" in call for call in scenario.calls)) + self.assertFalse(any(call[1:4] == ["containerapp", "job", "start"] for call in scenario.calls)) + + def test_bootstrap_refuses_an_existing_hub(self): + scenario = AzureScenario() + self.assertEqual(scenario.run("bootstrap"), 1) + self.assertFalse(any(call[0] == "terraform" and call[2] == "apply" for call in scenario.calls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/infrastructure/azure/tests/deployment.tftest.hcl b/infrastructure/azure/tests/deployment.tftest.hcl new file mode 100644 index 00000000..b3d2c5ab --- /dev/null +++ b/infrastructure/azure/tests/deployment.tftest.hcl @@ -0,0 +1,147 @@ +mock_provider "azurerm" { + override_during = plan + mock_resource "azurerm_virtual_network" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Network/virtualNetworks/test" + } + } + mock_resource "azurerm_subnet" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Network/virtualNetworks/test/subnets/test" + } + } + mock_resource "azurerm_private_dns_zone" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Network/privateDnsZones/test.postgres.database.azure.com" + } + } + mock_resource "azurerm_postgresql_flexible_server" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.DBforPostgreSQL/flexibleServers/test" + fqdn = "test.postgres.database.azure.com" + } + } + mock_resource "azurerm_storage_account" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Storage/storageAccounts/testhub" + } + } + mock_resource "azurerm_user_assigned_identity" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test" + client_id = "00000000-0000-0000-0000-000000000001" + principal_id = "00000000-0000-0000-0000-000000000002" + } + } + mock_resource "azurerm_log_analytics_workspace" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.OperationalInsights/workspaces/test" + } + } + mock_resource "azurerm_container_app_environment" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.App/managedEnvironments/test" + } + } +} + +variables { + subscription_id = "00000000-0000-0000-0000-000000000000" + name = "test-hub" + owner = "test-team" + location = "eastus2" + image = "example.azurecr.io/hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + postgres_admin_password = "NotARealPassword-12345" +} + +run "bootstrap_keeps_hub_stopped" { + command = plan + + assert { + condition = length(azurerm_container_app.hub) == 0 + error_message = "Bootstrap must not create a combined process before setup." + } + assert { + condition = ( + azurerm_container_app_job.setup.template[0].container[0].command == tolist(["node", "dist/server/scripts/setup/index.js"]) && + azurerm_container_app_job.setup.manual_trigger_config[0].parallelism == 1 && + azurerm_container_app_job.setup.replica_retry_limit == 0 + ) + error_message = "Setup must run explicitly, once per execution, without automatic retries." + } +} + +run "durable_state_requires_private_authenticated_access" { + command = plan + + assert { + condition = ( + !azurerm_postgresql_flexible_server.database.public_network_access_enabled && + azurerm_postgresql_flexible_server_configuration.tls.value == "on" && + endswith(nonsensitive(local.database_url), "?sslmode=verify-full") && + azurerm_postgresql_flexible_server.database.backup_retention_days >= 7 + ) + error_message = "PostgreSQL must be private, TLS-only, and retain recoverable backups." + } + assert { + condition = ( + !azurerm_storage_account.artifacts.allow_nested_items_to_be_public && + !azurerm_storage_account.artifacts.shared_access_key_enabled && + azurerm_storage_container.artifacts.container_access_type == "private" && + azurerm_storage_account.artifacts.blob_properties[0].versioning_enabled && + endswith(azurerm_role_assignment.artifacts.scope, "/blobServices/default/containers/artifacts") && + azurerm_role_assignment.artifacts.role_definition_name == "Storage Blob Data Contributor" && + length(azurerm_role_assignment.image_pull) == 0 + ) + error_message = "The workload needs container-scoped Blob access, not keys or registry permissions by default." + } +} + +run "combined_release_uses_one_image_and_bounded_shutdown" { + command = plan + + variables { + deploy_hub = true + acr_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/existing/providers/Microsoft.ContainerRegistry/registries/existing" + } + + assert { + condition = ( + azurerm_container_app.hub[0].template[0].min_replicas == 1 && + azurerm_container_app.hub[0].template[0].max_replicas == 1 && + azurerm_container_app.hub[0].revision_mode == "Multiple" && + azurerm_container_app.hub[0].template[0].termination_grace_period_seconds >= 35 && + azurerm_container_app.hub[0].template[0].termination_grace_period_seconds <= 60 && + azurerm_container_app.hub[0].template[0].container[0].image == azurerm_container_app_job.setup.template[0].container[0].image && + azurerm_role_assignment.image_pull[0].scope == var.acr_id + ) + error_message = "Combined Hub and setup must use the same digest, with bounded shutdown and explicit ACR access." + } + assert { + condition = ( + azurerm_container_app.hub[0].ingress[0].external_enabled && + !azurerm_container_app.hub[0].ingress[0].allow_insecure_connections && + one(azurerm_container_app.hub[0].template[0].container[0].readiness_probe).path == "/api/ready" && + one(azurerm_container_app.hub[0].template[0].container[0].liveness_probe).path == "/api/health" && + local.runtime_environment.RUNTIME_MODE == "combined" && + local.runtime_environment.AZURE_STORAGE_CONTAINER == "artifacts" + ) + error_message = "HTTPS, dependency readiness, process liveness, and runtime configuration must match Hub." + } +} + +run "mutable_image_is_rejected" { + command = plan + variables { + image = "example.azurecr.io/hub:latest" + } + expect_failures = [var.image] +} + +run "hostname_without_certificate_is_rejected" { + command = plan + variables { + custom_hostname = "hub.example.com" + } + expect_failures = [var.certificate_id] +} diff --git a/infrastructure/azure/variables.tf b/infrastructure/azure/variables.tf new file mode 100644 index 00000000..1013470b --- /dev/null +++ b/infrastructure/azure/variables.tf @@ -0,0 +1,118 @@ +variable "subscription_id" { + type = string + description = "Explicit Azure subscription for this installation." +} + +variable "name" { + type = string + description = "Globally unique installation prefix, also used to confirm ownership." + + validation { + condition = can(regex("^[a-z][a-z0-9-]{2,18}[a-z0-9]$", var.name)) && !strcontains(var.name, "--") + error_message = "Use 4–20 lowercase letters, digits, or single hyphens, beginning with a letter." + } +} + +variable "location" { + type = string + description = "Azure region supporting Container Apps and PostgreSQL Flexible Server." +} + +variable "owner" { + type = string + description = "Team responsible for cost, releases, and recovery." + + validation { + condition = length(trimspace(var.owner)) > 0 + error_message = "An installation owner is required." + } +} + +variable "image" { + type = string + description = "Existing public or ACR image pinned to an immutable SHA-256 digest." + + validation { + condition = can(regex("^[a-z0-9][a-z0-9.:/-]+@sha256:[a-f0-9]{64}$", var.image)) + error_message = "Supply a registry/repository@sha256:<64 lowercase hex characters>, not a tag." + } +} + +variable "deploy_hub" { + type = bool + default = false + description = "Keep false for bootstrap. Only deploy.py enables Hub after setup succeeds." +} + +variable "acr_id" { + type = string + default = null + description = "Optional existing ACR resource ID. No registry is created or deleted." +} + +variable "postgres_admin_password" { + type = string + sensitive = true + description = "Explicit PostgreSQL administrator password. Stored in protected Terraform state." + + validation { + condition = length(var.postgres_admin_password) >= 16 + error_message = "Use an independently generated PostgreSQL password of at least 16 characters." + } +} + +variable "postgres_sku" { + type = string + default = "B_Standard_B1ms" + description = "PostgreSQL Flexible Server SKU; assess capacity before production use." +} + +variable "postgres_version" { + type = string + default = "17" + description = "PostgreSQL major version supported by Hub and the selected region." +} + +variable "backup_retention_days" { + type = number + default = 14 + description = "PostgreSQL point-in-time recovery retention." + + validation { + condition = var.backup_retention_days >= 7 && var.backup_retention_days <= 35 + error_message = "PostgreSQL retention must be between 7 and 35 days." + } +} + +variable "runtime_secrets" { + type = map(string) + sensitive = true + default = {} + description = "Explicit additional runtime secret values keyed by uppercase environment name." + + validation { + condition = alltrue([ + for name in keys(var.runtime_secrets) : + can(regex("^[A-Z][A-Z0-9_]{0,40}$", name)) && + !contains(["DATABASE_URL", "NODE_ENV", "RUNTIME_MODE", "HOST", "PORT", "CLIENT_DIRECTORY", "SHUTDOWN_TIMEOUT_MS", "ARTIFACT_STORAGE_PROVIDER", "ARTIFACT_STORAGE_DIRECTORY", "AZURE_STORAGE_ACCOUNT", "AZURE_STORAGE_CONTAINER", "AZURE_CLIENT_ID"], name) + ]) + error_message = "Use uppercase environment names of at most 41 characters; deployment-owned settings cannot be overridden." + } +} + +variable "custom_hostname" { + type = string + default = null + description = "Optional DNS hostname already verified for this app." +} + +variable "certificate_id" { + type = string + default = null + description = "Existing Container Apps environment certificate ID for custom_hostname." + + validation { + condition = (var.custom_hostname == null) == (var.certificate_id == null) + error_message = "Supply both custom_hostname and certificate_id, or neither." + } +} diff --git a/infrastructure/azure/versions.tf b/infrastructure/azure/versions.tf new file mode 100644 index 00000000..a93abfa8 --- /dev/null +++ b/infrastructure/azure/versions.tf @@ -0,0 +1,24 @@ +terraform { + required_version = "~> 1.13.0" + + backend "azurerm" {} + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "= 4.50.0" + } + } +} + +provider "azurerm" { + subscription_id = var.subscription_id + storage_use_azuread = true + resource_provider_registrations = "none" + + features { + resource_group { + prevent_deletion_if_contains_resources = true + } + } +} From 06340dfc457a102fa9dd3d593978df3c82ae4b98 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:16:53 +0000 Subject: [PATCH 2/4] refactor(infra): use TypeScript for Hub deployment Replace Python release orchestration and tests with native Node.js TypeScript. Keep Terraform's native tests and existing release safety gates. Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/hub-infrastructure.yml | 5 +- infrastructure/.gitignore | 1 - infrastructure/README.md | 12 +- infrastructure/azure/deploy-hub/index.test.ts | 234 +++++++++++ infrastructure/azure/deploy-hub/index.ts | 365 ++++++++++++++++++ infrastructure/azure/deploy-hub/types.ts | 32 ++ infrastructure/azure/deploy.py | 235 ----------- infrastructure/azure/test_deploy.py | 143 ------- infrastructure/azure/variables.tf | 2 +- scripts/tsconfig.json | 2 +- 10 files changed, 644 insertions(+), 387 deletions(-) create mode 100644 infrastructure/azure/deploy-hub/index.test.ts create mode 100644 infrastructure/azure/deploy-hub/index.ts create mode 100644 infrastructure/azure/deploy-hub/types.ts delete mode 100644 infrastructure/azure/deploy.py delete mode 100644 infrastructure/azure/test_deploy.py diff --git a/.github/workflows/hub-infrastructure.yml b/.github/workflows/hub-infrastructure.yml index bca57411..b827faff 100644 --- a/.github/workflows/hub-infrastructure.yml +++ b/.github/workflows/hub-infrastructure.yml @@ -34,4 +34,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - run: python3 -m unittest discover -s . -p 'test_*.py' -v + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + - run: node --test deploy-hub/index.test.ts diff --git a/infrastructure/.gitignore b/infrastructure/.gitignore index 153bcaf3..5e77b516 100644 --- a/infrastructure/.gitignore +++ b/infrastructure/.gitignore @@ -1,5 +1,4 @@ .tools/ -**/__pycache__/ **/.terraform/ **/*.tfstate* **/*.tfplan diff --git a/infrastructure/README.md b/infrastructure/README.md index 349222e2..b8555f99 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -24,7 +24,7 @@ The reference uses the PostgreSQL administrator login for both setup and runtime ## Prepare access and protected state -Use the Terraform version constrained in [`versions.tf`](azure/versions.tf), Python 3, and a current Azure CLI with Container Apps job and revision commands. CI performs formatting, validation, mocked Terraform tests, and release-orchestration tests without cloud credentials. It does not deploy. +Use the Terraform version constrained in [`versions.tf`](azure/versions.tf), a Node.js version supported by `scripts/package.json`, and a current Azure CLI with Container Apps job and revision commands. CI performs formatting, validation, mocked Terraform tests, and release-orchestration tests without cloud credentials. It does not deploy. An authorized administrator must first create the **separate** Terraform backend: a dedicated state resource group, Storage account, and private container. Enable TLS, Entra authorization, Blob versioning and soft deletion; restrict network access to deployment runners and grant only the deployment identity `Storage Blob Data Contributor` on the state container. These bootstrap resources are not part of this installation's state or teardown. Never store this state in the app's artifacts account. @@ -47,7 +47,7 @@ Run these commands from the repository root in an authorized environment. They c ```sh export HUB_SUBSCRIPTION='your-subscription-id' export HUB_IMAGE='your-registry/replica-hub@sha256:your-real-64-character-digest' -python3 infrastructure/azure/deploy.py bootstrap \ +node infrastructure/azure/deploy-hub/index.ts bootstrap \ --subscription "$HUB_SUBSCRIPTION" \ --backend infrastructure/azure/backend.hcl \ --variables infrastructure/azure/installation.tfvars \ @@ -59,7 +59,7 @@ Bootstrap creates infrastructure and the manual setup job with `deploy_hub=false Use the same release command for the first installation, subsequent image releases, and infrastructure or secret changes: ```sh -python3 infrastructure/azure/deploy.py release \ +node infrastructure/azure/deploy-hub/index.ts release \ --subscription "$HUB_SUBSCRIPTION" \ --backend infrastructure/azure/backend.hcl \ --variables infrastructure/azure/installation.tfvars \ @@ -119,7 +119,7 @@ To move an existing installation, pause writes and workers, export PostgreSQL, c Destruction removes database and artifact state. Take verified independent backups first. The default app lifecycle guard intentionally prevents a complete destroy: remove **only** the app's `prevent_destroy` block in a reviewed local change after the owner authorizes permanent deletion. Restore the guard afterward. For a bootstrap that never created the app, no guard edit is needed. ```sh -python3 infrastructure/azure/deploy.py destroy \ +node infrastructure/azure/deploy-hub/index.ts destroy \ --subscription "$HUB_SUBSCRIPTION" \ --backend infrastructure/azure/backend.hcl \ --variables infrastructure/azure/installation.tfvars \ @@ -140,7 +140,9 @@ terraform init -backend=false -input=false -lockfile=readonly terraform fmt -check -recursive terraform validate terraform test -python3 -m unittest discover -s . -p 'test_*.py' -v +node --test deploy-hub/index.test.ts ``` Terraform tests use AzureRM mocks, not an Azure subscription. Orchestration tests replace command execution and HTTPS requests, proving ordering and failure gates rather than Azure behavior. Before production, an authorized tenant exercise must verify provisioning, private PostgreSQL connectivity and TLS, identity-based Blob/ACR access after RBAC propagation, failed setup blocking rollout, maintenance shutdown and revision behavior, HTTPS/readiness, persistence across replacement, custom TLS when configured, backup restoration, and owned-resource teardown. No real-cloud result is implied by these local checks. + +The release command and its tests are TypeScript running directly in Node.js. Terraform's own tests use `.tftest.hcl`; neither requires Python. From the repository root, `npm run typecheck --prefix scripts` checks the TypeScript operational scripts as well. diff --git a/infrastructure/azure/deploy-hub/index.test.ts b/infrastructure/azure/deploy-hub/index.test.ts new file mode 100644 index 00000000..c3194989 --- /dev/null +++ b/infrastructure/azure/deploy-hub/index.test.ts @@ -0,0 +1,234 @@ +import assert from "node:assert/strict" +import childProcess from "node:child_process" +import { afterEach, describe, it, mock } from "node:test" +import timers from "node:timers/promises" + +import { deployHub } from "./index.ts" + +afterEach(() => { + mock.restoreAll() +}) + +function installationScenario({ + appCreated = true, + setupStatus = "Succeeded", + lockAvailable = true, +} = {}) { + const installation = { + name: "test-hub", + owner: "test-team", + subscription_id: "test-subscription", + resource_group: "test-hub", + app_name: "test-hub", + setup_job_name: "test-setup", + storage_account: "testhub", + release_container: "releases", + release_blob: "deployment.lock", + app_created: appCreated, + url: "https://hub.example.com", + } + const events: string[][] = [] + const replicaReads = new Map() + let targeted = false + mock.method(childProcess, "execFileSync", (file: string, args: readonly string[]) => { + const command: string[] = [file, ...args] + events.push(command) + if (file === "terraform") { + switch (args[1]) { + case "output": + return JSON.stringify(installation) + case "state": + return appCreated ? "azurerm_container_app.hub[0]" : "" + case "plan": + targeted = args.some((argument: string) => argument.startsWith("-target=")) + return "" + case "show": + return JSON.stringify({ + variables: { + name: { value: installation.name }, + subscription_id: { value: installation.subscription_id }, + }, + resource_changes: [], + }) + case "apply": + if (!targeted) installation.app_created = true + return "" + case "init": + return "" + default: + throw new Error(`Unexpected Terraform command: ${command.join(" ")}`) + } + } + if (file !== "az") throw new Error(`Unexpected executable: ${file}`) + const operation = args.slice(0, 3).join(" ") + if (args.slice(0, 2).join(" ") === "account show") { + return JSON.stringify({ id: installation.subscription_id }) + } + if (args.slice(0, 2).join(" ") === "group show") { + return JSON.stringify({ + tags: { + application: "replica-hub", + managed_by: "terraform", + installation: installation.name, + owner: installation.owner, + }, + }) + } + switch (operation) { + case "storage blob lease": + if (args.includes("acquire")) { + if (!lockAvailable) throw new Error("Release lease is already held") + return JSON.stringify("lease-id") + } + return "null" + case "containerapp revision list": + return JSON.stringify([ + { name: "old-active", properties: { active: true } }, + { name: "old-terminating", properties: { active: false } }, + ]) + case "containerapp replica list": { + const revision = args[args.indexOf("--revision") + 1] + const reads = (replicaReads.get(revision) ?? 0) + 1 + replicaReads.set(revision, reads) + return JSON.stringify(reads === 1 ? [{ name: `${revision}-replica` }] : []) + } + case "containerapp job execution": + return JSON.stringify(args.includes("list") ? [] : { properties: { status: setupStatus } }) + case "containerapp job start": + return JSON.stringify({ name: "setup-execution" }) + case "containerapp revision deactivate": + case "containerapp revision activate": + return "null" + default: + if (args.slice(0, 2).join(" ") === "containerapp show") { + return JSON.stringify({ + properties: { latestRevisionName: "new", latestReadyRevisionName: "new" }, + }) + } + throw new Error(`Unexpected Azure command: ${command.join(" ")}`) + } + }) + mock.method(timers, "setTimeout", async () => undefined) + mock.method(globalThis, "fetch", async (url: Parameters[0]) => { + events.push(["HTTPS", String(url)]) + return new Response(null, { status: 200 }) + }) + const args = [ + "release", + "--subscription", + installation.subscription_id, + "--backend", + "backend.hcl", + "--variables", + "installation.tfvars", + "--image", + `example.azurecr.io/hub@sha256:${"a".repeat(64)}`, + "--accept-maintenance", + ] + return { args, events, replicaReads } +} + +describe("deployHub()", () => { + it("drains all old revisions and completes setup before rollout, readiness, and lease release", async () => { + const { args, events, replicaReads } = installationScenario() + + await deployHub(args) + + const setup = events.findIndex( + (event) => event.slice(0, 4).join(" ") === "az containerapp job start", + ) + const drained = events + .map((event, index) => ({ event, index })) + .filter(({ event }) => event.slice(0, 4).join(" ") === "az containerapp replica list") + const deactivate = events.findIndex( + (event) => event.includes("deactivate") && event.includes("old-active"), + ) + assert.ok(deactivate >= 0) + assert.ok(deactivate < drained[0].index) + assert.equal(replicaReads.get("old-active"), 2) + assert.equal(replicaReads.get("old-terminating"), 2) + assert.ok(Math.max(...drained.map(({ index }) => index)) < setup) + const setupSucceeded = events.findIndex( + (event) => event.slice(0, 5).join(" ") === "az containerapp job execution show", + ) + const rollout = events.findIndex( + (event) => + event[0] === "terraform" && + event[2] === "plan" && + event.includes("-var=deploy_hub=true") && + !event.some((argument) => argument.startsWith("-target=")), + ) + assert.ok(setupSucceeded > setup) + assert.ok(rollout > setupSucceeded) + const rolloutApplied = events.findIndex( + (event, index) => index > rollout && event[0] === "terraform" && event[2] === "apply", + ) + assert.ok(rolloutApplied > rollout) + const ready = events.findIndex((event) => event[0] === "HTTPS") + const release = events.findIndex( + (event) => event.slice(0, 5).join(" ") === "az storage blob lease release", + ) + assert.deepEqual(events[ready], ["HTTPS", "https://hub.example.com/api/ready"]) + assert.ok(ready > rolloutApplied) + assert.ok(release > ready) + }) + + it("leaves Hub stopped and retains its lease when setup fails", async () => { + const { args, events } = installationScenario({ setupStatus: "Failed" }) + + await assert.rejects(deployHub(args), /Setup failed/) + + assert.ok(events.some((event) => event.includes("deactivate"))) + const plans = events.filter((event) => event[0] === "terraform" && event[2] === "plan") + assert.equal(plans.length, 1) + assert.ok(plans[0].includes("-target=azurerm_container_app_job.setup")) + assert.ok(!events.some((event) => event.includes("activate"))) + assert.ok(!events.some((event) => event.includes("release"))) + assert.ok(!events.some((event) => event[0] === "HTTPS")) + }) + + it("runs setup before creating the first Hub app", async () => { + const { args, events } = installationScenario({ appCreated: false }) + + await deployHub(args) + + assert.ok(!events.some((event) => event.includes("deactivate"))) + const plans = events.filter((event) => event[0] === "terraform" && event[2] === "plan") + assert.equal(plans.length, 2) + assert.ok(plans[0].includes("-var=deploy_hub=false")) + assert.ok(plans[0].includes("-target=azurerm_container_app_job.setup")) + assert.ok(plans[1].includes("-var=deploy_hub=true")) + const setup = events.findIndex( + (event) => event.slice(0, 4).join(" ") === "az containerapp job start", + ) + assert.ok(setup > events.indexOf(plans[0])) + assert.ok(events.indexOf(plans[1]) > setup) + }) + + it("does not stop Hub or run setup while another release holds the lease", async () => { + const { args, events } = installationScenario({ lockAvailable: false }) + + await assert.rejects(deployHub(args), /Release lease is already held/) + + assert.ok( + events.some((event) => event.slice(0, 5).join(" ") === "az storage blob lease acquire"), + ) + assert.ok(!events.some((event) => event.includes("deactivate"))) + assert.ok(!events.some((event) => event.slice(0, 4).join(" ") === "az containerapp job start")) + assert.ok(!events.some((event) => event[0] === "terraform" && event[2] === "apply")) + }) + + it("refuses bootstrap when Terraform state already contains resources", async () => { + const { args, events } = installationScenario() + args[0] = "bootstrap" + + await assert.rejects(deployHub(args), /Infrastructure already exists/) + + assert.ok( + events.some( + (event) => event[0] === "terraform" && event[2] === "state" && event[3] === "list", + ), + ) + assert.ok(!events.some((event) => event[0] === "terraform" && event[2] === "apply")) + }) +}) diff --git a/infrastructure/azure/deploy-hub/index.ts b/infrastructure/azure/deploy-hub/index.ts new file mode 100644 index 00000000..32ff2d68 --- /dev/null +++ b/infrastructure/azure/deploy-hub/index.ts @@ -0,0 +1,365 @@ +import childProcess from "node:child_process" +import { randomUUID } from "node:crypto" +import { rm } from "node:fs/promises" +import { resolve } from "node:path" +import timers from "node:timers/promises" +import { pathToFileURL } from "node:url" +import { parseArgs } from "node:util" + +import type { ContainerApp, DeploymentPlan, Installation, JobExecution, Revision } from "./types.ts" + +export async function deployHub(args: string[] = process.argv.slice(2)): Promise { + const { values, positionals } = parseArgs({ + args, + allowPositionals: true, + options: { + subscription: { type: "string" }, + backend: { type: "string" }, + variables: { type: "string" }, + image: { type: "string" }, + "accept-maintenance": { type: "boolean" }, + confirm: { type: "string" }, + help: { type: "boolean" }, + }, + }) + if (values.help) { + console.log( + "Usage: node infrastructure/azure/deploy-hub/index.ts bootstrap|release|destroy " + + "--subscription ID --backend FILE --variables FILE --image REGISTRY/IMAGE@sha256:DIGEST " + + "[--accept-maintenance] [--confirm INSTALLATION]", + ) + return + } + const operation = positionals[0] + if ( + positionals.length !== 1 || + !operation || + !["bootstrap", "release", "destroy"].includes(operation) + ) { + throw new Error("Choose exactly one operation: bootstrap, release, or destroy.") + } + const { subscription, backend, variables, image } = values + if (!subscription || !backend || !variables || !image) { + throw new Error("Supply --subscription, --backend, --variables, and --image.") + } + if (!/^[a-z0-9][a-z0-9.:/-]+@sha256:[a-f0-9]{64}$/.test(image)) { + throw new Error("--image must name an immutable SHA-256 digest.") + } + if (operation !== "bootstrap" && !values["accept-maintenance"]) { + throw new Error("Releases and destruction stop Hub. Supply --accept-maintenance.") + } + + const subscriptionId = subscription + const variablesPath = resolve(variables) + const root = resolve(import.meta.dirname, "..") + const planPath = resolve(root, ".terraform", `deployment-${randomUUID()}.tfplan`) + const previousMask = process.umask(0o077) + let lease: string | undefined + + function command(executable: string, arguments_: string[], capture = false): string { + return ( + childProcess + .execFileSync(executable, arguments_, { + encoding: "utf8", + stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit", + maxBuffer: 32 * 1024 * 1024, + }) + ?.trim() ?? "" + ) + } + function terraform(arguments_: string[], capture = false): string { + return command("terraform", [`-chdir=${root}`, ...arguments_], capture) + } + function azure(...arguments_: string[]): T { + return JSON.parse( + command( + "az", + [...arguments_, "--subscription", subscriptionId, "--only-show-errors", "--output", "json"], + true, + ), + ) as T + } + function readInstallation(): Installation { + const installation = JSON.parse( + terraform(["output", "-json", "installation"], true), + ) as Installation + if (installation.subscription_id !== subscription) { + throw new Error("The selected subscription does not match this Terraform state.") + } + return installation + } + function verifyOwnership(installation: Installation): void { + const group = azure<{ tags?: Record }>( + "group", + "show", + "--name", + installation.resource_group, + ) + const expected = { + application: "replica-hub", + managed_by: "terraform", + installation: installation.name, + owner: installation.owner, + } + if (Object.entries(expected).some(([key, value]) => group.tags?.[key] !== value)) { + throw new Error("Resource group ownership does not match this installation.") + } + } + function appArguments(installation: Installation): string[] { + return ["--name", installation.app_name, "--resource-group", installation.resource_group] + } + function jobArguments(installation: Installation): string[] { + return ["--name", installation.setup_job_name, "--resource-group", installation.resource_group] + } + function blobArguments(installation: Installation, deleting = false): string[] { + return [ + "--account-name", + installation.storage_account, + "--container-name", + installation.release_container, + deleting ? "--name" : "--blob-name", + installation.release_blob, + "--auth-mode", + "login", + ] + } + function plan( + deploy: boolean, + installation?: Installation, + target?: string, + destroy = false, + ): void { + const arguments_ = [ + "plan", + "-input=false", + "-lock-timeout=60s", + `-var-file=${variablesPath}`, + `-var=image=${image}`, + `-var=deploy_hub=${deploy}`, + `-out=${planPath}`, + ] + if (target) arguments_.push(`-target=${target}`) + if (destroy) arguments_.push("-destroy") + terraform(arguments_) + const planned = JSON.parse(terraform(["show", "-json", planPath], true)) as DeploymentPlan + if (!destroy) { + if (planned.resource_changes?.some(({ change }) => change.actions.includes("delete"))) { + throw new Error( + "Release plans cannot delete or replace resources. Review recovery separately.", + ) + } + if (planned.variables.subscription_id?.value !== subscription) { + throw new Error("The variable file selects a different subscription.") + } + if (installation && planned.variables.name?.value !== installation.name) { + throw new Error("The variable file selects a different installation.") + } + } + } + function apply(): void { + terraform(["apply", "-input=false", "-lock-timeout=60s", planPath]) + } + async function stopHub(installation: Installation): Promise { + if (!installation.app_created) return + const revisions = azure( + "containerapp", + "revision", + "list", + ...appArguments(installation), + ) + for (const revision of revisions) { + if (revision.properties.active) { + azure( + "containerapp", + "revision", + "deactivate", + ...appArguments(installation), + "--revision", + revision.name, + ) + } + } + const deadline = performance.now() + 300_000 + // Inactive revisions may still have terminating replicas. + for (const revision of revisions) { + while ( + azure( + "containerapp", + "replica", + "list", + ...appArguments(installation), + "--revision", + revision.name, + ).length + ) { + if (performance.now() >= deadline) { + throw new Error("Old replicas have not stopped. Hub remains in maintenance.") + } + await timers.setTimeout(5_000) + } + } + } + async function setup(installation: Installation): Promise { + const execution = azure( + "containerapp", + "job", + "start", + ...jobArguments(installation), + ) + const deadline = performance.now() + 1_920_000 + while (performance.now() < deadline) { + const result = azure( + "containerapp", + "job", + "execution", + "show", + ...jobArguments(installation), + "--job-execution-name", + execution.name, + ) + if (result.properties.status === "Succeeded") return + if (["Failed", "Stopped", "Canceled"].includes(result.properties.status)) { + throw new Error("Setup failed. Hub remains stopped; inspect the setup execution.") + } + await timers.setTimeout(5_000) + } + throw new Error("Setup timed out. Check or stop the execution before releasing the lock.") + } + async function waitReady(installation: Installation): Promise { + const app = azure("containerapp", "show", ...appArguments(installation)) + const revision = app.properties.latestRevisionName + azure( + "containerapp", + "revision", + "activate", + ...appArguments(installation), + "--revision", + revision, + ) + const deadline = performance.now() + 300_000 + while (performance.now() < deadline) { + const latest = azure("containerapp", "show", ...appArguments(installation)) + if (latest.properties.latestReadyRevisionName === revision) { + try { + const response = await fetch(`${installation.url}/api/ready`, { + signal: AbortSignal.timeout(10_000), + }) + await response.body?.cancel() + if (response.status === 200) return + console.warn(`Hub readiness returned HTTP ${response.status}; retrying.`) + } catch (error) { + if ( + !(error instanceof TypeError) && + !(error instanceof DOMException && error.name === "TimeoutError") + ) + throw error + console.warn("Hub readiness request failed; retrying.") + } + } + await timers.setTimeout(5_000) + } + throw new Error("Hub did not become ready. Keep the release lock and investigate.") + } + + try { + terraform(["init", "-input=false", `-backend-config=${resolve(backend)}`]) + if (azure<{ id: string }>("account", "show").id !== subscription) { + throw new Error("Azure returned a different subscription.") + } + if (operation === "bootstrap") { + if (terraform(["state", "list"], true).trim()) { + throw new Error( + "Infrastructure already exists. Use release; review partial bootstrap recovery separately.", + ) + } + plan(false) + apply() + console.log("Infrastructure is ready. Hub is not running. Run release to apply setup first.") + return + } + let installation = readInstallation() + verifyOwnership(installation) + if (operation === "destroy" && values.confirm !== installation.name) { + throw new Error("Destroy requires --confirm with the exact installation name.") + } + const acquired = azure( + "storage", + "blob", + "lease", + "acquire", + ...blobArguments(installation), + "--lease-duration", + "-1", + ) + if (typeof acquired !== "string" || !acquired) { + throw new Error("Azure did not return a release lease ID.") + } + lease = acquired + installation = readInstallation() + verifyOwnership(installation) + const executions = azure( + "containerapp", + "job", + "execution", + "list", + ...jobArguments(installation), + ) + if ( + executions.some( + ({ properties }) => + !["Succeeded", "Failed", "Stopped", "Canceled"].includes(properties.status), + ) + ) { + throw new Error("A setup execution is still running; do not start another release.") + } + await stopHub(installation) + if (operation === "destroy") { + plan(installation.app_created, installation, undefined, true) + azure("storage", "blob", "delete", ...blobArguments(installation, true), "--lease-id", lease) + lease = undefined + apply() + console.log("Terraform-tracked installation resources were destroyed.") + return + } + // Terraform must not update the app until the entire setup sequence succeeds. + plan(installation.app_created, installation, "azurerm_container_app_job.setup") + apply() + await setup(installation) + plan(true, installation) + apply() + installation = readInstallation() + await waitReady(installation) + azure( + "storage", + "blob", + "lease", + "release", + ...blobArguments(installation), + "--lease-id", + lease, + ) + lease = undefined + console.log("Hub is ready. The maintenance window is complete.") + } catch (error) { + if (lease) + console.error( + "The infinite release lease is retained. Verify all processes and setup executions have stopped before an operator breaks the lease.", + ) + throw error + } finally { + try { + await rm(planPath, { force: true }) + } finally { + process.umask(previousMask) + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + await deployHub() + } catch (error) { + console.error("Deployment stopped:", error instanceof Error ? error.message : "Unknown failure") + process.exitCode = 1 + } +} diff --git a/infrastructure/azure/deploy-hub/types.ts b/infrastructure/azure/deploy-hub/types.ts new file mode 100644 index 00000000..7090c808 --- /dev/null +++ b/infrastructure/azure/deploy-hub/types.ts @@ -0,0 +1,32 @@ +export interface Installation { + name: string + owner: string + subscription_id: string + resource_group: string + app_name: string + setup_job_name: string + storage_account: string + release_container: string + release_blob: string + app_created: boolean + url: string +} + +export interface DeploymentPlan { + variables: Record + resource_changes?: { change: { actions: string[] } }[] +} + +export interface Revision { + name: string + properties: { active: boolean } +} + +export interface JobExecution { + name: string + properties: { status: string } +} + +export interface ContainerApp { + properties: { latestRevisionName: string; latestReadyRevisionName?: string } +} diff --git a/infrastructure/azure/deploy.py b/infrastructure/azure/deploy.py deleted file mode 100644 index e2f556be..00000000 --- a/infrastructure/azure/deploy.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 -"""Provision once, then release Hub with an explicit maintenance window.""" - -import argparse -import json -import os -from pathlib import Path -import re -import subprocess -import sys -import time -import urllib.error -import urllib.request -from uuid import uuid4 - - -ROOT = Path(__file__).resolve().parent -TERMINAL_EXECUTIONS = {"Succeeded", "Failed", "Stopped", "Canceled"} - - -def command(arguments, capture=False): - result = subprocess.run(arguments, check=True, text=True, capture_output=capture) - return result.stdout.strip() if capture else None - - -class Deployment: - def __init__(self, arguments): - self.args = arguments - self.installation = None - self.lease = None - self.plan_path = ROOT / ".terraform" / f"deployment-{uuid4().hex}.tfplan" - - def terraform(self, *arguments, capture=False): - return command(["terraform", f"-chdir={ROOT}", *arguments], capture=capture) - - def azure(self, *arguments): - return json.loads(command([ - "az", *arguments, "--subscription", self.args.subscription, - "--only-show-errors", "--output", "json", - ], capture=True) or "null") - - def app_arguments(self): - return ["--name", self.installation["app_name"], - "--resource-group", self.installation["resource_group"]] - - def job_arguments(self): - return ["--name", self.installation["setup_job_name"], - "--resource-group", self.installation["resource_group"]] - - def blob_arguments(self, lease=True): - return ["--account-name", self.installation["storage_account"], - "--container-name", self.installation["release_container"], - "--blob-name" if lease else "--name", self.installation["release_blob"], - "--auth-mode", "login"] - - def read_installation(self): - self.installation = json.loads(self.terraform("output", "-json", "installation", capture=True)) - if self.installation["subscription_id"] != self.args.subscription: - raise RuntimeError("The selected subscription does not match this Terraform state.") - - def verify_ownership(self): - group = self.azure("group", "show", "--name", self.installation["resource_group"]) - expected = { - "application": "replica-hub", "managed_by": "terraform", - "installation": self.installation["name"], "owner": self.installation["owner"], - } - if any(group.get("tags", {}).get(key) != value for key, value in expected.items()): - raise RuntimeError("Resource group ownership does not match this installation.") - - def acquire_lock(self): - self.lease = self.azure("storage", "blob", "lease", "acquire", *self.blob_arguments(), - "--lease-duration", "-1") - if not isinstance(self.lease, str) or not self.lease: - raise RuntimeError("Azure did not return a release lease ID.") - - def release_lock(self): - self.azure("storage", "blob", "lease", "release", *self.blob_arguments(), - "--lease-id", self.lease) - self.lease = None - - def plan(self, deploy_hub, target=None, destroy=False): - arguments = [ - "plan", "-input=false", "-lock-timeout=60s", - f"-var-file={self.args.variables}", f"-var=image={self.args.image}", - f"-var=deploy_hub={str(deploy_hub).lower()}", f"-out={self.plan_path}", - ] - if target: - arguments.append(f"-target={target}") - if destroy: - arguments.append("-destroy") - self.terraform(*arguments) - plan = json.loads(self.terraform("show", "-json", str(self.plan_path), capture=True)) - if not destroy: - changes = plan.get("resource_changes", []) - if any("delete" in change["change"]["actions"] for change in changes): - raise RuntimeError("Release plans cannot delete or replace resources. Review recovery separately.") - variables = plan["variables"] - if variables["subscription_id"]["value"] != self.args.subscription: - raise RuntimeError("The variable file selects a different subscription.") - if self.installation and variables["name"]["value"] != self.installation["name"]: - raise RuntimeError("The variable file selects a different installation.") - - def apply(self): - self.terraform("apply", "-input=false", "-lock-timeout=60s", str(self.plan_path)) - - def stop_hub(self): - if not self.installation["app_created"]: - return - revisions = self.azure("containerapp", "revision", "list", *self.app_arguments()) - # Inspect every revision: an inactive revision may still be terminating. - for revision in revisions: - if revision["properties"]["active"]: - self.azure("containerapp", "revision", "deactivate", *self.app_arguments(), - "--revision", revision["name"]) - deadline = time.monotonic() + 300 - for revision in revisions: - while self.azure("containerapp", "replica", "list", *self.app_arguments(), - "--revision", revision["name"]): - if time.monotonic() >= deadline: - raise RuntimeError("Old replicas have not stopped. Hub remains in maintenance.") - time.sleep(5) - - def ensure_no_setup(self): - executions = self.azure("containerapp", "job", "execution", "list", *self.job_arguments()) - if any(item["properties"]["status"] not in TERMINAL_EXECUTIONS for item in executions): - raise RuntimeError("A setup execution is still running; do not start another release.") - - def setup(self): - execution = self.azure("containerapp", "job", "start", *self.job_arguments()) - deadline = time.monotonic() + 1920 - while time.monotonic() < deadline: - result = self.azure("containerapp", "job", "execution", "show", - *self.job_arguments(), "--job-execution-name", execution["name"]) - status = result["properties"]["status"] - if status == "Succeeded": - return - if status in TERMINAL_EXECUTIONS: - raise RuntimeError("Setup failed. Hub remains stopped; inspect the setup execution.") - time.sleep(5) - raise RuntimeError("Setup timed out. Check or stop the execution before releasing the lock.") - - def wait_ready(self): - self.read_installation() - app = self.azure("containerapp", "show", *self.app_arguments()) - revision = app["properties"]["latestRevisionName"] - self.azure("containerapp", "revision", "activate", *self.app_arguments(), - "--revision", revision) - deadline = time.monotonic() + 300 - while time.monotonic() < deadline: - app = self.azure("containerapp", "show", *self.app_arguments()) - properties = app["properties"] - if properties.get("latestReadyRevisionName") == revision: - try: - with urllib.request.urlopen(self.installation["url"] + "/api/ready", timeout=10) as response: - if response.status == 200: - return - except (urllib.error.URLError, TimeoutError): - pass - time.sleep(5) - raise RuntimeError("Hub did not become ready. Keep the release lock and investigate.") - - def run(self): - self.terraform("init", "-input=false", f"-backend-config={self.args.backend}") - account = self.azure("account", "show") - if account["id"] != self.args.subscription: - raise RuntimeError("Azure returned a different subscription.") - if self.args.operation == "bootstrap": - resources = self.terraform("state", "list", capture=True).splitlines() - if resources: - raise RuntimeError("Infrastructure already exists. Use release; review partial bootstrap recovery separately.") - self.plan(False) - self.apply() - print("Infrastructure is ready. Hub is not running. Run release to apply setup first.") - return - - self.read_installation() - self.verify_ownership() - if self.args.operation == "destroy" and self.args.confirm != self.installation["name"]: - raise RuntimeError("Destroy requires --confirm with the exact installation name.") - self.acquire_lock() - self.read_installation() - self.verify_ownership() - self.ensure_no_setup() - self.stop_hub() - if self.args.operation == "destroy": - self.plan(self.installation["app_created"], destroy=True) - # Deleting our leased lock blob prevents new releases while Terraform destroys its container. - self.azure("storage", "blob", "delete", *self.blob_arguments(lease=False), "--lease-id", self.lease) - self.lease = None - self.apply() - print("Terraform-tracked installation resources were destroyed.") - return - - # The target deliberately excludes the app until the complete setup sequence succeeds. - self.plan(self.installation["app_created"], target="azurerm_container_app_job.setup") - self.apply() - self.setup() - self.plan(True) - self.apply() - self.wait_ready() - self.release_lock() - print("Hub is ready. The maintenance window is complete.") - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("operation", choices=["bootstrap", "release", "destroy"]) - parser.add_argument("--subscription", required=True) - parser.add_argument("--backend", type=lambda value: str(Path(value).resolve()), required=True) - parser.add_argument("--variables", type=lambda value: str(Path(value).resolve()), required=True) - parser.add_argument("--image", required=True, help="Existing registry/repository@sha256:digest") - parser.add_argument("--accept-maintenance", action="store_true") - parser.add_argument("--confirm", help="Exact installation name; required for destruction") - arguments = parser.parse_args(argv) - if not re.fullmatch(r"[a-z0-9][a-z0-9.:/-]+@sha256:[a-f0-9]{64}", arguments.image): - parser.error("--image must name an immutable SHA-256 digest.") - if arguments.operation != "bootstrap" and not arguments.accept_maintenance: - parser.error("Releases and destruction stop Hub. Supply --accept-maintenance.") - os.umask(0o077) - deployment = Deployment(arguments) - try: - deployment.run() - except (RuntimeError, subprocess.CalledProcessError, KeyboardInterrupt) as error: - print(f"Deployment stopped: {error}", file=sys.stderr) - if deployment.lease: - print("The infinite release lease is retained. Verify all processes and setup executions " - "have stopped before an operator breaks the lease.", file=sys.stderr) - return 1 - finally: - deployment.plan_path.unlink(missing_ok=True) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/infrastructure/azure/test_deploy.py b/infrastructure/azure/test_deploy.py deleted file mode 100644 index a5864c0c..00000000 --- a/infrastructure/azure/test_deploy.py +++ /dev/null @@ -1,143 +0,0 @@ -import contextlib -import io -import json -import subprocess -import unittest -from unittest.mock import MagicMock, patch - -import deploy - - -IMAGE = "example.azurecr.io/hub@sha256:" + "a" * 64 - - -class AzureScenario: - def __init__(self, app_created=True, setup_status="Succeeded", lock_available=True): - self.installation = { - "name": "test-hub", "owner": "test-team", "subscription_id": "test-subscription", - "resource_group": "test-hub", "app_name": "test-hub", "setup_job_name": "test-setup", - "storage_account": "testhub", "release_container": "releases", - "release_blob": "deployment.lock", "app_created": app_created, - "url": "https://hub.example.com", - } - self.calls = [] - self.setup_status = setup_status - self.lock_available = lock_available - self.targeted = False - self.replicas_read = 0 - - def command(self, arguments, capture=False): - self.calls.append(arguments) - if arguments[0] == "terraform": - operation = arguments[2] - if operation == "output": - return json.dumps(self.installation) - if operation == "state": - return "azurerm_container_app.hub[0]" if self.installation["app_created"] else "" - if operation == "plan": - self.targeted = any(item.startswith("-target=") for item in arguments) - if operation == "show": - return json.dumps({ - "variables": { - "name": {"value": self.installation["name"]}, - "subscription_id": {"value": self.installation["subscription_id"]}, - }, - "resource_changes": [], - }) - if operation == "apply" and not self.targeted: - self.installation["app_created"] = True - return "" - operation = arguments[1:4] - if operation[:2] == ["account", "show"]: - value = {"id": self.installation["subscription_id"]} - elif operation[:2] == ["group", "show"]: - value = {"tags": { - "application": "replica-hub", "managed_by": "terraform", - "installation": self.installation["name"], "owner": self.installation["owner"], - }} - elif operation == ["storage", "blob", "lease"]: - if "acquire" in arguments: - if not self.lock_available: - raise subprocess.CalledProcessError(1, arguments) - value = "lease-id" - else: - value = None - elif operation == ["containerapp", "revision", "list"]: - value = [{"name": "old-revision", "properties": {"active": True}}] - elif operation == ["containerapp", "replica", "list"]: - self.replicas_read += 1 - value = [{"name": "old-replica"}] if self.replicas_read == 1 else [] - elif operation == ["containerapp", "job", "execution"]: - value = [] if "list" in arguments else {"properties": {"status": self.setup_status}} - elif operation == ["containerapp", "job", "start"]: - value = {"name": "setup-execution"} - elif operation[:2] == ["containerapp", "show"]: - value = {"properties": {"latestRevisionName": "new", "latestReadyRevisionName": "new"}} - elif operation[:2] == ["containerapp", "revision"]: - value = None - else: - raise AssertionError(f"Unexpected Azure command: {arguments}") - return json.dumps(value) - - def run(self, operation="release"): - response = MagicMock() - response.__enter__.return_value.status = 200 - with patch.object(deploy, "command", side_effect=self.command), \ - patch.object(deploy.time, "sleep"), \ - patch.object(deploy.urllib.request, "urlopen", return_value=response), \ - contextlib.redirect_stdout(io.StringIO()), \ - contextlib.redirect_stderr(io.StringIO()): - return deploy.main([ - operation, "--subscription", "test-subscription", - "--backend", "backend.hcl", "--variables", "installation.tfvars", - "--image", IMAGE, "--accept-maintenance", - ]) - - -class DeploymentTests(unittest.TestCase): - def test_setup_failure_leaves_hub_stopped_and_retains_release_lock(self): - scenario = AzureScenario(setup_status="Failed") - self.assertEqual(scenario.run(), 1) - self.assertTrue(any("deactivate" in call for call in scenario.calls)) - plans = [call for call in scenario.calls if call[0] == "terraform" and call[2] == "plan"] - self.assertEqual(len(plans), 1) - self.assertIn("-target=azurerm_container_app_job.setup", plans[0]) - self.assertFalse(any("activate" in call for call in scenario.calls)) - self.assertFalse(any("release" in call for call in scenario.calls)) - - def test_release_waits_for_old_replicas_and_setup_before_rollout(self): - scenario = AzureScenario() - self.assertEqual(scenario.run(), 0) - events = scenario.calls - setup = next(index for index, call in enumerate(events) if call[1:4] == ["containerapp", "job", "start"]) - drained = [index for index, call in enumerate(events) if call[1:4] == ["containerapp", "replica", "list"]] - rollout = next(index for index, call in enumerate(events) - if call[0] == "terraform" and call[2] == "plan" and "-var=deploy_hub=true" in call - and not any(item.startswith("-target=") for item in call)) - self.assertEqual(len(drained), 2) - self.assertLess(max(drained), setup) - self.assertLess(setup, rollout) - self.assertTrue(any("release" in call for call in events)) - - def test_initial_release_runs_setup_before_creating_the_app(self): - scenario = AzureScenario(app_created=False) - self.assertEqual(scenario.run(), 0) - self.assertFalse(any("deactivate" in call for call in scenario.calls)) - plans = [call for call in scenario.calls if call[0] == "terraform" and call[2] == "plan"] - self.assertIn("-var=deploy_hub=false", plans[0]) - self.assertIn("-var=deploy_hub=true", plans[1]) - - def test_competing_release_cannot_stop_hub_or_run_setup(self): - scenario = AzureScenario(lock_available=False) - self.assertEqual(scenario.run(), 1) - self.assertFalse(any("deactivate" in call for call in scenario.calls)) - self.assertFalse(any(call[1:4] == ["containerapp", "job", "start"] for call in scenario.calls)) - - def test_bootstrap_refuses_an_existing_hub(self): - scenario = AzureScenario() - self.assertEqual(scenario.run("bootstrap"), 1) - self.assertFalse(any(call[0] == "terraform" and call[2] == "apply" for call in scenario.calls)) - - -if __name__ == "__main__": - unittest.main() diff --git a/infrastructure/azure/variables.tf b/infrastructure/azure/variables.tf index 1013470b..b08c6ec3 100644 --- a/infrastructure/azure/variables.tf +++ b/infrastructure/azure/variables.tf @@ -41,7 +41,7 @@ variable "image" { variable "deploy_hub" { type = bool default = false - description = "Keep false for bootstrap. Only deploy.py enables Hub after setup succeeds." + description = "Keep false for bootstrap. Only deploy-hub enables Hub after setup succeeds." } variable "acr_id" { diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index 99e58089..e839e4a9 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -10,6 +10,6 @@ "skipLibCheck": true, "types": ["node"] }, - "include": ["**/*.ts"], + "include": ["**/*.ts", "../infrastructure/**/*.ts"], "exclude": ["node_modules"] } From 280faa7382f5511a73ad53a265851503551ee826 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:22:27 +0000 Subject: [PATCH 3/4] fix(infra): isolate Hub deployment access and safe release checks Use separate database roles and workload identities, native Key Vault references, bounded artifact readiness, and private command diagnostics. Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-storage.md | 8 +- hub/docs/database.md | 16 +++ hub/docs/deployment.md | 3 +- hub/docs/testing-and-operations.md | 2 +- hub/script/check-image/index.ts | 112 ++++++++++++++++- .../create-artifact-storage/index.test.ts | 28 ++++- .../create-artifact-storage/index.ts | 15 +++ .../create-artifact-storage/types.ts | 1 + hub/server/index.ts | 18 ++- .../index.test.ts | 43 ++++++- .../index.ts | 8 ++ .../types.ts | 1 + .../index.ts | 9 ++ .../types.ts | 1 + .../grant-runtime-database-access/index.ts | 46 +++++++ .../provision-database-access/index.ts | 73 +++++++++++ hub/server/scripts/setup/index.ts | 17 ++- .../start-server-processes/index.test.ts | 21 +++- hub/server/start-server-processes/index.ts | 2 + infrastructure/README.md | 24 ++-- infrastructure/azure/compute.tf | 60 +++++---- infrastructure/azure/deploy-hub/index.test.ts | 64 +++++++++- infrastructure/azure/deploy-hub/index.ts | 41 +++--- .../azure/installation.tfvars.example | 19 ++- infrastructure/azure/main.tf | 33 ++++- infrastructure/azure/outputs.tf | 1 + .../azure/tests/deployment.tftest.hcl | 118 +++++++++++++++++- infrastructure/azure/variables.tf | 60 ++++++++- 28 files changed, 753 insertions(+), 91 deletions(-) create mode 100644 hub/server/scripts/grant-runtime-database-access/index.ts create mode 100644 hub/server/scripts/provision-database-access/index.ts diff --git a/hub/docs/artifact-storage.md b/hub/docs/artifact-storage.md index 9519515e..5f2bde01 100644 --- a/hub/docs/artifact-storage.md +++ b/hub/docs/artifact-storage.md @@ -4,7 +4,7 @@ The artifact storage foundation saves immutable bytes behind a provider-neutral ## Composition -Call `createArtifactStorage(settings)` from `server/domains/artifacts/create-artifact-storage/index.ts` during startup. The asynchronous factory validates settings and returns `read(key)`, `write(key, bytes)`, and `delete(key)`. `parseArtifactStorageSettings(input)` validates unknown configuration separately when startup needs a typed selection before composition. +Call `createArtifactStorage(settings)` from `server/domains/artifacts/create-artifact-storage/index.ts` during startup. The asynchronous factory validates settings and returns `isReady()`, `read(key)`, `write(key, bytes)`, and `delete(key)`. `parseArtifactStorageSettings(input)` validates unknown configuration separately when startup needs a typed selection before composition. Choose exactly one settings shape: @@ -35,11 +35,13 @@ Alternatively, startup can call `createConfiguredArtifactStorage(environment)` f The wrapper constructs the public-cloud HTTPS account URL and calls the typed factory. Missing or invalid providers, incomplete configuration, and mixed-provider settings fail with a safe error. An empty value still counts as a supplied setting when checking mixed providers. Unrelated environment variables are ignored. Startup may leave storage uncomposed when artifact storage is not configured; calling the wrapper without a provider always rejects. -API, combined, and independent worker entry points validate and construct configured storage before startup. Any supplied provider, directory, account, or container setting opts in, so incomplete settings fail rather than being ignored. Setup does not need storage. Without storage settings, development keeps working unchanged. This startup check does not read or write artifacts, prove Azure access, or connect an unimplemented product flow. +API, combined, and independent worker entry points validate and construct configured storage before startup. Any supplied provider, directory, account, or container setting opts in, so incomplete settings fail rather than being ignored. Setup does not need storage. Without storage settings, development keeps working unchanged. Construction alone does not prove Azure access or connect an unimplemented product flow. Unknown typed settings, relative directories, account credentials, URL query strings, and unsupported endpoints fail validation with a safe error. Azure selection never falls back to local storage. -The Azure endpoint accepts public-cloud HTTPS account URLs only. The adapter uses `ManagedIdentityCredential`, with either the system-assigned identity or the explicitly selected user-assigned identity. It never uses account keys, connection strings, developer credentials, or `DefaultAzureCredential`. Operators provision the account and container and grant the runtime identity `Storage Blob Data Contributor` at the intended container scope. Construction does not contact Azure; missing identity, permissions, or container availability is reported when an operation runs. +The Azure endpoint accepts public-cloud HTTPS account URLs only. The adapter uses `ManagedIdentityCredential`, with either the system-assigned identity or the explicitly selected user-assigned identity. It never uses account keys, connection strings, developer credentials, or `DefaultAzureCredential`. Operators provision the account and container and grant the runtime identity `Storage Blob Data Contributor` at the intended container scope. + +The API's `/api/ready` includes configured storage. `isReady()` has a two-second deadline and makes a non-writing container-properties request through the selected managed identity, or verifies the filesystem directory. Provider failure or timeout fails readiness with a safe diagnostic. The Azure check verifies container read access, not write/delete permission, available capacity, or product operations. It never creates or deletes a probe artifact. Standalone workers have no readiness endpoint. ## Bytes and results diff --git a/hub/docs/database.md b/hub/docs/database.md index c64e948e..2142ffb5 100644 --- a/hub/docs/database.md +++ b/hub/docs/database.md @@ -68,4 +68,20 @@ npm run db:generate Use `db:migrate` only with a local development database. It needs permission to create a shadow database to calculate migrations. `npm run db:deploy` applies committed migrations without creating new ones; database tests use this command so constraints in committed migrations are exercised. It is not a production deployment setup. `npm run db:studio` opens Prisma's database editor and should be used only with an intended development database. +## Deployment database roles + +The Azure installation separates database provisioning, migrations, and normal data access: + +| Principal | Access | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Administrator | One-time database and role provisioning; never passed to Hub or its setup job. | +| `replica_migration` | Can create schemas in this database and owns `public`, `bullmq`, and their migration-created objects. Runs setup, not the long-running app. | +| `replica_runtime` | Uses both schemas, reads/writes application and queue data, uses sequences and executes queue functions. Cannot create schemas, change permanent schema objects, create roles, truncate tables, or change migration ledgers. BullMQ requires read access to its ledger and temporary-table permission for bulk jobs. | + +`server/scripts/provision-database-access/` creates the two login roles and schema ownership on a fresh database. It requires administrator access over the private database network and separate passwords of at least 16 characters. Existing roles, public tables, or a pre-existing BullMQ schema cause failure and rollback rather than adopting unknown permissions. Use a reviewed ownership migration for existing installations. + +When `DATABASE_RUNTIME_ROLE=replica_runtime`, setup runs `server/scripts/grant-runtime-database-access/` after Prisma and BullMQ setup. That script initializes the two queues' lazy ID sequences before granting data access, so runtime never needs permanent schema creation rights. It refreshes grants for newly migrated tables, sequences, and functions, excludes the Prisma ledger, and permits only reading BullMQ's ledger. Product additions that need other database capabilities must explicitly update and verify this contract. + +Local development keeps its existing database user and leaves `DATABASE_RUNTIME_ROLE` unset. See the [Azure installation guide](../../infrastructure/README.md) for private-network provisioning, secret references, release ordering, and recovery. + The [database-enabled test projects](testing-and-operations.md#parallel-database-tests) use separate databases for every project, worker, and test run. Setup deploys the committed migrations and clears records before each case. Tests do not use the development database for records. diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index e2d30d41..258740fc 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -13,6 +13,7 @@ Publishing is an explicit operator step, not a side effect of CI. From the appro | `RUNTIME_MODE` | `api` for source development; the image defaults to `combined`. `on-demand`, `scheduled`, and `setup` are separate roles using the same image. | | `HOST`, `PORT` | Source defaults to `127.0.0.1:3001`; the image binds `0.0.0.0:3001`. | | `DATABASE_URL` | PostgreSQL connection URL, required for combined, worker, and setup modes. Configure verified TLS for remote databases. | +| `DATABASE_RUNTIME_ROLE` | Setup-only opt-in for separate database roles: exactly `replica_runtime`. See [database access](database.md#deployment-database-roles). | | `CLIENT_DIRECTORY` | Absolute directory containing built frontend assets and `index.html`; the image uses `/app/dist/client`. Leave unset for the development API. | | `SHUTDOWN_TIMEOUT_MS` | Request and worker drain deadline, default 30000 milliseconds. Allow another five seconds for telemetry flushing. | | `ON_DEMAND_JOB_CONCURRENCY`, `SCHEDULED_JOB_CONCURRENCY` | Independent worker concurrency, as described in [Background jobs](background-jobs.md). | @@ -21,7 +22,7 @@ Configuration errors stop startup without printing credentials. Logging and opti The runtime serves built frontend navigation and static files on the API port. Unknown `/api` endpoints remain JSON 404 responses. Missing files with extensions do not return the frontend document. -`/api/health` is liveness. `/api/ready` returns 503 during initialization, shutdown, or a failed dependency check. In combined mode it requires both workers to initialize and a successful PostgreSQL query. Worker readiness records initialization only, not continuous consumer health: a later worker connection error can be logged while readiness still succeeds. The PostgreSQL query checks current database availability. Readiness is an operational signal, not authorization. +`/api/health` is liveness. `/api/ready` returns 503 during initialization, shutdown, or a failed dependency check. In combined mode it requires both workers to initialize and a successful PostgreSQL query. Worker readiness records initialization only, not continuous consumer health: a later worker connection error can be logged while readiness still succeeds. The PostgreSQL query checks current database availability. When [artifact storage](artifact-storage.md) is configured, both API and combined modes also require its bounded, non-writing availability check. Readiness is an operational signal, not authorization. On termination, combined mode stops all three roles and drains work within the configured deadline. A required role failing or stopping unexpectedly fails the container. Roles remain independently runnable for future deployments; cross-instance WebSocket delivery is not implemented. diff --git a/hub/docs/testing-and-operations.md b/hub/docs/testing-and-operations.md index 5a171077..8eb860e7 100644 --- a/hub/docs/testing-and-operations.md +++ b/hub/docs/testing-and-operations.md @@ -18,7 +18,7 @@ npm run storybook:build The API health check proves only that Express responds. Product API routes that have not been built must return 404 rather than pretending to work. -Run `npm run image:check` with a Docker-compatible daemon to build the production image and exercise setup, readiness, serving, and shutdown against disposable PostgreSQL. It creates uniquely named resources and removes them afterward. The image compiles the Hub runtime only; the normal `build` command also checks the frontend, test tooling, and shared extension. +Run `npm run image:check` with a Docker-compatible daemon to build the production image and exercise role provisioning, repeated setup, runtime data access and denied schema changes, queue processing, readiness, serving, and shutdown against disposable PostgreSQL. It creates uniquely named resources and removes them afterward. The image compiles the Hub runtime only; the normal `build` command also checks the frontend, test tooling, and shared extension. Installation generates Prisma Client without a database connection. Client tests check reuse across development reloads, production behavior, and errors for missing configuration. diff --git a/hub/script/check-image/index.ts b/hub/script/check-image/index.ts index 7659a8d5..09df6c93 100644 --- a/hub/script/check-image/index.ts +++ b/hub/script/check-image/index.ts @@ -63,6 +63,29 @@ export async function checkImage() { await new Promise((resolve) => setTimeout(resolve, 500)) } const databaseUrl = `DATABASE_URL=postgresql://replica:replica@${database}:5432/replica` + const migrationPassword = "replica-migration-image-only" + const runtimePassword = "replica-runtime-image-only" + docker( + "run", + "--rm", + "--network", + network, + "-e", + databaseUrl, + "-e", + `HUB_MIGRATION_PASSWORD=${migrationPassword}`, + "-e", + `HUB_RUNTIME_PASSWORD=${runtimePassword}`, + image, + "node", + "dist/server/scripts/provision-database-access/index.js", + ) + const migrationUrl = new URL(databaseUrl.slice("DATABASE_URL=".length)) + migrationUrl.username = "replica_migration" + migrationUrl.password = migrationPassword + const runtimeUrl = new URL(migrationUrl) + runtimeUrl.username = "replica_runtime" + runtimeUrl.password = runtimePassword for (let attempt = 0; attempt < 2; attempt++) { docker( "run", @@ -70,13 +93,96 @@ export async function checkImage() { "--network", network, "-e", - databaseUrl, + `DATABASE_URL=${migrationUrl}`, + "-e", + "DATABASE_RUNTIME_ROLE=replica_runtime", image, "node", "dist/server/scripts/setup/index.js", ) } - docker("run", "-d", "--name", application, "--network", network, "-e", databaseUrl, image) + docker( + "run", + "--rm", + "--network", + network, + "-e", + `DATABASE_URL=${runtimeUrl}`, + image, + "node", + "--input-type=module", + "-e", + `import assert from "node:assert/strict"; + import { randomUUID } from "node:crypto"; + import pg from "pg"; + import pino from "pino"; + import { createJobQueue } from "./dist/server/integrations/bullmq/create-job-queue/index.js"; + import { createJobWorker } from "./dist/server/integrations/bullmq/create-job-worker/index.js"; + setTimeout(() => { console.error("Runtime database access probe timed out"); process.exit(1); }, 30_000).unref(); + const databaseUrl = process.env.DATABASE_URL; + const client = new pg.Client({ connectionString: databaseUrl }); + await client.connect(); + try { + assert.equal((await client.query("SELECT current_user AS role")).rows[0].role, "replica_runtime"); + const id = randomUUID(); + await client.query("INSERT INTO public.users (id) VALUES ($1)", [id]); + assert.equal((await client.query("SELECT id FROM public.users WHERE id = $1", [id])).rows[0].id, id); + assert.equal((await client.query("UPDATE public.users SET updated_at = now() WHERE id = $1", [id])).rowCount, 1); + assert.equal((await client.query("DELETE FROM public.users WHERE id = $1", [id])).rowCount, 1); + for (const sql of [ + "CREATE TABLE public.image_permission_probe (id integer)", + "CREATE TABLE bullmq.image_permission_probe (id integer)", + "DROP TABLE public.users", + "TRUNCATE TABLE public.users", + "CREATE ROLE image_permission_probe", + "SELECT * FROM public._prisma_migrations", + "UPDATE public._prisma_migrations SET checksum = checksum", + "UPDATE bullmq.migration SET version = version", + ]) { + await assert.rejects(client.query(sql), { code: "42501" }, sql); + } + } finally { + await client.end(); + } + for (const name of ["on-demand", "scheduled"]) { + const options = { name, databaseUrl, logger: pino({ level: "silent" }) }; + const queue = createJobQueue(options); + const worker = createJobWorker({ + ...options, + processors: [{ name: "image-check", process: async ({ value }) => value * 2 }], + }); + let running; + try { + await worker.waitUntilReady(); + const jobs = await queue.addBulk([ + { name: "image-check", data: { value: 21 } }, + { name: "image-check", data: { value: 7 } }, + ]); + running = worker.run(); + void running.catch(() => {}); + const deadline = Date.now() + 10_000; + while (!(await Promise.all(jobs.map(job => job.getState()))).every(state => state === "completed")) { + assert(Date.now() < deadline, "Runtime worker did not complete queued jobs"); + await new Promise(resolve => setTimeout(resolve, 50)); + } + assert.deepEqual(await Promise.all(jobs.map(async job => (await queue.getJob(job.id)).returnvalue)), [42, 14]); + } finally { + await Promise.all([worker.close(), queue.close()]); + await running; + } + }`, + ) + docker( + "run", + "-d", + "--name", + application, + "--network", + network, + "-e", + `DATABASE_URL=${runtimeUrl}`, + image, + ) applicationCreated = true const readyDeadline = Date.now() + 60_000 while ( @@ -106,7 +212,7 @@ export async function checkImage() { "0", "Hub must exit cleanly after termination", ) - console.log("Image setup, readiness, serving, and shutdown passed.") + console.log("Image database access, setup, jobs, readiness, serving, and shutdown passed.") } catch (error) { if (applicationCreated) spawnSync("docker", ["logs", application], { stdio: "inherit" }) throw error diff --git a/hub/server/domains/artifacts/create-artifact-storage/index.test.ts b/hub/server/domains/artifacts/create-artifact-storage/index.test.ts index a6587122..1c3466f3 100644 --- a/hub/server/domains/artifacts/create-artifact-storage/index.test.ts +++ b/hub/server/domains/artifacts/create-artifact-storage/index.test.ts @@ -1,11 +1,35 @@ import { randomUUID } from "node:crypto" import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises" import { resolve } from "node:path" -import { describe, expect, it, onTestFinished } from "vitest" +import { describe, expect, it, onTestFinished, vi } from "vitest" + +import { createAzureBlobArtifactStorage } from "../../../integrations/azure-blob/create-azure-blob-artifact-storage" import { artifactMaxBytes, createArtifactStorage } from "." +vi.mock("../../../integrations/azure-blob/create-azure-blob-artifact-storage", () => ({ + createAzureBlobArtifactStorage: vi.fn(), +})) + describe("createArtifactStorage()", () => { + it("bounds readiness even when the provider ignores its abort signal", async () => { + const isReady = vi.fn<(signal: AbortSignal) => Promise>(() => new Promise(() => {})) + vi.mocked(createAzureBlobArtifactStorage).mockReturnValue({ + isReady, + read: vi.fn(), + write: vi.fn(), + delete: vi.fn(), + }) + const storage = await createArtifactStorage({ + provider: "azure-blob", + accountUrl: "https://example.blob.core.windows.net", + container: "artifacts", + }) + + await expect(storage.isReady()).rejects.toThrow("Artifact storage readiness timed out.") + expect(isReady.mock.calls[0]![0].aborted).toBe(true) + }, 3_000) + it("publishes complete immutable bytes and explicitly reports missing reads and deletes", async () => { const directory = resolve(`server/domains/artifacts/.storage-test-${randomUUID()}`) await mkdir(directory) @@ -110,8 +134,10 @@ describe("createArtifactStorage()", () => { await mkdir(directory) onTestFinished(() => rm(directory, { recursive: true, force: true })) const storage = await createArtifactStorage({ provider: "filesystem", directory }) + expect(await storage.isReady()).toBe(true) await rm(directory, { recursive: true }) + await expect(storage.isReady()).rejects.toThrow("Artifact storage is unavailable.") expect(await storage.read("key")).toEqual({ status: "unavailable" }) expect(await storage.write("key", Uint8Array.of(1))).toEqual({ status: "unavailable" }) expect(await storage.delete("key")).toEqual({ status: "unavailable" }) diff --git a/hub/server/domains/artifacts/create-artifact-storage/index.ts b/hub/server/domains/artifacts/create-artifact-storage/index.ts index 383abca7..6841cbac 100644 --- a/hub/server/domains/artifacts/create-artifact-storage/index.ts +++ b/hub/server/domains/artifacts/create-artifact-storage/index.ts @@ -19,6 +19,21 @@ export async function createArtifactStorage( : createAzureBlobArtifactStorage(settings, artifactMaxBytes) return { + async isReady() { + const signal = AbortSignal.timeout(2_000) + let abort!: () => void + const timeout = new Promise((_resolve, reject) => { + abort = () => reject(new Error("Artifact storage readiness timed out.")) + signal.addEventListener("abort", abort, { once: true }) + }) + try { + const ready = await Promise.race([adapter.isReady(signal), timeout]) + if (!ready) throw new Error("Artifact storage is unavailable.") + return true + } finally { + signal.removeEventListener("abort", abort) + } + }, async read(key) { if (typeof key !== "string" || !keyPattern.test(key)) return { status: "invalid-key" } return adapter.read(key) diff --git a/hub/server/domains/artifacts/create-artifact-storage/types.ts b/hub/server/domains/artifacts/create-artifact-storage/types.ts index 4918776b..86410338 100644 --- a/hub/server/domains/artifacts/create-artifact-storage/types.ts +++ b/hub/server/domains/artifacts/create-artifact-storage/types.ts @@ -20,6 +20,7 @@ export type ArtifactDeleteResult = { } export interface ArtifactStorage { + isReady(): Promise read(key: string): Promise write(key: string, bytes: Uint8Array): Promise delete(key: string): Promise diff --git a/hub/server/index.ts b/hub/server/index.ts index 63816c08..dc9f1890 100644 --- a/hub/server/index.ts +++ b/hub/server/index.ts @@ -12,18 +12,26 @@ import { runJobWorker } from "./workers/run-job-worker/index.ts" try { const configuration = parseRuntimeConfig(process.env) - if (configuration.mode !== "setup" && configuration.hasArtifactStorage) { - await createConfiguredArtifactStorage(process.env) - } + const storage = + configuration.mode !== "setup" && configuration.hasArtifactStorage + ? await createConfiguredArtifactStorage(process.env) + : undefined + const isStorageReady = storage ? () => storage.isReady() : undefined const telemetry = startTelemetry(process.env, logger) try { if (configuration.mode === "setup") await setup() - else if (configuration.mode === "api") await runApi({ ...configuration, logger }) + else if (configuration.mode === "api") + await runApi({ ...configuration, logger, isReady: isStorageReady }) else { const databaseUrl = configuration.databaseUrl if (!databaseUrl) throw new Error("This runtime mode requires DATABASE_URL.") if (configuration.mode === "combined") { - await startServerProcesses({ ...configuration, databaseUrl, logger }) + await startServerProcesses({ + ...configuration, + databaseUrl, + logger, + isStorageReady, + }) } else { const name = configuration.mode await runJobWorker({ diff --git a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts index 9cf487fe..737b6a1a 100644 --- a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts +++ b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ credential: vi.fn(), service: vi.fn(), + container: vi.fn(), + getProperties: vi.fn(), download: vi.fn(), uploadData: vi.fn(), delete: vi.fn(), @@ -20,8 +22,9 @@ vi.mock("@azure/storage-blob", () => ({ constructor(...args: unknown[]) { mocks.service(...args) } - getContainerClient() { - return { getBlockBlobClient: () => mocks } + getContainerClient(name: string) { + mocks.container(name) + return { getProperties: mocks.getProperties, getBlockBlobClient: () => mocks } } }, })) @@ -29,6 +32,42 @@ vi.mock("@azure/storage-blob", () => ({ import { createAzureBlobArtifactStorage } from "." describe("createAzureBlobArtifactStorage()", () => { + it("checks the configured container with the supplied abort signal without touching blobs", async () => { + const storage = createAzureBlobArtifactStorage( + { accountUrl: "https://example.blob.core.windows.net", container: "artifacts" }, + 3, + ) + const signal = new AbortController().signal + let succeed!: () => void + mocks.getProperties.mockReturnValue(new Promise((resolve) => (succeed = resolve))) + let settled = false + const ready = storage.isReady(signal).then((result) => { + settled = true + return result + }) + await Promise.resolve() + expect(settled).toBe(false) + + succeed() + + expect(await ready).toBe(true) + expect(mocks.container).toHaveBeenCalledExactlyOnceWith("artifacts") + expect(mocks.getProperties).toHaveBeenCalledExactlyOnceWith({ abortSignal: signal }) + expect(mocks.download).not.toHaveBeenCalled() + expect(mocks.uploadData).not.toHaveBeenCalled() + expect(mocks.delete).not.toHaveBeenCalled() + }) + + it("reports unavailable storage when the authenticated container check fails", async () => { + const storage = createAzureBlobArtifactStorage( + { accountUrl: "https://example.blob.core.windows.net", container: "artifacts" }, + 3, + ) + mocks.getProperties.mockRejectedValue(new Error("Private provider diagnostics")) + + expect(await storage.isReady(new AbortController().signal)).toBe(false) + }) + it("uses only managed identity and conditional writes with bounded binary reads", async () => { const clientId = "ee5eeebb-74e0-4b79-8030-c325e50fec43" const storage = createAzureBlobArtifactStorage( diff --git a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts index 2e22b9a5..6bed11ba 100644 --- a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts +++ b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts @@ -15,6 +15,14 @@ export function createAzureBlobArtifactStorage( ) return { + async isReady(signal) { + try { + await container.getProperties({ abortSignal: signal }) + return true + } catch { + return false + } + }, async read(key) { try { const response = await container.getBlockBlobClient(key).download() diff --git a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts index 49b1dec7..b11812e0 100644 --- a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts +++ b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts @@ -5,6 +5,7 @@ export interface AzureBlobArtifactSettings { } export interface AzureBlobArtifactStorage { + isReady(signal: AbortSignal): Promise read( key: string, ): Promise< diff --git a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts index 57644ab7..5c602aa9 100644 --- a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts +++ b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts @@ -16,6 +16,15 @@ export async function createFilesystemArtifactStorage( throw new Error("Artifact directory is unavailable.") } return { + async isReady(signal) { + signal.throwIfAborted() + try { + await withArtifactDirectory(directory, () => Promise.resolve(undefined)) + return true + } catch { + return false + } + }, async read(key) { if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(key)) return { status: "unavailable" } try { diff --git a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts index 90c84f19..12e7c38c 100644 --- a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts +++ b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts @@ -1,4 +1,5 @@ export interface FilesystemArtifactStorage { + isReady(signal: AbortSignal): Promise read( key: string, ): Promise< diff --git a/hub/server/scripts/grant-runtime-database-access/index.ts b/hub/server/scripts/grant-runtime-database-access/index.ts new file mode 100644 index 00000000..b290323f --- /dev/null +++ b/hub/server/scripts/grant-runtime-database-access/index.ts @@ -0,0 +1,46 @@ +import "dotenv/config" +import { resolve } from "node:path" +import { pathToFileURL } from "node:url" +import pg from "pg" + +import { logger } from "../../integrations/pino/index.ts" + +export async function grantRuntimeDatabaseAccess(databaseUrl: string) { + if (!databaseUrl) throw new Error("Runtime grants require DATABASE_URL.") + const client = new pg.Client({ connectionString: databaseUrl, connectionTimeoutMillis: 5_000 }) + try { + await client.connect() + await client.query("BEGIN") + const principal = await client.query<{ name: string }>("SELECT current_user AS name") + if (principal.rows[0]!.name !== "replica_migration") { + throw new Error("Runtime grants require the replica_migration principal.") + } + // BullMQ lazily creates per-queue sequences; initialize the two allowed queues as their owner. + await client.query("SELECT bullmq.next_job_id('on-demand'), bullmq.next_job_id('scheduled')") + await client.query( + "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public, bullmq TO replica_runtime", + ) + await client.query( + "GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public, bullmq TO replica_runtime", + ) + await client.query("REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public, bullmq FROM PUBLIC") + await client.query("GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public, bullmq TO replica_runtime") + await client.query( + "REVOKE ALL ON public._prisma_migrations, bullmq.migration FROM replica_runtime", + ) + await client.query("GRANT SELECT ON bullmq.migration TO replica_runtime") + await client.query("COMMIT") + } finally { + await client.end() + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + await grantRuntimeDatabaseAccess(process.env.DATABASE_URL ?? "") + logger.info("Runtime database access granted") + } catch { + logger.fatal("Runtime database grants failed; check migration ownership and the runtime role") + process.exitCode = 1 + } +} diff --git a/hub/server/scripts/provision-database-access/index.ts b/hub/server/scripts/provision-database-access/index.ts new file mode 100644 index 00000000..964be248 --- /dev/null +++ b/hub/server/scripts/provision-database-access/index.ts @@ -0,0 +1,73 @@ +import "dotenv/config" +import { resolve } from "node:path" +import { pathToFileURL } from "node:url" +import pg from "pg" + +import { logger } from "../../integrations/pino/index.ts" + +export async function provisionDatabaseAccess({ + databaseUrl, + migrationPassword, + runtimePassword, +}: { + databaseUrl: string + migrationPassword: string + runtimePassword: string +}) { + if (!databaseUrl || migrationPassword.length < 16 || runtimePassword.length < 16) { + throw new Error( + "Database provisioning requires a URL and two passwords of at least 16 characters.", + ) + } + if (migrationPassword === runtimePassword) { + throw new Error("Database provisioning requires different migration and runtime passwords.") + } + const client = new pg.Client({ connectionString: databaseUrl, connectionTimeoutMillis: 5_000 }) + try { + await client.connect() + await client.query("BEGIN") + const existing = await client.query<{ populated: boolean }>( + "SELECT EXISTS (SELECT 1 FROM pg_class WHERE relnamespace = 'public'::regnamespace) AS populated", + ) + if (existing.rows[0]!.populated) { + throw new Error("Database access provisioning requires an empty public schema.") + } + const database = await client.query<{ name: string }>("SELECT current_database() AS name") + const name = pg.escapeIdentifier(database.rows[0]!.name) + // Existing roles are deliberately rejected rather than adopting unknown grants or ownership. + await client.query( + `CREATE ROLE replica_migration LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD ${pg.escapeLiteral(migrationPassword)}`, + ) + await client.query( + `CREATE ROLE replica_runtime LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD ${pg.escapeLiteral(runtimePassword)}`, + ) + await client.query("GRANT replica_migration TO CURRENT_USER") + await client.query(`REVOKE CREATE, TEMPORARY ON DATABASE ${name} FROM PUBLIC`) + await client.query( + `GRANT CONNECT, TEMPORARY ON DATABASE ${name} TO replica_migration, replica_runtime`, + ) + await client.query(`GRANT CREATE ON DATABASE ${name} TO replica_migration`) + await client.query("ALTER SCHEMA public OWNER TO replica_migration") + await client.query("REVOKE ALL ON SCHEMA public FROM PUBLIC") + await client.query("CREATE SCHEMA bullmq AUTHORIZATION replica_migration") + await client.query("GRANT USAGE ON SCHEMA public, bullmq TO replica_runtime") + await client.query("COMMIT") + } finally { + // Closing also rolls back failed provisioning, including role creation. + await client.end() + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + await provisionDatabaseAccess({ + databaseUrl: process.env.DATABASE_URL ?? "", + migrationPassword: process.env.HUB_MIGRATION_PASSWORD ?? "", + runtimePassword: process.env.HUB_RUNTIME_PASSWORD ?? "", + }) + logger.info("Database access provisioned") + } catch { + logger.fatal("Database access provisioning failed; check initial database ownership and roles") + process.exitCode = 1 + } +} diff --git a/hub/server/scripts/setup/index.ts b/hub/server/scripts/setup/index.ts index 673d132a..4d76cfd7 100644 --- a/hub/server/scripts/setup/index.ts +++ b/hub/server/scripts/setup/index.ts @@ -9,6 +9,13 @@ import { withSetupLock } from "../../integrations/postgres/with-setup-lock/index export async function setup() { const databaseUrl = process.env.DATABASE_URL if (!databaseUrl) throw new Error("Setup requires DATABASE_URL.") + if ( + process.env.DATABASE_RUNTIME_ROLE !== undefined && + (process.env.DATABASE_RUNTIME_ROLE !== "replica_runtime" || + new URL(databaseUrl).username !== "replica_migration") + ) { + throw new Error("Separate roles require replica_runtime and a replica_migration database URL.") + } const controller = new AbortController() function stop() { controller.abort() @@ -22,13 +29,21 @@ export async function setup() { run: async (signal) => { const extension = import.meta.url.endsWith(".js") ? "js" : "ts" const jobScript = new URL(`../setup-jobs/index.${extension}`, import.meta.url) + const grantScript = new URL( + `../grant-runtime-database-access/index.${extension}`, + import.meta.url, + ) const steps = [ [resolve("node_modules/prisma/build/index.js"), "migrate", "deploy"], [jobScript.pathname], + ...(process.env.DATABASE_RUNTIME_ROLE ? [[grantScript.pathname]] : []), ] for (const [index, args] of steps.entries()) { signal.throwIfAborted() - logger.info({ step: index === 0 ? "prisma" : "jobs" }, "Hub setup step started") + logger.info( + { step: ["prisma", "jobs", "runtime-grants"][index] }, + "Hub setup step started", + ) // Migration tools can include database URLs in diagnostics; keep their output private. const child = spawn(process.execPath, args, { stdio: "ignore" }) let timeout: ReturnType | undefined diff --git a/hub/server/start-server-processes/index.test.ts b/hub/server/start-server-processes/index.test.ts index f243e23d..d28ad2d0 100644 --- a/hub/server/start-server-processes/index.test.ts +++ b/hub/server/start-server-processes/index.test.ts @@ -13,7 +13,7 @@ vi.mock("../integrations/postgres/open-database", () => ({ openDatabase: vi.fn() vi.mock("../run-api", () => ({ runApi: vi.fn() })) vi.mock("../workers/run-job-worker", () => ({ runJobWorker: vi.fn() })) -function arrangeRuntime() { +function arrangeRuntime(isStorageReady?: () => Promise) { const query = vi.fn().mockResolvedValue({ command: "SELECT", rowCount: 1, @@ -45,6 +45,7 @@ function arrangeRuntime() { logger: createLogger({ level: "silent" }), onDemandConcurrency: 2, scheduledConcurrency: 1, + isStorageReady, }) void running.catch(() => {}) function stop() { @@ -59,6 +60,24 @@ function arrangeRuntime() { } describe("startServerProcesses()", () => { + it("requires configured storage as well as initialized workers and a live database", async () => { + const isStorageReady = vi.fn().mockResolvedValue(false) + const { query } = arrangeRuntime(isStorageReady) + await vi.waitFor(() => expect(runApi).toHaveBeenCalledTimes(1)) + const api = vi.mocked(runApi).mock.calls[0]![0] + const workers = vi.mocked(runJobWorker).mock.calls.map(([options]) => options) + expect(await api.isReady?.()).toBe(false) + expect(isStorageReady).not.toHaveBeenCalled() + workers.forEach((worker) => worker.onReady?.()) + + expect(await api.isReady?.()).toBe(false) + isStorageReady.mockResolvedValue(true) + expect(await api.isReady?.()).toBe(true) + query.mockRejectedValueOnce(new Error("Database unavailable")) + await expect(api.isReady?.()).rejects.toThrow("Database unavailable") + expect(isStorageReady).toHaveBeenCalledTimes(2) + }) + it("reports readiness only after both workers and the database are ready", async () => { const { query, end, running, stop } = arrangeRuntime() await vi.waitFor(() => expect(runApi).toHaveBeenCalledTimes(1)) diff --git a/hub/server/start-server-processes/index.ts b/hub/server/start-server-processes/index.ts index 196c4c6b..840289f0 100644 --- a/hub/server/start-server-processes/index.ts +++ b/hub/server/start-server-processes/index.ts @@ -14,6 +14,7 @@ export async function startServerProcesses(options: { logger: Logger onDemandConcurrency: number scheduledConcurrency: number + isStorageReady?: () => Promise }) { const controller = new AbortController() function stop() { @@ -52,6 +53,7 @@ export async function startServerProcesses(options: { isReady: async () => { if (controller.signal.aborted || ready.size !== 2) return false await database.query("SELECT 1") + if (options.isStorageReady && !(await options.isStorageReady())) return false return true }, }), diff --git a/infrastructure/README.md b/infrastructure/README.md index b8555f99..71f7fe42 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -11,16 +11,16 @@ Each installation has its own resource group tagged with its application, instal - A virtual network, delegated Container Apps and PostgreSQL subnets, and private PostgreSQL DNS. - PostgreSQL Flexible Server, the `replica` database, enforced secure transport, and configurable point-in-time recovery retention. - An Azure Storage account with private `artifacts` and `releases` containers, Blob versioning, and 14-day Blob and container soft deletion. Shared keys and anonymous access are disabled. The Blob endpoint is public-network reachable but requires Entra authorization; this is not a private-endpoint installation. -- One user-assigned workload identity with `Storage Blob Data Contributor` on **only** the artifacts container. An optional existing ACR receives an `AcrPull` assignment for that identity. No registry is created or owned. +- Separate Hub and setup user-assigned identities. Only Hub receives `Storage Blob Data Contributor` on the artifacts container. Each identity can read only its referenced Key Vault secrets and, when configured, pull from an existing ACR. No vault or registry is created or owned. - Log Analytics with 30-day retention, a Container Apps environment, a manual setup job, and, after successful setup, one combined Hub app. The app has external HTTPS ingress and minimum and maximum one replica **per revision**. Multiple-revision mode permits explicit deactivation before releases. Azure's single-revision mode deliberately overlaps old and new processes while readiness is checked, so it is unsuitable for this maintenance-window strategy. Do not activate old revisions alongside the combined worker. This is not a high-availability or zero-downtime installation. -The runtime receives `RUNTIME_MODE=combined`, `HOST=0.0.0.0`, `PORT=3001`, `CLIENT_DIRECTORY=/app/dist/client`, `SHUTDOWN_TIMEOUT_MS=30000`, and `NODE_ENV=production`. Artifacts use `ARTIFACT_STORAGE_PROVIDER=azure`, the provisioned `AZURE_STORAGE_ACCOUNT`, `AZURE_STORAGE_CONTAINER=artifacts`, and the workload's `AZURE_CLIENT_ID`. `DATABASE_URL` is a Container Apps secret using `sslmode=verify-full`. Additional `runtime_secrets` are explicit, sensitive environment-name/value inputs; none are Terraform outputs. +The runtime receives `RUNTIME_MODE=combined`, `HOST=0.0.0.0`, `PORT=3001`, `CLIENT_DIRECTORY=/app/dist/client`, `SHUTDOWN_TIMEOUT_MS=30000`, and `NODE_ENV=production`. Artifacts use `ARTIFACT_STORAGE_PROVIDER=azure`, the provisioned `AZURE_STORAGE_ACCOUNT`, `AZURE_STORAGE_CONTAINER=artifacts`, and Hub's `AZURE_CLIENT_ID`. `DATABASE_URL` references a Key Vault secret for the `replica_runtime` database role with `sslmode=verify-full`. Optional `runtime_secret_references` map environment names to vault resource IDs and HTTPS secret URLs. Container Apps resolves these references using Hub's identity; Hub does not implement a separate secret-resolution framework. -The setup job uses the same digest, `RUNTIME_MODE=setup`, and command `node dist/server/scripts/setup/index.js`. It has one replica, no automatic retries, and a 30-minute timeout. Hub's readiness probe is `/api/ready`; startup and liveness use `/api/health`. Azure allows 60 seconds for termination, covering the 30-second application drain and up to five seconds of telemetry flushing with platform margin. +The setup job uses the same digest and command `node dist/server/scripts/setup/index.js`. Its only environment settings are `NODE_ENV=production`, `RUNTIME_MODE=setup`, `DATABASE_RUNTIME_ROLE=replica_runtime`, and its own Key Vault-backed `DATABASE_URL` for `replica_migration`. It receives no artifact configuration or additional runtime secrets. It has one replica, no automatic retries, and a 30-minute timeout. Hub's readiness probe is `/api/ready`; startup and liveness use `/api/health`. Azure allows 60 seconds for termination, covering the 30-second application drain and up to five seconds of telemetry flushing with platform margin. -The reference uses the PostgreSQL administrator login for both setup and runtime so AzureRM alone can provision it without another database provider or remote SQL provisioner. Operators requiring separate migration/runtime roles must provision and test those grants before adapting this reference. Managed identity is used for Blob and ACR, not database authentication. +Neither workload receives the PostgreSQL administrator credential. A separate, one-time operator script provisions the migration and runtime roles over the private database network. Setup owns schema changes and refreshes the runtime's data permissions after migrations. See [database access](../hub/docs/database.md#deployment-database-roles). Managed identity is used for Blob, Key Vault, and ACR, not database authentication. ## Prepare access and protected state @@ -32,13 +32,15 @@ For automation, configure a dedicated deployment identity with a federated workl Set `ARM_CLIENT_ID`, `ARM_TENANT_ID`, `ARM_SUBSCRIPTION_ID`, and `ARM_USE_OIDC=true` in that authorized runner. Terraform's AzureRM backend supports GitHub's OIDC request environment; other runners must supply their supported OIDC token source. For local authorized operation, use Azure CLI sign-in and set `use_oidc=false` in your private backend configuration instead. Do not copy tokens into Terraform inputs. -The deployment identity needs resource creation/update rights for the installation, role-assignment rights for the container-scoped Blob grant and optional registry grant, and Blob data-plane rights on the installation's account for the lock blob. Resource `Contributor` alone does not grant Blob data access or role assignment. An administrator must authorize the initial resource-group creation and inherited Blob data access for bootstrap, then narrow ongoing deployment grants to that installation resource group, the existing registry, and the backend container. Precreated/imported infrastructure needs a separately reviewed initial plan rather than the empty-state bootstrap command. Do not grant the workload identity deployment or state access. +The deployment identity needs resource creation/update rights for the installation, role-assignment rights for the artifact container, referenced Key Vault secrets and optional registry, and Blob data-plane rights on the installation's account for the lock blob. Resource `Contributor` alone does not grant data access or role assignment. An administrator must authorize initial resource-group creation and inherited Blob data access for bootstrap, then narrow ongoing deployment grants to the installation resource group, referenced secrets, existing registry, and backend container. Precreated/imported infrastructure needs a separately reviewed initial plan rather than the empty-state bootstrap command. Do not grant either workload identity deployment or state access. Register `Microsoft.App`, `Microsoft.OperationalInsights`, `Microsoft.Network`, `Microsoft.Storage`, `Microsoft.DBforPostgreSQL`, and `Microsoft.ManagedIdentity` beforehand. The provider does not register services across the subscription. Allow time for newly assigned RBAC roles to propagate before setup. Copy [`backend.hcl.example`](azure/backend.hcl.example) to `infrastructure/azure/backend.hcl` and [`installation.tfvars.example`](azure/installation.tfvars.example) to `infrastructure/azure/installation.tfvars`. Both real files are ignored. Replace every placeholder, choose a globally unique name, and resolve an existing image to its real `registry/repository@sha256:...` digest. Use either a public image or an existing ACR; private non-ACR registry credentials and ACR's ABAC repository-permissions mode are not covered by this reference. -Supply `postgres_admin_password` from a secret manager as `TF_VAR_postgres_admin_password`, or through the ignored variables file with mode `0600`. Supply optional `TF_VAR_runtime_secrets` as a JSON object of uppercase environment names to secret values. Never enable shell tracing while injecting secrets. Terraform's sensitive marking redacts normal CLI output but **does not encrypt state or saved plans**. Protect backend access, runner logs, environment variables, and local `.terraform/`; never publish a JSON plan or upload it as a CI artifact. +Prepare an existing RBAC-enabled Key Vault reachable by the Container Apps environment. Supply `runtime_database_secret` and `setup_database_secret`, each with the vault's resource ID and a distinct HTTPS secret URL. Before bootstrap, store separate database URLs for `replica_runtime` and `replica_migration` with independently generated passwords, the planned private PostgreSQL hostname, database `replica`, and `sslmode=verify-full`. Percent-encode URL credentials. The identities receive secret-scoped `Key Vault Secrets User` grants, so sharing a vault does not give setup access to runtime secrets. Additional runtime references must not point at the migration credential. The vault, secret values, rotation and retention remain operator-owned; Terraform never reads those values. Prefer versioned URLs for deliberate rotation through a serialized release. + +Supply `postgres_admin_password` from a secret manager as `TF_VAR_postgres_admin_password`, or through the ignored variables file with mode `0600`. This provisioning password still enters Terraform state; replacing application secret values with references does not remove it or erase historical secret values in older state versions. Never enable shell tracing while injecting secrets. Terraform's sensitive marking redacts normal CLI output but **does not encrypt state or saved plans**. Protect backend access, runner logs, environment variables, and local `.terraform/`; never publish a JSON plan or upload it as a CI artifact. ## Bootstrap and release @@ -54,7 +56,9 @@ node infrastructure/azure/deploy-hub/index.ts bootstrap \ --image "$HUB_IMAGE" ``` -Bootstrap creates infrastructure and the manual setup job with `deploy_hub=false`. No combined app exists yet. Review the configuration and a read-only Terraform plan before authorizing this command: the script prints its plans and applies them without an interactive prompt after checking for deletes/replacements. Bootstrap refuses any nonempty state so it cannot race a release on an existing installation. If initial provisioning failed partway, inspect the partial state and recover from a separately reviewed plan with `deploy_hub=false`; never use this recovery path after an app has been created. +Bootstrap creates infrastructure and the manual setup job with `deploy_hub=false`. No combined app exists yet. Review the configuration and a read-only Terraform plan before authorizing this command: the script applies saved plans without an interactive prompt after checking for deletes/replacements. Command output is kept private because tool diagnostics can contain credentials. Bootstrap refuses any nonempty state so it cannot race a release on an existing installation. If initial provisioning failed partway, inspect the partial state and recover from a separately reviewed plan with `deploy_hub=false`; never use this recovery path after an app has been created. + +Before the first release, use an authorized private-network host with Hub's dependencies installed to run `node server/scripts/provision-database-access/index.ts` from `hub/`. Inject the administrator `DATABASE_URL`, `HUB_MIGRATION_PASSWORD`, and `HUB_RUNTIME_PASSWORD` directly from the secret manager without shell tracing or command-line values. Use the same two passwords stored in the workload URL secrets. This first-installation script creates the roles and schemas transactionally and rejects existing roles, public tables, or a pre-existing BullMQ schema; it is not an upgrade or password-rotation command. Remove these environment variables afterward. Do not inject administrator credentials into the setup job. Existing installations require a reviewed ownership/credential migration while Hub is stopped, not rerunning this fresh-database script. Use the same release command for the first installation, subsequent image releases, and infrastructure or secret changes: @@ -70,7 +74,7 @@ node infrastructure/azure/deploy-hub/index.ts release \ The operator must approve the maintenance window and inspect intended configuration changes before invoking the command. A read-only plan can be reviewed with `terraform -chdir=infrastructure/azure plan -var-file=installation.tfvars -var="image=$HUB_IMAGE" -var=deploy_hub=true` after initializing the configured backend. Do not apply that preflight plan: setup must run first. The script then: 1. Checks the explicit subscription, Terraform state's installation, and resource group ownership. -2. Acquires an infinite lease on `releases/deployment.lock`. A competing release fails before stopping Hub. This serializes the whole release, not just individual Terraform operations; setup's database advisory lock remains a separate defense. +2. Acquires an infinite lease on `releases/deployment.lock`. A competing release fails before stopping Hub. The orchestrator owns serialization of the whole release, not just individual Terraform operations. Setup's advisory lock rejects concurrent setup while its database connection remains healthy; it cannot protect the child-termination interval after that connection is lost. 3. Checks that no setup execution is running, deactivates every active revision, and waits for **all** old revision replicas to disappear before setup. Users cannot use Hub during this window. 4. Uses a deliberate targeted Terraform plan/apply for the setup job and its infrastructure dependencies. It cannot update the app at this point. 5. Starts that job, checks that exact execution until it succeeds, and stops on failure or timeout. @@ -84,7 +88,7 @@ Plans reject resource deletion and replacement during normal releases. A databas Inspect the named setup execution and Container Apps/Log Analytics logs using authorized Azure access. Check that the release process has exited, every setup execution is terminal, and no old revision still has replicas. After a successful but partially deployed setup, decide whether to retry the same digest or use a compatible recovery image. Prisma migrations, queue migrations, and schedule registration are not a single atomic transaction. -Only after those checks may an operator break the retained lease: +Confirm that the prior migration database sessions have also stopped before retrying, including after a lost setup-lock connection. Only after those checks may an operator break the retained lease: ```sh az storage blob lease break --account-name YOUR_ACCOUNT \ @@ -143,6 +147,6 @@ terraform test node --test deploy-hub/index.test.ts ``` -Terraform tests use AzureRM mocks, not an Azure subscription. Orchestration tests replace command execution and HTTPS requests, proving ordering and failure gates rather than Azure behavior. Before production, an authorized tenant exercise must verify provisioning, private PostgreSQL connectivity and TLS, identity-based Blob/ACR access after RBAC propagation, failed setup blocking rollout, maintenance shutdown and revision behavior, HTTPS/readiness, persistence across replacement, custom TLS when configured, backup restoration, and owned-resource teardown. No real-cloud result is implied by these local checks. +Terraform tests use AzureRM mocks, not an Azure subscription. Orchestration tests replace command execution and HTTPS requests, proving ordering and failure gates rather than Azure behavior. Readiness includes a bounded, non-writing container-properties request through Hub's managed identity; it detects unavailable storage but does not prove write or delete authorization. Worker readiness remains initialization-only. Before production, an authorized tenant exercise must verify provisioning, private PostgreSQL connectivity and TLS, separate database roles, Key Vault resolution, identity-based Blob/ACR access after RBAC propagation, failed setup blocking rollout, maintenance shutdown and revision behavior, HTTPS/readiness, persistence across replacement, custom TLS when configured, backup restoration, and owned-resource teardown. No real-cloud result is implied by these local checks. The release command and its tests are TypeScript running directly in Node.js. Terraform's own tests use `.tftest.hcl`; neither requires Python. From the repository root, `npm run typecheck --prefix scripts` checks the TypeScript operational scripts as well. diff --git a/infrastructure/azure/compute.tf b/infrastructure/azure/compute.tf index 12cd33ca..d88ae1c5 100644 --- a/infrastructure/azure/compute.tf +++ b/infrastructure/azure/compute.tf @@ -15,28 +15,21 @@ resource "azurerm_container_app_job" "setup" { identity { type = "UserAssigned" - identity_ids = [azurerm_user_assigned_identity.hub.id] + identity_ids = [azurerm_user_assigned_identity.setup.id] } dynamic "registry" { for_each = var.acr_id == null ? [] : [var.acr_id] content { server = split("/", var.image)[0] - identity = azurerm_user_assigned_identity.hub.id + identity = azurerm_user_assigned_identity.setup.id } } secret { - name = "database-url" - value = local.database_url - } - - dynamic "secret" { - for_each = local.secret_names - content { - name = "runtime-${substr(sha256(secret.value), 0, 12)}" - value = var.runtime_secrets[secret.value] - } + name = "database-url" + key_vault_secret_id = var.setup_database_secret.secret_id + identity = azurerm_user_assigned_identity.setup.id } template { @@ -48,7 +41,11 @@ resource "azurerm_container_app_job" "setup" { command = ["node", "dist/server/scripts/setup/index.js"] dynamic "env" { - for_each = merge(local.runtime_environment, { RUNTIME_MODE = "setup" }) + for_each = { + NODE_ENV = "production" + RUNTIME_MODE = "setup" + DATABASE_RUNTIME_ROLE = "replica_runtime" + } content { name = env.key value = env.value @@ -59,22 +56,14 @@ resource "azurerm_container_app_job" "setup" { name = "DATABASE_URL" secret_name = "database-url" } - - dynamic "env" { - for_each = local.secret_names - content { - name = env.value - secret_name = "runtime-${substr(sha256(env.value), 0, 12)}" - } - } } } depends_on = [ azurerm_postgresql_flexible_server_database.hub, azurerm_postgresql_flexible_server_configuration.tls, - azurerm_role_assignment.artifacts, - azurerm_role_assignment.image_pull, + azurerm_role_assignment.setup_secrets, + azurerm_role_assignment.setup_image_pull, ] } @@ -101,15 +90,17 @@ resource "azurerm_container_app" "hub" { } secret { - name = "database-url" - value = local.database_url + name = "database-url" + key_vault_secret_id = var.runtime_database_secret.secret_id + identity = azurerm_user_assigned_identity.hub.id } dynamic "secret" { - for_each = local.secret_names + for_each = var.runtime_secret_references content { - name = "runtime-${substr(sha256(secret.value), 0, 12)}" - value = var.runtime_secrets[secret.value] + name = "runtime-${substr(sha256(secret.key), 0, 12)}" + key_vault_secret_id = secret.value.secret_id + identity = azurerm_user_assigned_identity.hub.id } } @@ -149,10 +140,10 @@ resource "azurerm_container_app" "hub" { } dynamic "env" { - for_each = local.secret_names + for_each = var.runtime_secret_references content { - name = env.value - secret_name = "runtime-${substr(sha256(env.value), 0, 12)}" + name = env.key + secret_name = "runtime-${substr(sha256(env.key), 0, 12)}" } } @@ -185,7 +176,12 @@ resource "azurerm_container_app" "hub" { prevent_destroy = true } - depends_on = [azurerm_container_app_job.setup] + depends_on = [ + azurerm_container_app_job.setup, + azurerm_role_assignment.artifacts, + azurerm_role_assignment.image_pull, + azurerm_role_assignment.runtime_secrets, + ] } resource "azurerm_container_app_custom_domain" "hub" { diff --git a/infrastructure/azure/deploy-hub/index.test.ts b/infrastructure/azure/deploy-hub/index.test.ts index c3194989..d3206551 100644 --- a/infrastructure/azure/deploy-hub/index.test.ts +++ b/infrastructure/azure/deploy-hub/index.test.ts @@ -13,6 +13,12 @@ function installationScenario({ appCreated = true, setupStatus = "Succeeded", lockAvailable = true, + releaseError, +}: { + appCreated?: boolean + setupStatus?: string + lockAvailable?: boolean + releaseError?: Error } = {}) { const installation = { name: "test-hub", @@ -28,11 +34,17 @@ function installationScenario({ url: "https://hub.example.com", } const events: string[][] = [] + const commandOptions: childProcess.ExecFileSyncOptions[] = [] const replicaReads = new Map() let targeted = false - mock.method(childProcess, "execFileSync", (file: string, args: readonly string[]) => { + function execute( + file: string, + args: readonly string[], + options: childProcess.ExecFileSyncOptions, + ) { const command: string[] = [file, ...args] events.push(command) + commandOptions.push(options) if (file === "terraform") { switch (args[1]) { case "output": @@ -80,6 +92,7 @@ function installationScenario({ if (!lockAvailable) throw new Error("Release lease is already held") return JSON.stringify("lease-id") } + if (args.includes("release") && releaseError) throw releaseError return "null" case "containerapp revision list": return JSON.stringify([ @@ -102,12 +115,16 @@ function installationScenario({ default: if (args.slice(0, 2).join(" ") === "containerapp show") { return JSON.stringify({ - properties: { latestRevisionName: "new", latestReadyRevisionName: "new" }, + properties: { + latestRevisionName: "new", + latestReadyRevisionName: "new", + }, }) } throw new Error(`Unexpected Azure command: ${command.join(" ")}`) } - }) + } + mock.method(childProcess, "execFileSync", execute) mock.method(timers, "setTimeout", async () => undefined) mock.method(globalThis, "fetch", async (url: Parameters[0]) => { events.push(["HTTPS", String(url)]) @@ -125,12 +142,12 @@ function installationScenario({ `example.azurecr.io/hub@sha256:${"a".repeat(64)}`, "--accept-maintenance", ] - return { args, events, replicaReads } + return { args, events, replicaReads, commandOptions } } describe("deployHub()", () => { it("drains all old revisions and completes setup before rollout, readiness, and lease release", async () => { - const { args, events, replicaReads } = installationScenario() + const { args, events, replicaReads, commandOptions } = installationScenario() await deployHub(args) @@ -171,6 +188,12 @@ describe("deployHub()", () => { assert.deepEqual(events[ready], ["HTTPS", "https://hub.example.com/api/ready"]) assert.ok(ready > rolloutApplied) assert.ok(release > ready) + assert.ok( + commandOptions.every( + ({ stdio }) => + stdio === "pipe" || (Array.isArray(stdio) && stdio[1] === "pipe" && stdio[2] === "pipe"), + ), + ) }) it("leaves Hub stopped and retains its lease when setup fails", async () => { @@ -208,7 +231,7 @@ describe("deployHub()", () => { it("does not stop Hub or run setup while another release holds the lease", async () => { const { args, events } = installationScenario({ lockAvailable: false }) - await assert.rejects(deployHub(args), /Release lease is already held/) + await assert.rejects(deployHub(args), /Azure command failed/) assert.ok( events.some((event) => event.slice(0, 5).join(" ") === "az storage blob lease acquire"), @@ -218,6 +241,35 @@ describe("deployHub()", () => { assert.ok(!events.some((event) => event[0] === "terraform" && event[2] === "apply")) }) + it("redacts child diagnostics and retains the lock warning when lease release fails", async () => { + const leaseId = "lease-id" + const privateOutput = `private-credential ${leaseId}` + const releaseError = Object.assign(new Error(`az --lease-id ${privateOutput}`), { + stdout: Buffer.from(privateOutput), + stderr: Buffer.from(privateOutput), + cause: new Error(privateOutput), + }) + const { args, events } = installationScenario({ releaseError }) + const output: unknown[][] = [] + mock.method(console, "error", (...args: unknown[]) => output.push(args)) + mock.method(console, "log", (...args: unknown[]) => output.push(args)) + + await assert.rejects(deployHub(args), (error: unknown) => { + assert.ok(error instanceof Error) + assert.match(error.message, /Azure command failed/) + assert.equal(error.cause, undefined) + const rendered = `${error.stack} ${JSON.stringify(error)} ${JSON.stringify(output)}` + assert.ok(!rendered.includes(leaseId)) + assert.ok(!rendered.includes("private-credential")) + return true + }) + + assert.ok(events.some((event) => event[0] === "HTTPS")) + assert.ok(events.some((event) => event.includes("--lease-id") && event.includes(leaseId))) + assert.match(JSON.stringify(output), /lease is retained/) + assert.ok(!JSON.stringify(output).includes("maintenance window is complete")) + }) + it("refuses bootstrap when Terraform state already contains resources", async () => { const { args, events } = installationScenario() args[0] = "bootstrap" diff --git a/infrastructure/azure/deploy-hub/index.ts b/infrastructure/azure/deploy-hub/index.ts index 32ff2d68..82eb168a 100644 --- a/infrastructure/azure/deploy-hub/index.ts +++ b/infrastructure/azure/deploy-hub/index.ts @@ -57,32 +57,45 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise let lease: string | undefined function command(executable: string, arguments_: string[], capture = false): string { - return ( - childProcess - .execFileSync(executable, arguments_, { - encoding: "utf8", - stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit", - maxBuffer: 32 * 1024 * 1024, - }) - ?.trim() ?? "" - ) + try { + const output = childProcess.execFileSync(executable, arguments_, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 32 * 1024 * 1024, + }) + return capture ? (output?.trim() ?? "") : "" + } catch { + // Child errors and output can contain credentials and release lease arguments. + throw new Error( + executable === "terraform" + ? "Terraform command failed; inspect the installation through authorized tooling." + : "Azure command failed; inspect the installation through authorized tooling.", + ) + } } function terraform(arguments_: string[], capture = false): string { return command("terraform", [`-chdir=${root}`, ...arguments_], capture) } + function parseOutput(output: string): T { + try { + return JSON.parse(output) as T + } catch { + throw new Error("Deployment tooling returned an invalid response.") + } + } function azure(...arguments_: string[]): T { - return JSON.parse( + return parseOutput( command( "az", [...arguments_, "--subscription", subscriptionId, "--only-show-errors", "--output", "json"], true, ), - ) as T + ) } function readInstallation(): Installation { - const installation = JSON.parse( + const installation = parseOutput( terraform(["output", "-json", "installation"], true), - ) as Installation + ) if (installation.subscription_id !== subscription) { throw new Error("The selected subscription does not match this Terraform state.") } @@ -141,7 +154,7 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise if (target) arguments_.push(`-target=${target}`) if (destroy) arguments_.push("-destroy") terraform(arguments_) - const planned = JSON.parse(terraform(["show", "-json", planPath], true)) as DeploymentPlan + const planned = parseOutput(terraform(["show", "-json", planPath], true)) if (!destroy) { if (planned.resource_changes?.some(({ change }) => change.actions.includes("delete"))) { throw new Error( diff --git a/infrastructure/azure/installation.tfvars.example b/infrastructure/azure/installation.tfvars.example index 1a6034d0..3a24abc3 100644 --- a/infrastructure/azure/installation.tfvars.example +++ b/infrastructure/azure/installation.tfvars.example @@ -4,8 +4,23 @@ location = "eastus2" owner = "your-team" image = "ghcr.io/your-org/replica-hub@sha256:REPLACE_WITH_REAL_DIGEST" -# Supply postgres_admin_password and runtime_secrets through your secret store, -# TF_VAR_ environment variables, or an ignored, access-restricted tfvars file. +runtime_database_secret = { + key_vault_id = "/subscriptions/YOUR_SUBSCRIPTION/resourceGroups/YOUR_SECRETS_GROUP/providers/Microsoft.KeyVault/vaults/YOUR_VAULT" + secret_id = "https://YOUR-VAULT.vault.azure.net/secrets/hub-runtime-database" +} +setup_database_secret = { + key_vault_id = "/subscriptions/YOUR_SUBSCRIPTION/resourceGroups/YOUR_SECRETS_GROUP/providers/Microsoft.KeyVault/vaults/YOUR_VAULT" + secret_id = "https://YOUR-VAULT.vault.azure.net/secrets/hub-migration-database" +} + +# Supply postgres_admin_password through a secret manager as TF_VAR_postgres_admin_password. +# Application secrets stay in Key Vault. Optional references, never values: +# runtime_secret_references = { +# EXAMPLE_API_TOKEN = { +# key_vault_id = "/subscriptions/.../resourceGroups/.../providers/Microsoft.KeyVault/vaults/..." +# secret_id = "https://YOUR-VAULT.vault.azure.net/secrets/example-api-token" +# } +# } # acr_id = "/subscriptions/.../resourceGroups/.../providers/Microsoft.ContainerRegistry/registries/..." # custom_hostname = "hub.example.com" # certificate_id = "/subscriptions/.../resourceGroups/.../providers/Microsoft.App/managedEnvironments/.../certificates/..." diff --git a/infrastructure/azure/main.tf b/infrastructure/azure/main.tf index fb48b891..3b55b5e4 100644 --- a/infrastructure/azure/main.tf +++ b/infrastructure/azure/main.tf @@ -5,7 +5,6 @@ locals { owner = var.owner managed_by = "terraform" } - database_url = "postgresql://replicaadmin:${replace(urlencode(var.postgres_admin_password), "+", "%20")}@${azurerm_postgresql_flexible_server.database.fqdn}:5432/replica?sslmode=verify-full" runtime_environment = { NODE_ENV = "production" RUNTIME_MODE = "combined" @@ -18,7 +17,10 @@ locals { AZURE_STORAGE_CONTAINER = azurerm_storage_container.artifacts.name AZURE_CLIENT_ID = azurerm_user_assigned_identity.hub.client_id } - secret_names = nonsensitive(toset(keys(var.runtime_secrets))) + runtime_secret_scopes = toset([ + for secret in concat([var.runtime_database_secret], values(var.runtime_secret_references)) : + "${secret.key_vault_id}/secrets/${split("/", secret.secret_id)[4]}" + ]) } resource "azurerm_resource_group" "hub" { @@ -159,6 +161,26 @@ resource "azurerm_user_assigned_identity" "hub" { tags = local.tags } +resource "azurerm_user_assigned_identity" "setup" { + name = "${var.name}-setup" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + tags = local.tags +} + +resource "azurerm_role_assignment" "runtime_secrets" { + for_each = local.runtime_secret_scopes + scope = each.value + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_role_assignment" "setup_secrets" { + scope = "${var.setup_database_secret.key_vault_id}/secrets/${split("/", var.setup_database_secret.secret_id)[4]}" + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_user_assigned_identity.setup.principal_id +} + resource "azurerm_role_assignment" "artifacts" { scope = "${azurerm_storage_account.artifacts.id}/blobServices/default/containers/${azurerm_storage_container.artifacts.name}" role_definition_name = "Storage Blob Data Contributor" @@ -172,6 +194,13 @@ resource "azurerm_role_assignment" "image_pull" { principal_id = azurerm_user_assigned_identity.hub.principal_id } +resource "azurerm_role_assignment" "setup_image_pull" { + count = var.acr_id == null ? 0 : 1 + scope = var.acr_id + role_definition_name = "AcrPull" + principal_id = azurerm_user_assigned_identity.setup.principal_id +} + resource "azurerm_log_analytics_workspace" "hub" { name = "${var.name}-logs" resource_group_name = azurerm_resource_group.hub.name diff --git a/infrastructure/azure/outputs.tf b/infrastructure/azure/outputs.tf index a84a3dd1..7c361ec2 100644 --- a/infrastructure/azure/outputs.tf +++ b/infrastructure/azure/outputs.tf @@ -15,6 +15,7 @@ output "installation" { postgres_host = azurerm_postgresql_flexible_server.database.fqdn managed_identity_id = azurerm_user_assigned_identity.hub.id managed_identity_client_id = azurerm_user_assigned_identity.hub.client_id + setup_identity_id = azurerm_user_assigned_identity.setup.id app_created = var.deploy_hub url = var.deploy_hub ? "https://${azurerm_container_app.hub[0].ingress[0].fqdn}" : null } diff --git a/infrastructure/azure/tests/deployment.tftest.hcl b/infrastructure/azure/tests/deployment.tftest.hcl index b3d2c5ab..36ebac9f 100644 --- a/infrastructure/azure/tests/deployment.tftest.hcl +++ b/infrastructure/azure/tests/deployment.tftest.hcl @@ -45,6 +45,26 @@ mock_provider "azurerm" { } } +override_resource { + target = azurerm_user_assigned_identity.hub + override_during = plan + values = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.ManagedIdentity/userAssignedIdentities/hub" + client_id = "00000000-0000-0000-0000-000000000001" + principal_id = "00000000-0000-0000-0000-000000000002" + } +} + +override_resource { + target = azurerm_user_assigned_identity.setup + override_during = plan + values = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.ManagedIdentity/userAssignedIdentities/setup" + client_id = "00000000-0000-0000-0000-000000000003" + principal_id = "00000000-0000-0000-0000-000000000004" + } +} + variables { subscription_id = "00000000-0000-0000-0000-000000000000" name = "test-hub" @@ -52,6 +72,14 @@ variables { location = "eastus2" image = "example.azurecr.io/hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" postgres_admin_password = "NotARealPassword-12345" + runtime_database_secret = { + key_vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/secrets/providers/Microsoft.KeyVault/vaults/database" + secret_id = "https://database.vault.azure.net/secrets/runtime-database/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + setup_database_secret = { + key_vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/secrets/providers/Microsoft.KeyVault/vaults/database" + secret_id = "https://database.vault.azure.net/secrets/setup-database/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } } run "bootstrap_keeps_hub_stopped" { @@ -78,7 +106,6 @@ run "durable_state_requires_private_authenticated_access" { condition = ( !azurerm_postgresql_flexible_server.database.public_network_access_enabled && azurerm_postgresql_flexible_server_configuration.tls.value == "on" && - endswith(nonsensitive(local.database_url), "?sslmode=verify-full") && azurerm_postgresql_flexible_server.database.backup_retention_days >= 7 ) error_message = "PostgreSQL must be private, TLS-only, and retain recoverable backups." @@ -130,6 +157,82 @@ run "combined_release_uses_one_image_and_bounded_shutdown" { } } +run "setup_and_runtime_have_separate_secret_access" { + command = plan + + variables { + deploy_hub = true + acr_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/existing/providers/Microsoft.ContainerRegistry/registries/existing" + runtime_secret_references = { + SERVICE_API_KEY = { + key_vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/secrets/providers/Microsoft.KeyVault/vaults/service" + secret_id = "https://service.vault.azure.net/secrets/api-key/cccccccccccccccccccccccccccccccc" + } + } + } + + assert { + condition = ( + azurerm_user_assigned_identity.hub.principal_id != azurerm_user_assigned_identity.setup.principal_id && + azurerm_container_app.hub[0].identity[0].identity_ids == toset([azurerm_user_assigned_identity.hub.id]) && + azurerm_container_app_job.setup.identity[0].identity_ids == toset([azurerm_user_assigned_identity.setup.id]) && + azurerm_role_assignment.artifacts.principal_id == azurerm_user_assigned_identity.hub.principal_id && + azurerm_role_assignment.image_pull[0].principal_id == azurerm_user_assigned_identity.hub.principal_id && + azurerm_role_assignment.setup_image_pull[0].principal_id == azurerm_user_assigned_identity.setup.principal_id && + azurerm_role_assignment.setup_image_pull[0].scope == var.acr_id && + azurerm_role_assignment.setup_image_pull[0].role_definition_name == "AcrPull" + ) + error_message = "Setup and Hub need separate identities; only Hub receives artifact access and each pulls its own image." + } + assert { + condition = ( + toset([for grant in azurerm_role_assignment.runtime_secrets : grant.scope]) == toset([ + "${var.runtime_database_secret.key_vault_id}/secrets/runtime-database", + "${var.runtime_secret_references.SERVICE_API_KEY.key_vault_id}/secrets/api-key", + ]) && + alltrue([for grant in azurerm_role_assignment.runtime_secrets : + grant.principal_id == azurerm_user_assigned_identity.hub.principal_id && + grant.role_definition_name == "Key Vault Secrets User" + ]) && + azurerm_role_assignment.setup_secrets.scope == "${var.setup_database_secret.key_vault_id}/secrets/setup-database" && + azurerm_role_assignment.setup_secrets.principal_id == azurerm_user_assigned_identity.setup.principal_id && + azurerm_role_assignment.setup_secrets.role_definition_name == "Key Vault Secrets User" + ) + error_message = "Each identity may read only its own individual secrets, even when database secrets share a vault." + } + assert { + condition = ( + length(azurerm_container_app.hub[0].secret) == 2 && + toset([for secret in azurerm_container_app.hub[0].secret : secret.key_vault_secret_id]) == toset([ + var.runtime_database_secret.secret_id, + var.runtime_secret_references.SERVICE_API_KEY.secret_id, + ]) && + alltrue([for secret in azurerm_container_app.hub[0].secret : + secret.identity == azurerm_user_assigned_identity.hub.id && + (secret.value == null || secret.value == "") + ]) && + length(azurerm_container_app_job.setup.secret) == 1 && + one(azurerm_container_app_job.setup.secret).key_vault_secret_id == var.setup_database_secret.secret_id && + one(azurerm_container_app_job.setup.secret).identity == azurerm_user_assigned_identity.setup.id && + (one(azurerm_container_app_job.setup.secret).value == null || one(azurerm_container_app_job.setup.secret).value == "") + ) + error_message = "Containers must resolve only their own Key Vault references without raw secret values." + } + assert { + condition = ( + length(azurerm_container_app_job.setup.template[0].container[0].env) == 4 && + alltrue([for name, value in { + NODE_ENV = "production" + RUNTIME_MODE = "setup" + DATABASE_RUNTIME_ROLE = "replica_runtime" + } : one([for env in azurerm_container_app_job.setup.template[0].container[0].env : env.value if env.name == name]) == value + ]) && + one([for env in azurerm_container_app_job.setup.template[0].container[0].env : env.secret_name if env.name == "DATABASE_URL"]) == one(azurerm_container_app_job.setup.secret).name + ) + error_message = "Setup must receive only its migration URL, production mode, setup mode, and fixed runtime grant target." + } +} + run "mutable_image_is_rejected" { command = plan variables { @@ -138,6 +241,19 @@ run "mutable_image_is_rejected" { expect_failures = [var.image] } +run "runtime_cannot_receive_the_migration_secret" { + command = plan + variables { + runtime_secret_references = { + EXTRA_DATABASE_URL = { + key_vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/secrets/providers/Microsoft.KeyVault/vaults/database" + secret_id = "https://database.vault.azure.net/secrets/setup-database" + } + } + } + expect_failures = [var.runtime_secret_references] +} + run "hostname_without_certificate_is_rejected" { command = plan variables { diff --git a/infrastructure/azure/variables.tf b/infrastructure/azure/variables.tf index b08c6ec3..360e13bb 100644 --- a/infrastructure/azure/variables.tf +++ b/infrastructure/azure/variables.tf @@ -84,20 +84,68 @@ variable "backup_retention_days" { } } -variable "runtime_secrets" { - type = map(string) - sensitive = true +variable "runtime_database_secret" { + type = object({ + key_vault_id = string + secret_id = string + }) + description = "Existing RBAC Key Vault and HTTPS secret URL containing the replica_runtime database URL with sslmode=verify-full. Never supply the administrator URL." + + validation { + condition = ( + can(regex("^/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.KeyVault/vaults/[^/]+$", var.runtime_database_secret.key_vault_id)) && + can(regex("^https://[a-zA-Z0-9-]+\\.vault\\.azure\\.net/secrets/[a-zA-Z0-9-]+(/[a-fA-F0-9]{32})?$", var.runtime_database_secret.secret_id)) && + lower(try(split("/", var.runtime_database_secret.secret_id)[2], "")) == "${lower(try(split("/", var.runtime_database_secret.key_vault_id)[8], ""))}.vault.azure.net" + ) + error_message = "Supply a Key Vault resource ID and a public-cloud HTTPS secret URL, not a secret value." + } +} + +variable "setup_database_secret" { + type = object({ + key_vault_id = string + secret_id = string + }) + description = "Existing RBAC Key Vault and HTTPS secret URL containing the replica_migration database URL with sslmode=verify-full." + + validation { + condition = ( + can(regex("^/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.KeyVault/vaults/[^/]+$", var.setup_database_secret.key_vault_id)) && + can(regex("^https://[a-zA-Z0-9-]+\\.vault\\.azure\\.net/secrets/[a-zA-Z0-9-]+(/[a-fA-F0-9]{32})?$", var.setup_database_secret.secret_id)) && + lower(try(split("/", var.setup_database_secret.secret_id)[2], "")) == "${lower(try(split("/", var.setup_database_secret.key_vault_id)[8], ""))}.vault.azure.net" && + lower("${var.setup_database_secret.key_vault_id}/secrets/${try(split("/", var.setup_database_secret.secret_id)[4], "")}") != lower("${var.runtime_database_secret.key_vault_id}/secrets/${try(split("/", var.runtime_database_secret.secret_id)[4], "")}") + ) + error_message = "Supply a separate migration secret URL and its Key Vault resource ID." + } +} + +variable "runtime_secret_references" { + type = map(object({ + key_vault_id = string + secret_id = string + })) default = {} - description = "Explicit additional runtime secret values keyed by uppercase environment name." + description = "Existing RBAC Key Vault secret references keyed by uppercase runtime environment name. Values remain outside Terraform." validation { condition = alltrue([ - for name in keys(var.runtime_secrets) : + for name in keys(var.runtime_secret_references) : can(regex("^[A-Z][A-Z0-9_]{0,40}$", name)) && - !contains(["DATABASE_URL", "NODE_ENV", "RUNTIME_MODE", "HOST", "PORT", "CLIENT_DIRECTORY", "SHUTDOWN_TIMEOUT_MS", "ARTIFACT_STORAGE_PROVIDER", "ARTIFACT_STORAGE_DIRECTORY", "AZURE_STORAGE_ACCOUNT", "AZURE_STORAGE_CONTAINER", "AZURE_CLIENT_ID"], name) + !contains(["DATABASE_URL", "DATABASE_RUNTIME_ROLE", "NODE_ENV", "RUNTIME_MODE", "HOST", "PORT", "CLIENT_DIRECTORY", "SHUTDOWN_TIMEOUT_MS", "ARTIFACT_STORAGE_PROVIDER", "ARTIFACT_STORAGE_DIRECTORY", "AZURE_STORAGE_ACCOUNT", "AZURE_STORAGE_CONTAINER", "AZURE_CLIENT_ID"], name) ]) error_message = "Use uppercase environment names of at most 41 characters; deployment-owned settings cannot be overridden." } + + validation { + condition = alltrue([ + for secret in values(var.runtime_secret_references) : + can(regex("^/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.KeyVault/vaults/[^/]+$", secret.key_vault_id)) && + can(regex("^https://[a-zA-Z0-9-]+\\.vault\\.azure\\.net/secrets/[a-zA-Z0-9-]+(/[a-fA-F0-9]{32})?$", secret.secret_id)) && + lower(try(split("/", secret.secret_id)[2], "")) == "${lower(try(split("/", secret.key_vault_id)[8], ""))}.vault.azure.net" && + lower("${secret.key_vault_id}/secrets/${try(split("/", secret.secret_id)[4], "")}") != lower("${var.setup_database_secret.key_vault_id}/secrets/${try(split("/", var.setup_database_secret.secret_id)[4], "")}") + ]) + error_message = "Supply Key Vault resource IDs and HTTPS secret URLs, not secret values." + } } variable "custom_hostname" { From 2e8d80bf4fd6408f3a4a7194ab16b1c68efd6650 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:46:17 +0000 Subject: [PATCH 4/4] refactor(infra): group Azure definitions by responsibility Keep one Terraform module and preserve all resource addresses and expressions. Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- infrastructure/README.md | 2 + .../azure/{compute.tf => application.tf} | 78 +------ infrastructure/azure/database.tf | 30 +++ infrastructure/azure/environment.tf | 25 ++ infrastructure/azure/main.tf | 214 ------------------ infrastructure/azure/networking.tf | 49 ++++ infrastructure/azure/roles.tf | 53 +++++ infrastructure/azure/setup.tf | 68 ++++++ infrastructure/azure/storage.tf | 43 ++++ 9 files changed, 282 insertions(+), 280 deletions(-) rename infrastructure/azure/{compute.tf => application.tf} (65%) create mode 100644 infrastructure/azure/database.tf create mode 100644 infrastructure/azure/environment.tf create mode 100644 infrastructure/azure/networking.tf create mode 100644 infrastructure/azure/roles.tf create mode 100644 infrastructure/azure/setup.tf create mode 100644 infrastructure/azure/storage.tf diff --git a/infrastructure/README.md b/infrastructure/README.md index 71f7fe42..3fc58f1a 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -2,6 +2,8 @@ Operators can provision a durable Hub, run setup before every release, and replace the image without replacing its database or artifacts. This reference chooses Terraform with AzureRM to follow the team's Terraform experience; it is an implementation choice, not an application dependency. The configuration is flat under [`azure/`](azure/) rather than split into one-resource modules. +Definitions are grouped by responsibility: [`networking.tf`](azure/networking.tf), [`database.tf`](azure/database.tf), [`storage.tf`](azure/storage.tf), [`roles.tf`](azure/roles.tf), [`environment.tf`](azure/environment.tf), [`application.tf`](azure/application.tf), and [`setup.tf`](azure/setup.tf). Shared tags and the resource group remain in [`main.tf`](azure/main.tf). These files form one Terraform module; moving declarations between them does not change resource addresses or require state migration. + This installs the current Hub, not hosted previews, review collaboration, user admission, or a sign-in provider. A public HTTPS endpoint does not authorize users or isolate prototype previews. Confirm product access requirements before exposing sensitive data. ## What is created diff --git a/infrastructure/azure/compute.tf b/infrastructure/azure/application.tf similarity index 65% rename from infrastructure/azure/compute.tf rename to infrastructure/azure/application.tf index d88ae1c5..1616548e 100644 --- a/infrastructure/azure/compute.tf +++ b/infrastructure/azure/application.tf @@ -1,70 +1,16 @@ -resource "azurerm_container_app_job" "setup" { - name = "${var.name}-setup" - location = var.location - resource_group_name = azurerm_resource_group.hub.name - container_app_environment_id = azurerm_container_app_environment.hub.id - workload_profile_name = "Consumption" - replica_timeout_in_seconds = 1800 - replica_retry_limit = 0 - tags = local.tags - - manual_trigger_config { - parallelism = 1 - replica_completion_count = 1 +locals { + runtime_environment = { + NODE_ENV = "production" + RUNTIME_MODE = "combined" + HOST = "0.0.0.0" + PORT = "3001" + CLIENT_DIRECTORY = "/app/dist/client" + SHUTDOWN_TIMEOUT_MS = "30000" + ARTIFACT_STORAGE_PROVIDER = "azure" + AZURE_STORAGE_ACCOUNT = azurerm_storage_account.artifacts.name + AZURE_STORAGE_CONTAINER = azurerm_storage_container.artifacts.name + AZURE_CLIENT_ID = azurerm_user_assigned_identity.hub.client_id } - - identity { - type = "UserAssigned" - identity_ids = [azurerm_user_assigned_identity.setup.id] - } - - dynamic "registry" { - for_each = var.acr_id == null ? [] : [var.acr_id] - content { - server = split("/", var.image)[0] - identity = azurerm_user_assigned_identity.setup.id - } - } - - secret { - name = "database-url" - key_vault_secret_id = var.setup_database_secret.secret_id - identity = azurerm_user_assigned_identity.setup.id - } - - template { - container { - name = "setup" - image = var.image - cpu = 0.5 - memory = "1Gi" - command = ["node", "dist/server/scripts/setup/index.js"] - - dynamic "env" { - for_each = { - NODE_ENV = "production" - RUNTIME_MODE = "setup" - DATABASE_RUNTIME_ROLE = "replica_runtime" - } - content { - name = env.key - value = env.value - } - } - - env { - name = "DATABASE_URL" - secret_name = "database-url" - } - } - } - - depends_on = [ - azurerm_postgresql_flexible_server_database.hub, - azurerm_postgresql_flexible_server_configuration.tls, - azurerm_role_assignment.setup_secrets, - azurerm_role_assignment.setup_image_pull, - ] } resource "azurerm_container_app" "hub" { diff --git a/infrastructure/azure/database.tf b/infrastructure/azure/database.tf new file mode 100644 index 00000000..4775994b --- /dev/null +++ b/infrastructure/azure/database.tf @@ -0,0 +1,30 @@ +resource "azurerm_postgresql_flexible_server" "database" { + name = "${var.name}-postgres" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + version = var.postgres_version + administrator_login = "replicaadmin" + administrator_password = var.postgres_admin_password + sku_name = var.postgres_sku + storage_mb = 32768 + backup_retention_days = var.backup_retention_days + geo_redundant_backup_enabled = false + public_network_access_enabled = false + delegated_subnet_id = azurerm_subnet.database.id + private_dns_zone_id = azurerm_private_dns_zone.database.id + tags = local.tags + depends_on = [azurerm_private_dns_zone_virtual_network_link.database] +} + +resource "azurerm_postgresql_flexible_server_configuration" "tls" { + name = "require_secure_transport" + server_id = azurerm_postgresql_flexible_server.database.id + value = "on" +} + +resource "azurerm_postgresql_flexible_server_database" "hub" { + name = "replica" + server_id = azurerm_postgresql_flexible_server.database.id + charset = "UTF8" + collation = "en_US.utf8" +} diff --git a/infrastructure/azure/environment.tf b/infrastructure/azure/environment.tf new file mode 100644 index 00000000..52cd7c4c --- /dev/null +++ b/infrastructure/azure/environment.tf @@ -0,0 +1,25 @@ +resource "azurerm_log_analytics_workspace" "hub" { + name = "${var.name}-logs" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + sku = "PerGB2018" + retention_in_days = 30 + tags = local.tags +} + +resource "azurerm_container_app_environment" "hub" { + name = "${var.name}-environment" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + log_analytics_workspace_id = azurerm_log_analytics_workspace.hub.id + logs_destination = "log-analytics" + infrastructure_subnet_id = azurerm_subnet.apps.id + tags = local.tags + + workload_profile { + name = "Consumption" + workload_profile_type = "Consumption" + minimum_count = 0 + maximum_count = 0 + } +} diff --git a/infrastructure/azure/main.tf b/infrastructure/azure/main.tf index 3b55b5e4..38189c1d 100644 --- a/infrastructure/azure/main.tf +++ b/infrastructure/azure/main.tf @@ -5,22 +5,6 @@ locals { owner = var.owner managed_by = "terraform" } - runtime_environment = { - NODE_ENV = "production" - RUNTIME_MODE = "combined" - HOST = "0.0.0.0" - PORT = "3001" - CLIENT_DIRECTORY = "/app/dist/client" - SHUTDOWN_TIMEOUT_MS = "30000" - ARTIFACT_STORAGE_PROVIDER = "azure" - AZURE_STORAGE_ACCOUNT = azurerm_storage_account.artifacts.name - AZURE_STORAGE_CONTAINER = azurerm_storage_container.artifacts.name - AZURE_CLIENT_ID = azurerm_user_assigned_identity.hub.client_id - } - runtime_secret_scopes = toset([ - for secret in concat([var.runtime_database_secret], values(var.runtime_secret_references)) : - "${secret.key_vault_id}/secrets/${split("/", secret.secret_id)[4]}" - ]) } resource "azurerm_resource_group" "hub" { @@ -28,201 +12,3 @@ resource "azurerm_resource_group" "hub" { location = var.location tags = local.tags } - -resource "azurerm_virtual_network" "hub" { - name = "${var.name}-network" - location = var.location - resource_group_name = azurerm_resource_group.hub.name - address_space = ["10.40.0.0/16"] - tags = local.tags -} - -resource "azurerm_subnet" "apps" { - name = "container-apps" - resource_group_name = azurerm_resource_group.hub.name - virtual_network_name = azurerm_virtual_network.hub.name - address_prefixes = ["10.40.0.0/23"] - delegation { - name = "container-apps" - service_delegation { - name = "Microsoft.App/environments" - actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] - } - } -} - -resource "azurerm_subnet" "database" { - name = "postgresql" - resource_group_name = azurerm_resource_group.hub.name - virtual_network_name = azurerm_virtual_network.hub.name - address_prefixes = ["10.40.2.0/24"] - delegation { - name = "postgresql" - service_delegation { - name = "Microsoft.DBforPostgreSQL/flexibleServers" - actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] - } - } -} - -resource "azurerm_private_dns_zone" "database" { - name = "${var.name}.postgres.database.azure.com" - resource_group_name = azurerm_resource_group.hub.name - tags = local.tags -} - -resource "azurerm_private_dns_zone_virtual_network_link" "database" { - name = "hub" - resource_group_name = azurerm_resource_group.hub.name - private_dns_zone_name = azurerm_private_dns_zone.database.name - virtual_network_id = azurerm_virtual_network.hub.id - tags = local.tags -} - -resource "azurerm_postgresql_flexible_server" "database" { - name = "${var.name}-postgres" - resource_group_name = azurerm_resource_group.hub.name - location = var.location - version = var.postgres_version - administrator_login = "replicaadmin" - administrator_password = var.postgres_admin_password - sku_name = var.postgres_sku - storage_mb = 32768 - backup_retention_days = var.backup_retention_days - geo_redundant_backup_enabled = false - public_network_access_enabled = false - delegated_subnet_id = azurerm_subnet.database.id - private_dns_zone_id = azurerm_private_dns_zone.database.id - tags = local.tags - depends_on = [azurerm_private_dns_zone_virtual_network_link.database] -} - -resource "azurerm_postgresql_flexible_server_configuration" "tls" { - name = "require_secure_transport" - server_id = azurerm_postgresql_flexible_server.database.id - value = "on" -} - -resource "azurerm_postgresql_flexible_server_database" "hub" { - name = "replica" - server_id = azurerm_postgresql_flexible_server.database.id - charset = "UTF8" - collation = "en_US.utf8" -} - -resource "azurerm_storage_account" "artifacts" { - name = "${replace(var.name, "-", "")}hub" - location = var.location - resource_group_name = azurerm_resource_group.hub.name - account_tier = "Standard" - account_replication_type = "LRS" - min_tls_version = "TLS1_2" - https_traffic_only_enabled = true - allow_nested_items_to_be_public = false - shared_access_key_enabled = false - default_to_oauth_authentication = true - tags = local.tags - - blob_properties { - versioning_enabled = true - delete_retention_policy { - days = 14 - } - container_delete_retention_policy { - days = 14 - } - } -} - -resource "azurerm_storage_container" "artifacts" { - name = "artifacts" - storage_account_id = azurerm_storage_account.artifacts.id - container_access_type = "private" -} - -resource "azurerm_storage_container" "releases" { - name = "releases" - storage_account_id = azurerm_storage_account.artifacts.id - container_access_type = "private" -} - -resource "azurerm_storage_blob" "release_lock" { - name = "deployment.lock" - storage_account_name = azurerm_storage_account.artifacts.name - storage_container_name = azurerm_storage_container.releases.name - type = "Block" - source_content = "Replica Hub release lock. Do not delete while a release is running.\n" -} - -resource "azurerm_user_assigned_identity" "hub" { - name = "${var.name}-hub" - resource_group_name = azurerm_resource_group.hub.name - location = var.location - tags = local.tags -} - -resource "azurerm_user_assigned_identity" "setup" { - name = "${var.name}-setup" - resource_group_name = azurerm_resource_group.hub.name - location = var.location - tags = local.tags -} - -resource "azurerm_role_assignment" "runtime_secrets" { - for_each = local.runtime_secret_scopes - scope = each.value - role_definition_name = "Key Vault Secrets User" - principal_id = azurerm_user_assigned_identity.hub.principal_id -} - -resource "azurerm_role_assignment" "setup_secrets" { - scope = "${var.setup_database_secret.key_vault_id}/secrets/${split("/", var.setup_database_secret.secret_id)[4]}" - role_definition_name = "Key Vault Secrets User" - principal_id = azurerm_user_assigned_identity.setup.principal_id -} - -resource "azurerm_role_assignment" "artifacts" { - scope = "${azurerm_storage_account.artifacts.id}/blobServices/default/containers/${azurerm_storage_container.artifacts.name}" - role_definition_name = "Storage Blob Data Contributor" - principal_id = azurerm_user_assigned_identity.hub.principal_id -} - -resource "azurerm_role_assignment" "image_pull" { - count = var.acr_id == null ? 0 : 1 - scope = var.acr_id - role_definition_name = "AcrPull" - principal_id = azurerm_user_assigned_identity.hub.principal_id -} - -resource "azurerm_role_assignment" "setup_image_pull" { - count = var.acr_id == null ? 0 : 1 - scope = var.acr_id - role_definition_name = "AcrPull" - principal_id = azurerm_user_assigned_identity.setup.principal_id -} - -resource "azurerm_log_analytics_workspace" "hub" { - name = "${var.name}-logs" - resource_group_name = azurerm_resource_group.hub.name - location = var.location - sku = "PerGB2018" - retention_in_days = 30 - tags = local.tags -} - -resource "azurerm_container_app_environment" "hub" { - name = "${var.name}-environment" - resource_group_name = azurerm_resource_group.hub.name - location = var.location - log_analytics_workspace_id = azurerm_log_analytics_workspace.hub.id - logs_destination = "log-analytics" - infrastructure_subnet_id = azurerm_subnet.apps.id - tags = local.tags - - workload_profile { - name = "Consumption" - workload_profile_type = "Consumption" - minimum_count = 0 - maximum_count = 0 - } -} diff --git a/infrastructure/azure/networking.tf b/infrastructure/azure/networking.tf new file mode 100644 index 00000000..3af807e5 --- /dev/null +++ b/infrastructure/azure/networking.tf @@ -0,0 +1,49 @@ +resource "azurerm_virtual_network" "hub" { + name = "${var.name}-network" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + address_space = ["10.40.0.0/16"] + tags = local.tags +} + +resource "azurerm_subnet" "apps" { + name = "container-apps" + resource_group_name = azurerm_resource_group.hub.name + virtual_network_name = azurerm_virtual_network.hub.name + address_prefixes = ["10.40.0.0/23"] + delegation { + name = "container-apps" + service_delegation { + name = "Microsoft.App/environments" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_subnet" "database" { + name = "postgresql" + resource_group_name = azurerm_resource_group.hub.name + virtual_network_name = azurerm_virtual_network.hub.name + address_prefixes = ["10.40.2.0/24"] + delegation { + name = "postgresql" + service_delegation { + name = "Microsoft.DBforPostgreSQL/flexibleServers" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_private_dns_zone" "database" { + name = "${var.name}.postgres.database.azure.com" + resource_group_name = azurerm_resource_group.hub.name + tags = local.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "database" { + name = "hub" + resource_group_name = azurerm_resource_group.hub.name + private_dns_zone_name = azurerm_private_dns_zone.database.name + virtual_network_id = azurerm_virtual_network.hub.id + tags = local.tags +} diff --git a/infrastructure/azure/roles.tf b/infrastructure/azure/roles.tf new file mode 100644 index 00000000..8d610f1f --- /dev/null +++ b/infrastructure/azure/roles.tf @@ -0,0 +1,53 @@ +locals { + runtime_secret_scopes = toset([ + for secret in concat([var.runtime_database_secret], values(var.runtime_secret_references)) : + "${secret.key_vault_id}/secrets/${split("/", secret.secret_id)[4]}" + ]) +} + +resource "azurerm_user_assigned_identity" "hub" { + name = "${var.name}-hub" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + tags = local.tags +} + +resource "azurerm_user_assigned_identity" "setup" { + name = "${var.name}-setup" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + tags = local.tags +} + +resource "azurerm_role_assignment" "runtime_secrets" { + for_each = local.runtime_secret_scopes + scope = each.value + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_role_assignment" "setup_secrets" { + scope = "${var.setup_database_secret.key_vault_id}/secrets/${split("/", var.setup_database_secret.secret_id)[4]}" + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_user_assigned_identity.setup.principal_id +} + +resource "azurerm_role_assignment" "artifacts" { + scope = "${azurerm_storage_account.artifacts.id}/blobServices/default/containers/${azurerm_storage_container.artifacts.name}" + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_role_assignment" "image_pull" { + count = var.acr_id == null ? 0 : 1 + scope = var.acr_id + role_definition_name = "AcrPull" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_role_assignment" "setup_image_pull" { + count = var.acr_id == null ? 0 : 1 + scope = var.acr_id + role_definition_name = "AcrPull" + principal_id = azurerm_user_assigned_identity.setup.principal_id +} diff --git a/infrastructure/azure/setup.tf b/infrastructure/azure/setup.tf new file mode 100644 index 00000000..40564371 --- /dev/null +++ b/infrastructure/azure/setup.tf @@ -0,0 +1,68 @@ +resource "azurerm_container_app_job" "setup" { + name = "${var.name}-setup" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + container_app_environment_id = azurerm_container_app_environment.hub.id + workload_profile_name = "Consumption" + replica_timeout_in_seconds = 1800 + replica_retry_limit = 0 + tags = local.tags + + manual_trigger_config { + parallelism = 1 + replica_completion_count = 1 + } + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.setup.id] + } + + dynamic "registry" { + for_each = var.acr_id == null ? [] : [var.acr_id] + content { + server = split("/", var.image)[0] + identity = azurerm_user_assigned_identity.setup.id + } + } + + secret { + name = "database-url" + key_vault_secret_id = var.setup_database_secret.secret_id + identity = azurerm_user_assigned_identity.setup.id + } + + template { + container { + name = "setup" + image = var.image + cpu = 0.5 + memory = "1Gi" + command = ["node", "dist/server/scripts/setup/index.js"] + + dynamic "env" { + for_each = { + NODE_ENV = "production" + RUNTIME_MODE = "setup" + DATABASE_RUNTIME_ROLE = "replica_runtime" + } + content { + name = env.key + value = env.value + } + } + + env { + name = "DATABASE_URL" + secret_name = "database-url" + } + } + } + + depends_on = [ + azurerm_postgresql_flexible_server_database.hub, + azurerm_postgresql_flexible_server_configuration.tls, + azurerm_role_assignment.setup_secrets, + azurerm_role_assignment.setup_image_pull, + ] +} diff --git a/infrastructure/azure/storage.tf b/infrastructure/azure/storage.tf new file mode 100644 index 00000000..d56914ee --- /dev/null +++ b/infrastructure/azure/storage.tf @@ -0,0 +1,43 @@ +resource "azurerm_storage_account" "artifacts" { + name = "${replace(var.name, "-", "")}hub" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + account_tier = "Standard" + account_replication_type = "LRS" + min_tls_version = "TLS1_2" + https_traffic_only_enabled = true + allow_nested_items_to_be_public = false + shared_access_key_enabled = false + default_to_oauth_authentication = true + tags = local.tags + + blob_properties { + versioning_enabled = true + delete_retention_policy { + days = 14 + } + container_delete_retention_policy { + days = 14 + } + } +} + +resource "azurerm_storage_container" "artifacts" { + name = "artifacts" + storage_account_id = azurerm_storage_account.artifacts.id + container_access_type = "private" +} + +resource "azurerm_storage_container" "releases" { + name = "releases" + storage_account_id = azurerm_storage_account.artifacts.id + container_access_type = "private" +} + +resource "azurerm_storage_blob" "release_lock" { + name = "deployment.lock" + storage_account_name = azurerm_storage_account.artifacts.name + storage_container_name = azurerm_storage_container.releases.name + type = "Block" + source_content = "Replica Hub release lock. Do not delete while a release is running.\n" +}