diff --git a/.github/workflows/hub-infrastructure.yml b/.github/workflows/hub-infrastructure.yml new file mode 100644 index 00000000..b827faff --- /dev/null +++ b/.github/workflows/hub-infrastructure.yml @@ -0,0 +1,40 @@ +name: Hub infrastructure + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +defaults: + run: + working-directory: infrastructure/azure + +jobs: + terraform: + name: Terraform ${{ matrix.command }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + command: ["fmt -check -recursive", "validate", "test"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.13.4 + terraform_wrapper: false + - run: terraform init -backend=false -input=false -lockfile=readonly + - run: terraform ${{ matrix.command }} + + release: + name: Release orchestration tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + - run: node --test deploy-hub/index.test.ts diff --git a/docs/README.md b/docs/README.md index 418f7b2a..fbc8adb0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ Commits and pull requests follow the [contribution conventions](../CONTRIBUTING. ## Project guidance - [Hub documentation](../hub/docs/README.md) covers starting and checking the TypeScript API and frontend, and what is not built yet. +- [Azure Hub installation](../infrastructure/README.md) covers Terraform, release ordering, ownership, and recovery. - [Starter instructions](../starter/AGENTS.md) and [setup guide](../starter/SETUP.md) cover the copyable prototype app and skills. - [Build action setup](../CONTRIBUTING.md#replica-build-action) covers the shared action. - [Controls documentation](../hub/docs/prototype-controls.md) covers the shared engine and definition schema owned by Hub. diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index 19dc4d18..306a92cc 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -41,4 +41,6 @@ The setup lock does not serialize the rest of a release or make jobs exactly-onc CI checks the image against disposable PostgreSQL, including setup, frontend/API serving, readiness, and termination. This is not a supported local-production installation. The existing devcontainer remains the development environment. +The [Azure reference installation](../../infrastructure/README.md) uses Terraform and a serialized maintenance-window release. It consumes the same image and runtime settings without adding Azure deployment logic to Hub. + An actual cloud deployment must additionally verify credentials, networking, data survival, backup and restore, domain configuration, and release ordering in an authorized environment. Never commit secrets, tenant-specific parameter files, or generated service addresses. diff --git a/infrastructure/.gitignore b/infrastructure/.gitignore new file mode 100644 index 00000000..5e77b516 --- /dev/null +++ b/infrastructure/.gitignore @@ -0,0 +1,8 @@ +.tools/ +**/.terraform/ +**/*.tfstate* +**/*.tfplan +**/*.tfvars +**/*.tfvars.json +**/backend.hcl +**/crash.log diff --git a/infrastructure/README.md b/infrastructure/README.md new file mode 100644 index 00000000..b8555f99 --- /dev/null +++ b/infrastructure/README.md @@ -0,0 +1,148 @@ +# Azure Hub installation + +Operators can provision a durable Hub, run setup before every release, and replace the image without replacing its database or artifacts. This reference chooses Terraform with AzureRM to follow the team's Terraform experience; it is an implementation choice, not an application dependency. The configuration is flat under [`azure/`](azure/) rather than split into one-resource modules. + +This installs the current Hub, not hosted previews, review collaboration, user admission, or a sign-in provider. A public HTTPS endpoint does not authorize users or isolate prototype previews. Confirm product access requirements before exposing sensitive data. + +## What is created + +Each installation has its own resource group tagged with its application, installation name, owner, and Terraform ownership. It contains: + +- A virtual network, delegated Container Apps and PostgreSQL subnets, and private PostgreSQL DNS. +- PostgreSQL Flexible Server, the `replica` database, enforced secure transport, and configurable point-in-time recovery retention. +- An Azure Storage account with private `artifacts` and `releases` containers, Blob versioning, and 14-day Blob and container soft deletion. Shared keys and anonymous access are disabled. The Blob endpoint is public-network reachable but requires Entra authorization; this is not a private-endpoint installation. +- One user-assigned workload identity with `Storage Blob Data Contributor` on **only** the artifacts container. An optional existing ACR receives an `AcrPull` assignment for that identity. No registry is created or owned. +- Log Analytics with 30-day retention, a Container Apps environment, a manual setup job, and, after successful setup, one combined Hub app. + +The app has external HTTPS ingress and minimum and maximum one replica **per revision**. Multiple-revision mode permits explicit deactivation before releases. Azure's single-revision mode deliberately overlaps old and new processes while readiness is checked, so it is unsuitable for this maintenance-window strategy. Do not activate old revisions alongside the combined worker. This is not a high-availability or zero-downtime installation. + +The runtime receives `RUNTIME_MODE=combined`, `HOST=0.0.0.0`, `PORT=3001`, `CLIENT_DIRECTORY=/app/dist/client`, `SHUTDOWN_TIMEOUT_MS=30000`, and `NODE_ENV=production`. Artifacts use `ARTIFACT_STORAGE_PROVIDER=azure`, the provisioned `AZURE_STORAGE_ACCOUNT`, `AZURE_STORAGE_CONTAINER=artifacts`, and the workload's `AZURE_CLIENT_ID`. `DATABASE_URL` is a Container Apps secret using `sslmode=verify-full`. Additional `runtime_secrets` are explicit, sensitive environment-name/value inputs; none are Terraform outputs. + +The setup job uses the same digest, `RUNTIME_MODE=setup`, and command `node dist/server/scripts/setup/index.js`. It has one replica, no automatic retries, and a 30-minute timeout. Hub's readiness probe is `/api/ready`; startup and liveness use `/api/health`. Azure allows 60 seconds for termination, covering the 30-second application drain and up to five seconds of telemetry flushing with platform margin. + +The reference uses the PostgreSQL administrator login for both setup and runtime so AzureRM alone can provision it without another database provider or remote SQL provisioner. Operators requiring separate migration/runtime roles must provision and test those grants before adapting this reference. Managed identity is used for Blob and ACR, not database authentication. + +## Prepare access and protected state + +Use the Terraform version constrained in [`versions.tf`](azure/versions.tf), a Node.js version supported by `scripts/package.json`, and a current Azure CLI with Container Apps job and revision commands. CI performs formatting, validation, mocked Terraform tests, and release-orchestration tests without cloud credentials. It does not deploy. + +An authorized administrator must first create the **separate** Terraform backend: a dedicated state resource group, Storage account, and private container. Enable TLS, Entra authorization, Blob versioning and soft deletion; restrict network access to deployment runners and grant only the deployment identity `Storage Blob Data Contributor` on the state container. These bootstrap resources are not part of this installation's state or teardown. Never store this state in the app's artifacts account. + +For automation, configure a dedicated deployment identity with a federated workload identity credential rather than a client secret. Bind its issuer to `https://token.actions.githubusercontent.com`, audience to `api://AzureADTokenExchange`, and subject to the exact protected repository/environment, such as `repo:YOUR_ORG/YOUR_REPO:environment:hub-production`. Protect that environment and deployment branch. A future release workflow needs `id-token: write` and must authenticate Azure CLI as the same identity; the infrastructure-check workflow intentionally has no such permission. + +Set `ARM_CLIENT_ID`, `ARM_TENANT_ID`, `ARM_SUBSCRIPTION_ID`, and `ARM_USE_OIDC=true` in that authorized runner. Terraform's AzureRM backend supports GitHub's OIDC request environment; other runners must supply their supported OIDC token source. For local authorized operation, use Azure CLI sign-in and set `use_oidc=false` in your private backend configuration instead. Do not copy tokens into Terraform inputs. + +The deployment identity needs resource creation/update rights for the installation, role-assignment rights for the container-scoped Blob grant and optional registry grant, and Blob data-plane rights on the installation's account for the lock blob. Resource `Contributor` alone does not grant Blob data access or role assignment. An administrator must authorize the initial resource-group creation and inherited Blob data access for bootstrap, then narrow ongoing deployment grants to that installation resource group, the existing registry, and the backend container. Precreated/imported infrastructure needs a separately reviewed initial plan rather than the empty-state bootstrap command. Do not grant the workload identity deployment or state access. + +Register `Microsoft.App`, `Microsoft.OperationalInsights`, `Microsoft.Network`, `Microsoft.Storage`, `Microsoft.DBforPostgreSQL`, and `Microsoft.ManagedIdentity` beforehand. The provider does not register services across the subscription. Allow time for newly assigned RBAC roles to propagate before setup. + +Copy [`backend.hcl.example`](azure/backend.hcl.example) to `infrastructure/azure/backend.hcl` and [`installation.tfvars.example`](azure/installation.tfvars.example) to `infrastructure/azure/installation.tfvars`. Both real files are ignored. Replace every placeholder, choose a globally unique name, and resolve an existing image to its real `registry/repository@sha256:...` digest. Use either a public image or an existing ACR; private non-ACR registry credentials and ACR's ABAC repository-permissions mode are not covered by this reference. + +Supply `postgres_admin_password` from a secret manager as `TF_VAR_postgres_admin_password`, or through the ignored variables file with mode `0600`. Supply optional `TF_VAR_runtime_secrets` as a JSON object of uppercase environment names to secret values. Never enable shell tracing while injecting secrets. Terraform's sensitive marking redacts normal CLI output but **does not encrypt state or saved plans**. Protect backend access, runner logs, environment variables, and local `.terraform/`; never publish a JSON plan or upload it as a CI artifact. + +## Bootstrap and release + +Run these commands from the repository root in an authorized environment. They change Azure resources; do not run them merely to validate this repository. + +```sh +export HUB_SUBSCRIPTION='your-subscription-id' +export HUB_IMAGE='your-registry/replica-hub@sha256:your-real-64-character-digest' +node infrastructure/azure/deploy-hub/index.ts bootstrap \ + --subscription "$HUB_SUBSCRIPTION" \ + --backend infrastructure/azure/backend.hcl \ + --variables infrastructure/azure/installation.tfvars \ + --image "$HUB_IMAGE" +``` + +Bootstrap creates infrastructure and the manual setup job with `deploy_hub=false`. No combined app exists yet. Review the configuration and a read-only Terraform plan before authorizing this command: the script prints its plans and applies them without an interactive prompt after checking for deletes/replacements. Bootstrap refuses any nonempty state so it cannot race a release on an existing installation. If initial provisioning failed partway, inspect the partial state and recover from a separately reviewed plan with `deploy_hub=false`; never use this recovery path after an app has been created. + +Use the same release command for the first installation, subsequent image releases, and infrastructure or secret changes: + +```sh +node infrastructure/azure/deploy-hub/index.ts release \ + --subscription "$HUB_SUBSCRIPTION" \ + --backend infrastructure/azure/backend.hcl \ + --variables infrastructure/azure/installation.tfvars \ + --image "$HUB_IMAGE" \ + --accept-maintenance +``` + +The operator must approve the maintenance window and inspect intended configuration changes before invoking the command. A read-only plan can be reviewed with `terraform -chdir=infrastructure/azure plan -var-file=installation.tfvars -var="image=$HUB_IMAGE" -var=deploy_hub=true` after initializing the configured backend. Do not apply that preflight plan: setup must run first. The script then: + +1. Checks the explicit subscription, Terraform state's installation, and resource group ownership. +2. Acquires an infinite lease on `releases/deployment.lock`. A competing release fails before stopping Hub. This serializes the whole release, not just individual Terraform operations; setup's database advisory lock remains a separate defense. +3. Checks that no setup execution is running, deactivates every active revision, and waits for **all** old revision replicas to disappear before setup. Users cannot use Hub during this window. +4. Uses a deliberate targeted Terraform plan/apply for the setup job and its infrastructure dependencies. It cannot update the app at this point. +5. Starts that job, checks that exact execution until it succeeds, and stops on failure or timeout. +6. Only after success, plans/applies the complete configuration with `deploy_hub=true` and the same image digest. It activates the latest revision, waits for Azure's ready-revision signal and an HTTPS `/api/ready` response, then releases the lease. + +Terraform owns the image in both resources. There is deliberately **no** `ignore_changes` or out-of-band `az containerapp update --image`: a later infrastructure apply cannot silently use a stale image. Provisioning and rollout are separated by the two phases above. Never run a full Terraform apply directly on an installed Hub, start a setup job manually, or change revisions in the portal outside this serialized procedure. Terraform state locking alone does not protect setup-to-rollout ordering. The app's `prevent_destroy` guard rejects accidental `deploy_hub=false` on later full plans. + +Plans reject resource deletion and replacement during normal releases. A database major upgrade, region move, rename, or replacement needs a reviewed migration/recovery plan, not an override of that guard. The release script deletes its sensitive saved plan on exit. It leaves Hub stopped and retains the infinite lease after any release failure; it never reactivates an older process automatically against a possibly migrated schema. + +### Recover a failed release + +Inspect the named setup execution and Container Apps/Log Analytics logs using authorized Azure access. Check that the release process has exited, every setup execution is terminal, and no old revision still has replicas. After a successful but partially deployed setup, decide whether to retry the same digest or use a compatible recovery image. Prisma migrations, queue migrations, and schedule registration are not a single atomic transaction. + +Only after those checks may an operator break the retained lease: + +```sh +az storage blob lease break --account-name YOUR_ACCOUNT \ + --container-name releases --blob-name deployment.lock \ + --auth-mode login --lease-break-period 0 --subscription "$HUB_SUBSCRIPTION" +``` + +Retry the release command. Do not break an active release's lease. A lease is an operational lock, not protection from administrators bypassing the procedure. If destruction already deleted the lock blob, complete the reviewed teardown rather than starting another release. + +Rollback is another release with a previously published compatible digest. It reruns setup, so confirm that the old image tolerates the current schema and schedules. There is no automatic database or queue downgrade. Retry and shutdown do not guarantee exactly-once jobs; business effects must tolerate at-least-once delivery. Replica disappearance is the available control-plane gate, not evidence that Azure has been proven free of all revision overlap; verify it in the target tenant before relying on worker exclusivity. + +## Hostname and TLS + +The Azure-generated app hostname already uses HTTPS. For a custom domain, first bootstrap and release using that hostname. Create the required DNS ownership TXT and CNAME/A records using the app's verification identifier and address reported by Azure. Upload or provision a certificate in **this** Container Apps environment through your authorized certificate process, then provide both `custom_hostname` and its existing `certificate_id` and run another release. Certificates, DNS zones, certificate renewals, and private keys are not created or owned by this Terraform configuration. + +Verify the custom hostname, HTTPS certificate, redirects/cookies/callback origin configuration, and readiness separately. The release's automated HTTPS check uses the Azure-generated hostname and does not prove custom-domain TLS or authenticated product flows. + +## Backups, restore, and migration + +PostgreSQL retains point-in-time backups for `backup_retention_days` (default 14). Blob/container soft deletion retains deleted objects for 14 days; versioning preserves earlier content, but versions incur storage charges and are not an independent backup. These defaults provide neither regional disaster recovery nor high availability. Managed PostgreSQL, the app's minimum replica, private networking, logs, backups, and Blob versions have ongoing costs even when traffic is low. + +Agree on acceptable data loss and recovery time with the installation owner. Schedule independent database exports and artifact copies to a separately authorized backup destination with its own retention. Run database exports from an authorized private-network host or job; PostgreSQL has no public firewall exception. Encrypt exports and never place credentials in commands, logs, or committed files. + +For recovery, stop releases and Hub, record the database recovery point and matching artifact manifest, and restore PostgreSQL to a **new** server using Azure point-in-time recovery. Restore Blob versions or independent copies to a separate private container/account. Verify database records, queue state, and artifact references together. A database restore alone cannot restore matching artifact state. + +Reconcile the restored resources with a reviewed Terraform configuration and import/state-migration plan before deployment; a restored server has different identifiers and this normal release script intentionally rejects replacements. Do not overwrite or delete the original resources until the restored installation has passed setup, integrity checks, HTTPS/readiness, representative authenticated operations, and a measured recovery exercise. Document the observed recovery time and data loss. + +To move an existing installation, pause writes and workers, export PostgreSQL, copy artifacts with an object manifest, import both into the new installation, and verify counts/checksums and a representative operation before switching traffic. Keep source backups and a rollback window. No automatic migration between local and Azure installations is performed here. + +## Guarded destruction + +Destruction removes database and artifact state. Take verified independent backups first. The default app lifecycle guard intentionally prevents a complete destroy: remove **only** the app's `prevent_destroy` block in a reviewed local change after the owner authorizes permanent deletion. Restore the guard afterward. For a bootstrap that never created the app, no guard edit is needed. + +```sh +node infrastructure/azure/deploy-hub/index.ts destroy \ + --subscription "$HUB_SUBSCRIPTION" \ + --backend infrastructure/azure/backend.hcl \ + --variables infrastructure/azure/installation.tfvars \ + --image "$HUB_IMAGE" \ + --accept-maintenance --confirm YOUR_EXACT_INSTALLATION_NAME +``` + +The script requires matching ownership tags and the exact installation name, acquires the release lock, stops Hub, checks setup executions, and applies a Terraform destroy plan. It deletes only Terraform-tracked resources; it does not issue a broad resource-group deletion command. AzureRM refuses to delete a resource group containing unmanaged resources. Review a read-only `terraform plan -destroy` with the same variables and `deploy_hub=true` before authorizing the command; the script itself does not pause for review. An existing registry, state backend, DNS, and externally supplied certificate are never independently targeted by this configuration. Azure may remove its own environment infrastructure group when the environment is deleted. + +Storage-account deletion does not preserve a usable Blob backup just because object soft deletion was enabled. Provider-retained PostgreSQL backups, soft-deleted logs, or other Azure retention may outlive resource deletion; verify the service-specific retention and billing in your tenant. The separate Terraform state backend and its historical sensitive versions remain until an administrator applies their own retention/deletion policy. + +## Credential-free checks and evidence limits + +From `infrastructure/azure/`: + +```sh +terraform init -backend=false -input=false -lockfile=readonly +terraform fmt -check -recursive +terraform validate +terraform test +node --test deploy-hub/index.test.ts +``` + +Terraform tests use AzureRM mocks, not an Azure subscription. Orchestration tests replace command execution and HTTPS requests, proving ordering and failure gates rather than Azure behavior. Before production, an authorized tenant exercise must verify provisioning, private PostgreSQL connectivity and TLS, identity-based Blob/ACR access after RBAC propagation, failed setup blocking rollout, maintenance shutdown and revision behavior, HTTPS/readiness, persistence across replacement, custom TLS when configured, backup restoration, and owned-resource teardown. No real-cloud result is implied by these local checks. + +The release command and its tests are TypeScript running directly in Node.js. Terraform's own tests use `.tftest.hcl`; neither requires Python. From the repository root, `npm run typecheck --prefix scripts` checks the TypeScript operational scripts as well. diff --git a/infrastructure/azure/.terraform.lock.hcl b/infrastructure/azure/.terraform.lock.hcl new file mode 100644 index 00000000..30c89942 --- /dev/null +++ b/infrastructure/azure/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.50.0" + constraints = "4.50.0" + hashes = [ + "h1:j2bM6QYxBw4dKDlNLR2lAWP6Dggv5lhyGU5VREOfaDw=", + "zh:11899f771a9f03c4e5d222d73ce842e930d26af8aaba88b99a2f0638265331dc", + "zh:2450a4258f40f6ba1f6705a10b2bdb6ff2067ab4c2a6938135391df03c974251", + "zh:2754e761370a18dab5748e020f595e08a7859522889e800204e7a5f65b026590", + "zh:6072e006b1c1b5c1e0a39d373bb6beebf5b4dbb619627c510b6a741521d226c6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8ddb7263f15bc16f3b85be8377c9e32ffa4db4cf809586d8010633698ff66105", + "zh:96f973bc54db728defca4817fbd9b1a412c538542e497536d1ce5f6be9946801", + "zh:aa519b7bad1e75930d84ec9ea8462ec384ed26a6ea172281cbd93244ecf8059b", + "zh:adbc238c95915d23d610990770f4b83e576311d4114b9662c4e967d0b0065b1b", + "zh:eaf94d50425ad88cdd42170e9737d8cb1a557ca3737cb905ce34e0218ec621f3", + "zh:f7aa0004e92e67fcee5d8725b878c8de7e6db6f0f766cbe2257d49a994b314f7", + "zh:fbb1195ea2784d67c690ed399b27d59d015c417dec21667f07a6aaf4099cd174", + ] +} diff --git a/infrastructure/azure/backend.hcl.example b/infrastructure/azure/backend.hcl.example new file mode 100644 index 00000000..3832c63f --- /dev/null +++ b/infrastructure/azure/backend.hcl.example @@ -0,0 +1,6 @@ +resource_group_name = "your-existing-terraform-state-group" +storage_account_name = "youruniquestateaccount" +container_name = "terraform" +key = "replica-hub.tfstate" +use_azuread_auth = true +use_oidc = true diff --git a/infrastructure/azure/compute.tf b/infrastructure/azure/compute.tf new file mode 100644 index 00000000..12cd33ca --- /dev/null +++ b/infrastructure/azure/compute.tf @@ -0,0 +1,197 @@ +resource "azurerm_container_app_job" "setup" { + name = "${var.name}-setup" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + container_app_environment_id = azurerm_container_app_environment.hub.id + workload_profile_name = "Consumption" + replica_timeout_in_seconds = 1800 + replica_retry_limit = 0 + tags = local.tags + + manual_trigger_config { + parallelism = 1 + replica_completion_count = 1 + } + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.hub.id] + } + + dynamic "registry" { + for_each = var.acr_id == null ? [] : [var.acr_id] + content { + server = split("/", var.image)[0] + identity = azurerm_user_assigned_identity.hub.id + } + } + + secret { + name = "database-url" + value = local.database_url + } + + dynamic "secret" { + for_each = local.secret_names + content { + name = "runtime-${substr(sha256(secret.value), 0, 12)}" + value = var.runtime_secrets[secret.value] + } + } + + template { + container { + name = "setup" + image = var.image + cpu = 0.5 + memory = "1Gi" + command = ["node", "dist/server/scripts/setup/index.js"] + + dynamic "env" { + for_each = merge(local.runtime_environment, { RUNTIME_MODE = "setup" }) + content { + name = env.key + value = env.value + } + } + + env { + name = "DATABASE_URL" + secret_name = "database-url" + } + + dynamic "env" { + for_each = local.secret_names + content { + name = env.value + secret_name = "runtime-${substr(sha256(env.value), 0, 12)}" + } + } + } + } + + depends_on = [ + azurerm_postgresql_flexible_server_database.hub, + azurerm_postgresql_flexible_server_configuration.tls, + azurerm_role_assignment.artifacts, + azurerm_role_assignment.image_pull, + ] +} + +resource "azurerm_container_app" "hub" { + count = var.deploy_hub ? 1 : 0 + name = "${var.name}-hub" + resource_group_name = azurerm_resource_group.hub.name + container_app_environment_id = azurerm_container_app_environment.hub.id + workload_profile_name = "Consumption" + revision_mode = "Multiple" + tags = local.tags + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.hub.id] + } + + dynamic "registry" { + for_each = var.acr_id == null ? [] : [var.acr_id] + content { + server = split("/", var.image)[0] + identity = azurerm_user_assigned_identity.hub.id + } + } + + secret { + name = "database-url" + value = local.database_url + } + + dynamic "secret" { + for_each = local.secret_names + content { + name = "runtime-${substr(sha256(secret.value), 0, 12)}" + value = var.runtime_secrets[secret.value] + } + } + + ingress { + external_enabled = true + allow_insecure_connections = false + target_port = 3001 + transport = "http" + traffic_weight { + latest_revision = true + percentage = 100 + } + } + + template { + min_replicas = 1 + max_replicas = 1 + termination_grace_period_seconds = 60 + + container { + name = "hub" + image = var.image + cpu = 0.5 + memory = "1Gi" + + dynamic "env" { + for_each = local.runtime_environment + content { + name = env.key + value = env.value + } + } + + env { + name = "DATABASE_URL" + secret_name = "database-url" + } + + dynamic "env" { + for_each = local.secret_names + content { + name = env.value + secret_name = "runtime-${substr(sha256(env.value), 0, 12)}" + } + } + + startup_probe { + transport = "HTTP" + port = 3001 + path = "/api/health" + interval_seconds = 5 + failure_count_threshold = 30 + } + readiness_probe { + transport = "HTTP" + port = 3001 + path = "/api/ready" + interval_seconds = 5 + success_count_threshold = 1 + timeout = 3 + } + liveness_probe { + transport = "HTTP" + port = 3001 + path = "/api/health" + interval_seconds = 10 + timeout = 3 + } + } + } + + lifecycle { + prevent_destroy = true + } + + depends_on = [azurerm_container_app_job.setup] +} + +resource "azurerm_container_app_custom_domain" "hub" { + count = var.deploy_hub && var.custom_hostname != null ? 1 : 0 + name = var.custom_hostname + container_app_id = azurerm_container_app.hub[0].id + container_app_environment_certificate_id = var.certificate_id + certificate_binding_type = "SniEnabled" +} diff --git a/infrastructure/azure/deploy-hub/index.test.ts b/infrastructure/azure/deploy-hub/index.test.ts new file mode 100644 index 00000000..c3194989 --- /dev/null +++ b/infrastructure/azure/deploy-hub/index.test.ts @@ -0,0 +1,234 @@ +import assert from "node:assert/strict" +import childProcess from "node:child_process" +import { afterEach, describe, it, mock } from "node:test" +import timers from "node:timers/promises" + +import { deployHub } from "./index.ts" + +afterEach(() => { + mock.restoreAll() +}) + +function installationScenario({ + appCreated = true, + setupStatus = "Succeeded", + lockAvailable = true, +} = {}) { + const installation = { + name: "test-hub", + owner: "test-team", + subscription_id: "test-subscription", + resource_group: "test-hub", + app_name: "test-hub", + setup_job_name: "test-setup", + storage_account: "testhub", + release_container: "releases", + release_blob: "deployment.lock", + app_created: appCreated, + url: "https://hub.example.com", + } + const events: string[][] = [] + const replicaReads = new Map() + let targeted = false + mock.method(childProcess, "execFileSync", (file: string, args: readonly string[]) => { + const command: string[] = [file, ...args] + events.push(command) + if (file === "terraform") { + switch (args[1]) { + case "output": + return JSON.stringify(installation) + case "state": + return appCreated ? "azurerm_container_app.hub[0]" : "" + case "plan": + targeted = args.some((argument: string) => argument.startsWith("-target=")) + return "" + case "show": + return JSON.stringify({ + variables: { + name: { value: installation.name }, + subscription_id: { value: installation.subscription_id }, + }, + resource_changes: [], + }) + case "apply": + if (!targeted) installation.app_created = true + return "" + case "init": + return "" + default: + throw new Error(`Unexpected Terraform command: ${command.join(" ")}`) + } + } + if (file !== "az") throw new Error(`Unexpected executable: ${file}`) + const operation = args.slice(0, 3).join(" ") + if (args.slice(0, 2).join(" ") === "account show") { + return JSON.stringify({ id: installation.subscription_id }) + } + if (args.slice(0, 2).join(" ") === "group show") { + return JSON.stringify({ + tags: { + application: "replica-hub", + managed_by: "terraform", + installation: installation.name, + owner: installation.owner, + }, + }) + } + switch (operation) { + case "storage blob lease": + if (args.includes("acquire")) { + if (!lockAvailable) throw new Error("Release lease is already held") + return JSON.stringify("lease-id") + } + return "null" + case "containerapp revision list": + return JSON.stringify([ + { name: "old-active", properties: { active: true } }, + { name: "old-terminating", properties: { active: false } }, + ]) + case "containerapp replica list": { + const revision = args[args.indexOf("--revision") + 1] + const reads = (replicaReads.get(revision) ?? 0) + 1 + replicaReads.set(revision, reads) + return JSON.stringify(reads === 1 ? [{ name: `${revision}-replica` }] : []) + } + case "containerapp job execution": + return JSON.stringify(args.includes("list") ? [] : { properties: { status: setupStatus } }) + case "containerapp job start": + return JSON.stringify({ name: "setup-execution" }) + case "containerapp revision deactivate": + case "containerapp revision activate": + return "null" + default: + if (args.slice(0, 2).join(" ") === "containerapp show") { + return JSON.stringify({ + properties: { latestRevisionName: "new", latestReadyRevisionName: "new" }, + }) + } + throw new Error(`Unexpected Azure command: ${command.join(" ")}`) + } + }) + mock.method(timers, "setTimeout", async () => undefined) + mock.method(globalThis, "fetch", async (url: Parameters[0]) => { + events.push(["HTTPS", String(url)]) + return new Response(null, { status: 200 }) + }) + const args = [ + "release", + "--subscription", + installation.subscription_id, + "--backend", + "backend.hcl", + "--variables", + "installation.tfvars", + "--image", + `example.azurecr.io/hub@sha256:${"a".repeat(64)}`, + "--accept-maintenance", + ] + return { args, events, replicaReads } +} + +describe("deployHub()", () => { + it("drains all old revisions and completes setup before rollout, readiness, and lease release", async () => { + const { args, events, replicaReads } = installationScenario() + + await deployHub(args) + + const setup = events.findIndex( + (event) => event.slice(0, 4).join(" ") === "az containerapp job start", + ) + const drained = events + .map((event, index) => ({ event, index })) + .filter(({ event }) => event.slice(0, 4).join(" ") === "az containerapp replica list") + const deactivate = events.findIndex( + (event) => event.includes("deactivate") && event.includes("old-active"), + ) + assert.ok(deactivate >= 0) + assert.ok(deactivate < drained[0].index) + assert.equal(replicaReads.get("old-active"), 2) + assert.equal(replicaReads.get("old-terminating"), 2) + assert.ok(Math.max(...drained.map(({ index }) => index)) < setup) + const setupSucceeded = events.findIndex( + (event) => event.slice(0, 5).join(" ") === "az containerapp job execution show", + ) + const rollout = events.findIndex( + (event) => + event[0] === "terraform" && + event[2] === "plan" && + event.includes("-var=deploy_hub=true") && + !event.some((argument) => argument.startsWith("-target=")), + ) + assert.ok(setupSucceeded > setup) + assert.ok(rollout > setupSucceeded) + const rolloutApplied = events.findIndex( + (event, index) => index > rollout && event[0] === "terraform" && event[2] === "apply", + ) + assert.ok(rolloutApplied > rollout) + const ready = events.findIndex((event) => event[0] === "HTTPS") + const release = events.findIndex( + (event) => event.slice(0, 5).join(" ") === "az storage blob lease release", + ) + assert.deepEqual(events[ready], ["HTTPS", "https://hub.example.com/api/ready"]) + assert.ok(ready > rolloutApplied) + assert.ok(release > ready) + }) + + it("leaves Hub stopped and retains its lease when setup fails", async () => { + const { args, events } = installationScenario({ setupStatus: "Failed" }) + + await assert.rejects(deployHub(args), /Setup failed/) + + assert.ok(events.some((event) => event.includes("deactivate"))) + const plans = events.filter((event) => event[0] === "terraform" && event[2] === "plan") + assert.equal(plans.length, 1) + assert.ok(plans[0].includes("-target=azurerm_container_app_job.setup")) + assert.ok(!events.some((event) => event.includes("activate"))) + assert.ok(!events.some((event) => event.includes("release"))) + assert.ok(!events.some((event) => event[0] === "HTTPS")) + }) + + it("runs setup before creating the first Hub app", async () => { + const { args, events } = installationScenario({ appCreated: false }) + + await deployHub(args) + + assert.ok(!events.some((event) => event.includes("deactivate"))) + const plans = events.filter((event) => event[0] === "terraform" && event[2] === "plan") + assert.equal(plans.length, 2) + assert.ok(plans[0].includes("-var=deploy_hub=false")) + assert.ok(plans[0].includes("-target=azurerm_container_app_job.setup")) + assert.ok(plans[1].includes("-var=deploy_hub=true")) + const setup = events.findIndex( + (event) => event.slice(0, 4).join(" ") === "az containerapp job start", + ) + assert.ok(setup > events.indexOf(plans[0])) + assert.ok(events.indexOf(plans[1]) > setup) + }) + + it("does not stop Hub or run setup while another release holds the lease", async () => { + const { args, events } = installationScenario({ lockAvailable: false }) + + await assert.rejects(deployHub(args), /Release lease is already held/) + + assert.ok( + events.some((event) => event.slice(0, 5).join(" ") === "az storage blob lease acquire"), + ) + assert.ok(!events.some((event) => event.includes("deactivate"))) + assert.ok(!events.some((event) => event.slice(0, 4).join(" ") === "az containerapp job start")) + assert.ok(!events.some((event) => event[0] === "terraform" && event[2] === "apply")) + }) + + it("refuses bootstrap when Terraform state already contains resources", async () => { + const { args, events } = installationScenario() + args[0] = "bootstrap" + + await assert.rejects(deployHub(args), /Infrastructure already exists/) + + assert.ok( + events.some( + (event) => event[0] === "terraform" && event[2] === "state" && event[3] === "list", + ), + ) + assert.ok(!events.some((event) => event[0] === "terraform" && event[2] === "apply")) + }) +}) diff --git a/infrastructure/azure/deploy-hub/index.ts b/infrastructure/azure/deploy-hub/index.ts new file mode 100644 index 00000000..32ff2d68 --- /dev/null +++ b/infrastructure/azure/deploy-hub/index.ts @@ -0,0 +1,365 @@ +import childProcess from "node:child_process" +import { randomUUID } from "node:crypto" +import { rm } from "node:fs/promises" +import { resolve } from "node:path" +import timers from "node:timers/promises" +import { pathToFileURL } from "node:url" +import { parseArgs } from "node:util" + +import type { ContainerApp, DeploymentPlan, Installation, JobExecution, Revision } from "./types.ts" + +export async function deployHub(args: string[] = process.argv.slice(2)): Promise { + const { values, positionals } = parseArgs({ + args, + allowPositionals: true, + options: { + subscription: { type: "string" }, + backend: { type: "string" }, + variables: { type: "string" }, + image: { type: "string" }, + "accept-maintenance": { type: "boolean" }, + confirm: { type: "string" }, + help: { type: "boolean" }, + }, + }) + if (values.help) { + console.log( + "Usage: node infrastructure/azure/deploy-hub/index.ts bootstrap|release|destroy " + + "--subscription ID --backend FILE --variables FILE --image REGISTRY/IMAGE@sha256:DIGEST " + + "[--accept-maintenance] [--confirm INSTALLATION]", + ) + return + } + const operation = positionals[0] + if ( + positionals.length !== 1 || + !operation || + !["bootstrap", "release", "destroy"].includes(operation) + ) { + throw new Error("Choose exactly one operation: bootstrap, release, or destroy.") + } + const { subscription, backend, variables, image } = values + if (!subscription || !backend || !variables || !image) { + throw new Error("Supply --subscription, --backend, --variables, and --image.") + } + if (!/^[a-z0-9][a-z0-9.:/-]+@sha256:[a-f0-9]{64}$/.test(image)) { + throw new Error("--image must name an immutable SHA-256 digest.") + } + if (operation !== "bootstrap" && !values["accept-maintenance"]) { + throw new Error("Releases and destruction stop Hub. Supply --accept-maintenance.") + } + + const subscriptionId = subscription + const variablesPath = resolve(variables) + const root = resolve(import.meta.dirname, "..") + const planPath = resolve(root, ".terraform", `deployment-${randomUUID()}.tfplan`) + const previousMask = process.umask(0o077) + let lease: string | undefined + + function command(executable: string, arguments_: string[], capture = false): string { + return ( + childProcess + .execFileSync(executable, arguments_, { + encoding: "utf8", + stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit", + maxBuffer: 32 * 1024 * 1024, + }) + ?.trim() ?? "" + ) + } + function terraform(arguments_: string[], capture = false): string { + return command("terraform", [`-chdir=${root}`, ...arguments_], capture) + } + function azure(...arguments_: string[]): T { + return JSON.parse( + command( + "az", + [...arguments_, "--subscription", subscriptionId, "--only-show-errors", "--output", "json"], + true, + ), + ) as T + } + function readInstallation(): Installation { + const installation = JSON.parse( + terraform(["output", "-json", "installation"], true), + ) as Installation + if (installation.subscription_id !== subscription) { + throw new Error("The selected subscription does not match this Terraform state.") + } + return installation + } + function verifyOwnership(installation: Installation): void { + const group = azure<{ tags?: Record }>( + "group", + "show", + "--name", + installation.resource_group, + ) + const expected = { + application: "replica-hub", + managed_by: "terraform", + installation: installation.name, + owner: installation.owner, + } + if (Object.entries(expected).some(([key, value]) => group.tags?.[key] !== value)) { + throw new Error("Resource group ownership does not match this installation.") + } + } + function appArguments(installation: Installation): string[] { + return ["--name", installation.app_name, "--resource-group", installation.resource_group] + } + function jobArguments(installation: Installation): string[] { + return ["--name", installation.setup_job_name, "--resource-group", installation.resource_group] + } + function blobArguments(installation: Installation, deleting = false): string[] { + return [ + "--account-name", + installation.storage_account, + "--container-name", + installation.release_container, + deleting ? "--name" : "--blob-name", + installation.release_blob, + "--auth-mode", + "login", + ] + } + function plan( + deploy: boolean, + installation?: Installation, + target?: string, + destroy = false, + ): void { + const arguments_ = [ + "plan", + "-input=false", + "-lock-timeout=60s", + `-var-file=${variablesPath}`, + `-var=image=${image}`, + `-var=deploy_hub=${deploy}`, + `-out=${planPath}`, + ] + if (target) arguments_.push(`-target=${target}`) + if (destroy) arguments_.push("-destroy") + terraform(arguments_) + const planned = JSON.parse(terraform(["show", "-json", planPath], true)) as DeploymentPlan + if (!destroy) { + if (planned.resource_changes?.some(({ change }) => change.actions.includes("delete"))) { + throw new Error( + "Release plans cannot delete or replace resources. Review recovery separately.", + ) + } + if (planned.variables.subscription_id?.value !== subscription) { + throw new Error("The variable file selects a different subscription.") + } + if (installation && planned.variables.name?.value !== installation.name) { + throw new Error("The variable file selects a different installation.") + } + } + } + function apply(): void { + terraform(["apply", "-input=false", "-lock-timeout=60s", planPath]) + } + async function stopHub(installation: Installation): Promise { + if (!installation.app_created) return + const revisions = azure( + "containerapp", + "revision", + "list", + ...appArguments(installation), + ) + for (const revision of revisions) { + if (revision.properties.active) { + azure( + "containerapp", + "revision", + "deactivate", + ...appArguments(installation), + "--revision", + revision.name, + ) + } + } + const deadline = performance.now() + 300_000 + // Inactive revisions may still have terminating replicas. + for (const revision of revisions) { + while ( + azure( + "containerapp", + "replica", + "list", + ...appArguments(installation), + "--revision", + revision.name, + ).length + ) { + if (performance.now() >= deadline) { + throw new Error("Old replicas have not stopped. Hub remains in maintenance.") + } + await timers.setTimeout(5_000) + } + } + } + async function setup(installation: Installation): Promise { + const execution = azure( + "containerapp", + "job", + "start", + ...jobArguments(installation), + ) + const deadline = performance.now() + 1_920_000 + while (performance.now() < deadline) { + const result = azure( + "containerapp", + "job", + "execution", + "show", + ...jobArguments(installation), + "--job-execution-name", + execution.name, + ) + if (result.properties.status === "Succeeded") return + if (["Failed", "Stopped", "Canceled"].includes(result.properties.status)) { + throw new Error("Setup failed. Hub remains stopped; inspect the setup execution.") + } + await timers.setTimeout(5_000) + } + throw new Error("Setup timed out. Check or stop the execution before releasing the lock.") + } + async function waitReady(installation: Installation): Promise { + const app = azure("containerapp", "show", ...appArguments(installation)) + const revision = app.properties.latestRevisionName + azure( + "containerapp", + "revision", + "activate", + ...appArguments(installation), + "--revision", + revision, + ) + const deadline = performance.now() + 300_000 + while (performance.now() < deadline) { + const latest = azure("containerapp", "show", ...appArguments(installation)) + if (latest.properties.latestReadyRevisionName === revision) { + try { + const response = await fetch(`${installation.url}/api/ready`, { + signal: AbortSignal.timeout(10_000), + }) + await response.body?.cancel() + if (response.status === 200) return + console.warn(`Hub readiness returned HTTP ${response.status}; retrying.`) + } catch (error) { + if ( + !(error instanceof TypeError) && + !(error instanceof DOMException && error.name === "TimeoutError") + ) + throw error + console.warn("Hub readiness request failed; retrying.") + } + } + await timers.setTimeout(5_000) + } + throw new Error("Hub did not become ready. Keep the release lock and investigate.") + } + + try { + terraform(["init", "-input=false", `-backend-config=${resolve(backend)}`]) + if (azure<{ id: string }>("account", "show").id !== subscription) { + throw new Error("Azure returned a different subscription.") + } + if (operation === "bootstrap") { + if (terraform(["state", "list"], true).trim()) { + throw new Error( + "Infrastructure already exists. Use release; review partial bootstrap recovery separately.", + ) + } + plan(false) + apply() + console.log("Infrastructure is ready. Hub is not running. Run release to apply setup first.") + return + } + let installation = readInstallation() + verifyOwnership(installation) + if (operation === "destroy" && values.confirm !== installation.name) { + throw new Error("Destroy requires --confirm with the exact installation name.") + } + const acquired = azure( + "storage", + "blob", + "lease", + "acquire", + ...blobArguments(installation), + "--lease-duration", + "-1", + ) + if (typeof acquired !== "string" || !acquired) { + throw new Error("Azure did not return a release lease ID.") + } + lease = acquired + installation = readInstallation() + verifyOwnership(installation) + const executions = azure( + "containerapp", + "job", + "execution", + "list", + ...jobArguments(installation), + ) + if ( + executions.some( + ({ properties }) => + !["Succeeded", "Failed", "Stopped", "Canceled"].includes(properties.status), + ) + ) { + throw new Error("A setup execution is still running; do not start another release.") + } + await stopHub(installation) + if (operation === "destroy") { + plan(installation.app_created, installation, undefined, true) + azure("storage", "blob", "delete", ...blobArguments(installation, true), "--lease-id", lease) + lease = undefined + apply() + console.log("Terraform-tracked installation resources were destroyed.") + return + } + // Terraform must not update the app until the entire setup sequence succeeds. + plan(installation.app_created, installation, "azurerm_container_app_job.setup") + apply() + await setup(installation) + plan(true, installation) + apply() + installation = readInstallation() + await waitReady(installation) + azure( + "storage", + "blob", + "lease", + "release", + ...blobArguments(installation), + "--lease-id", + lease, + ) + lease = undefined + console.log("Hub is ready. The maintenance window is complete.") + } catch (error) { + if (lease) + console.error( + "The infinite release lease is retained. Verify all processes and setup executions have stopped before an operator breaks the lease.", + ) + throw error + } finally { + try { + await rm(planPath, { force: true }) + } finally { + process.umask(previousMask) + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + await deployHub() + } catch (error) { + console.error("Deployment stopped:", error instanceof Error ? error.message : "Unknown failure") + process.exitCode = 1 + } +} diff --git a/infrastructure/azure/deploy-hub/types.ts b/infrastructure/azure/deploy-hub/types.ts new file mode 100644 index 00000000..7090c808 --- /dev/null +++ b/infrastructure/azure/deploy-hub/types.ts @@ -0,0 +1,32 @@ +export interface Installation { + name: string + owner: string + subscription_id: string + resource_group: string + app_name: string + setup_job_name: string + storage_account: string + release_container: string + release_blob: string + app_created: boolean + url: string +} + +export interface DeploymentPlan { + variables: Record + resource_changes?: { change: { actions: string[] } }[] +} + +export interface Revision { + name: string + properties: { active: boolean } +} + +export interface JobExecution { + name: string + properties: { status: string } +} + +export interface ContainerApp { + properties: { latestRevisionName: string; latestReadyRevisionName?: string } +} diff --git a/infrastructure/azure/installation.tfvars.example b/infrastructure/azure/installation.tfvars.example new file mode 100644 index 00000000..1a6034d0 --- /dev/null +++ b/infrastructure/azure/installation.tfvars.example @@ -0,0 +1,11 @@ +subscription_id = "00000000-0000-0000-0000-000000000000" +name = "your-unique-hub" +location = "eastus2" +owner = "your-team" +image = "ghcr.io/your-org/replica-hub@sha256:REPLACE_WITH_REAL_DIGEST" + +# Supply postgres_admin_password and runtime_secrets through your secret store, +# TF_VAR_ environment variables, or an ignored, access-restricted tfvars file. +# acr_id = "/subscriptions/.../resourceGroups/.../providers/Microsoft.ContainerRegistry/registries/..." +# custom_hostname = "hub.example.com" +# certificate_id = "/subscriptions/.../resourceGroups/.../providers/Microsoft.App/managedEnvironments/.../certificates/..." diff --git a/infrastructure/azure/main.tf b/infrastructure/azure/main.tf new file mode 100644 index 00000000..fb48b891 --- /dev/null +++ b/infrastructure/azure/main.tf @@ -0,0 +1,199 @@ +locals { + tags = { + application = "replica-hub" + installation = var.name + owner = var.owner + managed_by = "terraform" + } + database_url = "postgresql://replicaadmin:${replace(urlencode(var.postgres_admin_password), "+", "%20")}@${azurerm_postgresql_flexible_server.database.fqdn}:5432/replica?sslmode=verify-full" + runtime_environment = { + NODE_ENV = "production" + RUNTIME_MODE = "combined" + HOST = "0.0.0.0" + PORT = "3001" + CLIENT_DIRECTORY = "/app/dist/client" + SHUTDOWN_TIMEOUT_MS = "30000" + ARTIFACT_STORAGE_PROVIDER = "azure" + AZURE_STORAGE_ACCOUNT = azurerm_storage_account.artifacts.name + AZURE_STORAGE_CONTAINER = azurerm_storage_container.artifacts.name + AZURE_CLIENT_ID = azurerm_user_assigned_identity.hub.client_id + } + secret_names = nonsensitive(toset(keys(var.runtime_secrets))) +} + +resource "azurerm_resource_group" "hub" { + name = "${var.name}-hub" + location = var.location + tags = local.tags +} + +resource "azurerm_virtual_network" "hub" { + name = "${var.name}-network" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + address_space = ["10.40.0.0/16"] + tags = local.tags +} + +resource "azurerm_subnet" "apps" { + name = "container-apps" + resource_group_name = azurerm_resource_group.hub.name + virtual_network_name = azurerm_virtual_network.hub.name + address_prefixes = ["10.40.0.0/23"] + delegation { + name = "container-apps" + service_delegation { + name = "Microsoft.App/environments" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_subnet" "database" { + name = "postgresql" + resource_group_name = azurerm_resource_group.hub.name + virtual_network_name = azurerm_virtual_network.hub.name + address_prefixes = ["10.40.2.0/24"] + delegation { + name = "postgresql" + service_delegation { + name = "Microsoft.DBforPostgreSQL/flexibleServers" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_private_dns_zone" "database" { + name = "${var.name}.postgres.database.azure.com" + resource_group_name = azurerm_resource_group.hub.name + tags = local.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "database" { + name = "hub" + resource_group_name = azurerm_resource_group.hub.name + private_dns_zone_name = azurerm_private_dns_zone.database.name + virtual_network_id = azurerm_virtual_network.hub.id + tags = local.tags +} + +resource "azurerm_postgresql_flexible_server" "database" { + name = "${var.name}-postgres" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + version = var.postgres_version + administrator_login = "replicaadmin" + administrator_password = var.postgres_admin_password + sku_name = var.postgres_sku + storage_mb = 32768 + backup_retention_days = var.backup_retention_days + geo_redundant_backup_enabled = false + public_network_access_enabled = false + delegated_subnet_id = azurerm_subnet.database.id + private_dns_zone_id = azurerm_private_dns_zone.database.id + tags = local.tags + depends_on = [azurerm_private_dns_zone_virtual_network_link.database] +} + +resource "azurerm_postgresql_flexible_server_configuration" "tls" { + name = "require_secure_transport" + server_id = azurerm_postgresql_flexible_server.database.id + value = "on" +} + +resource "azurerm_postgresql_flexible_server_database" "hub" { + name = "replica" + server_id = azurerm_postgresql_flexible_server.database.id + charset = "UTF8" + collation = "en_US.utf8" +} + +resource "azurerm_storage_account" "artifacts" { + name = "${replace(var.name, "-", "")}hub" + location = var.location + resource_group_name = azurerm_resource_group.hub.name + account_tier = "Standard" + account_replication_type = "LRS" + min_tls_version = "TLS1_2" + https_traffic_only_enabled = true + allow_nested_items_to_be_public = false + shared_access_key_enabled = false + default_to_oauth_authentication = true + tags = local.tags + + blob_properties { + versioning_enabled = true + delete_retention_policy { + days = 14 + } + container_delete_retention_policy { + days = 14 + } + } +} + +resource "azurerm_storage_container" "artifacts" { + name = "artifacts" + storage_account_id = azurerm_storage_account.artifacts.id + container_access_type = "private" +} + +resource "azurerm_storage_container" "releases" { + name = "releases" + storage_account_id = azurerm_storage_account.artifacts.id + container_access_type = "private" +} + +resource "azurerm_storage_blob" "release_lock" { + name = "deployment.lock" + storage_account_name = azurerm_storage_account.artifacts.name + storage_container_name = azurerm_storage_container.releases.name + type = "Block" + source_content = "Replica Hub release lock. Do not delete while a release is running.\n" +} + +resource "azurerm_user_assigned_identity" "hub" { + name = "${var.name}-hub" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + tags = local.tags +} + +resource "azurerm_role_assignment" "artifacts" { + scope = "${azurerm_storage_account.artifacts.id}/blobServices/default/containers/${azurerm_storage_container.artifacts.name}" + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_role_assignment" "image_pull" { + count = var.acr_id == null ? 0 : 1 + scope = var.acr_id + role_definition_name = "AcrPull" + principal_id = azurerm_user_assigned_identity.hub.principal_id +} + +resource "azurerm_log_analytics_workspace" "hub" { + name = "${var.name}-logs" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + sku = "PerGB2018" + retention_in_days = 30 + tags = local.tags +} + +resource "azurerm_container_app_environment" "hub" { + name = "${var.name}-environment" + resource_group_name = azurerm_resource_group.hub.name + location = var.location + log_analytics_workspace_id = azurerm_log_analytics_workspace.hub.id + logs_destination = "log-analytics" + infrastructure_subnet_id = azurerm_subnet.apps.id + tags = local.tags + + workload_profile { + name = "Consumption" + workload_profile_type = "Consumption" + minimum_count = 0 + maximum_count = 0 + } +} diff --git a/infrastructure/azure/outputs.tf b/infrastructure/azure/outputs.tf new file mode 100644 index 00000000..a84a3dd1 --- /dev/null +++ b/infrastructure/azure/outputs.tf @@ -0,0 +1,21 @@ +output "installation" { + description = "Non-secret release coordinates; no credentials are emitted." + value = { + name = var.name + owner = var.owner + subscription_id = var.subscription_id + resource_group = azurerm_resource_group.hub.name + app_name = "${var.name}-hub" + setup_job_name = azurerm_container_app_job.setup.name + storage_account = azurerm_storage_account.artifacts.name + release_container = azurerm_storage_container.releases.name + release_blob = azurerm_storage_blob.release_lock.name + environment_id = azurerm_container_app_environment.hub.id + postgres_server_id = azurerm_postgresql_flexible_server.database.id + postgres_host = azurerm_postgresql_flexible_server.database.fqdn + managed_identity_id = azurerm_user_assigned_identity.hub.id + managed_identity_client_id = azurerm_user_assigned_identity.hub.client_id + app_created = var.deploy_hub + url = var.deploy_hub ? "https://${azurerm_container_app.hub[0].ingress[0].fqdn}" : null + } +} diff --git a/infrastructure/azure/tests/deployment.tftest.hcl b/infrastructure/azure/tests/deployment.tftest.hcl new file mode 100644 index 00000000..b3d2c5ab --- /dev/null +++ b/infrastructure/azure/tests/deployment.tftest.hcl @@ -0,0 +1,147 @@ +mock_provider "azurerm" { + override_during = plan + mock_resource "azurerm_virtual_network" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Network/virtualNetworks/test" + } + } + mock_resource "azurerm_subnet" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Network/virtualNetworks/test/subnets/test" + } + } + mock_resource "azurerm_private_dns_zone" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Network/privateDnsZones/test.postgres.database.azure.com" + } + } + mock_resource "azurerm_postgresql_flexible_server" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.DBforPostgreSQL/flexibleServers/test" + fqdn = "test.postgres.database.azure.com" + } + } + mock_resource "azurerm_storage_account" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Storage/storageAccounts/testhub" + } + } + mock_resource "azurerm_user_assigned_identity" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test" + client_id = "00000000-0000-0000-0000-000000000001" + principal_id = "00000000-0000-0000-0000-000000000002" + } + } + mock_resource "azurerm_log_analytics_workspace" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.OperationalInsights/workspaces/test" + } + } + mock_resource "azurerm_container_app_environment" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.App/managedEnvironments/test" + } + } +} + +variables { + subscription_id = "00000000-0000-0000-0000-000000000000" + name = "test-hub" + owner = "test-team" + location = "eastus2" + image = "example.azurecr.io/hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + postgres_admin_password = "NotARealPassword-12345" +} + +run "bootstrap_keeps_hub_stopped" { + command = plan + + assert { + condition = length(azurerm_container_app.hub) == 0 + error_message = "Bootstrap must not create a combined process before setup." + } + assert { + condition = ( + azurerm_container_app_job.setup.template[0].container[0].command == tolist(["node", "dist/server/scripts/setup/index.js"]) && + azurerm_container_app_job.setup.manual_trigger_config[0].parallelism == 1 && + azurerm_container_app_job.setup.replica_retry_limit == 0 + ) + error_message = "Setup must run explicitly, once per execution, without automatic retries." + } +} + +run "durable_state_requires_private_authenticated_access" { + command = plan + + assert { + condition = ( + !azurerm_postgresql_flexible_server.database.public_network_access_enabled && + azurerm_postgresql_flexible_server_configuration.tls.value == "on" && + endswith(nonsensitive(local.database_url), "?sslmode=verify-full") && + azurerm_postgresql_flexible_server.database.backup_retention_days >= 7 + ) + error_message = "PostgreSQL must be private, TLS-only, and retain recoverable backups." + } + assert { + condition = ( + !azurerm_storage_account.artifacts.allow_nested_items_to_be_public && + !azurerm_storage_account.artifacts.shared_access_key_enabled && + azurerm_storage_container.artifacts.container_access_type == "private" && + azurerm_storage_account.artifacts.blob_properties[0].versioning_enabled && + endswith(azurerm_role_assignment.artifacts.scope, "/blobServices/default/containers/artifacts") && + azurerm_role_assignment.artifacts.role_definition_name == "Storage Blob Data Contributor" && + length(azurerm_role_assignment.image_pull) == 0 + ) + error_message = "The workload needs container-scoped Blob access, not keys or registry permissions by default." + } +} + +run "combined_release_uses_one_image_and_bounded_shutdown" { + command = plan + + variables { + deploy_hub = true + acr_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/existing/providers/Microsoft.ContainerRegistry/registries/existing" + } + + assert { + condition = ( + azurerm_container_app.hub[0].template[0].min_replicas == 1 && + azurerm_container_app.hub[0].template[0].max_replicas == 1 && + azurerm_container_app.hub[0].revision_mode == "Multiple" && + azurerm_container_app.hub[0].template[0].termination_grace_period_seconds >= 35 && + azurerm_container_app.hub[0].template[0].termination_grace_period_seconds <= 60 && + azurerm_container_app.hub[0].template[0].container[0].image == azurerm_container_app_job.setup.template[0].container[0].image && + azurerm_role_assignment.image_pull[0].scope == var.acr_id + ) + error_message = "Combined Hub and setup must use the same digest, with bounded shutdown and explicit ACR access." + } + assert { + condition = ( + azurerm_container_app.hub[0].ingress[0].external_enabled && + !azurerm_container_app.hub[0].ingress[0].allow_insecure_connections && + one(azurerm_container_app.hub[0].template[0].container[0].readiness_probe).path == "/api/ready" && + one(azurerm_container_app.hub[0].template[0].container[0].liveness_probe).path == "/api/health" && + local.runtime_environment.RUNTIME_MODE == "combined" && + local.runtime_environment.AZURE_STORAGE_CONTAINER == "artifacts" + ) + error_message = "HTTPS, dependency readiness, process liveness, and runtime configuration must match Hub." + } +} + +run "mutable_image_is_rejected" { + command = plan + variables { + image = "example.azurecr.io/hub:latest" + } + expect_failures = [var.image] +} + +run "hostname_without_certificate_is_rejected" { + command = plan + variables { + custom_hostname = "hub.example.com" + } + expect_failures = [var.certificate_id] +} diff --git a/infrastructure/azure/variables.tf b/infrastructure/azure/variables.tf new file mode 100644 index 00000000..b08c6ec3 --- /dev/null +++ b/infrastructure/azure/variables.tf @@ -0,0 +1,118 @@ +variable "subscription_id" { + type = string + description = "Explicit Azure subscription for this installation." +} + +variable "name" { + type = string + description = "Globally unique installation prefix, also used to confirm ownership." + + validation { + condition = can(regex("^[a-z][a-z0-9-]{2,18}[a-z0-9]$", var.name)) && !strcontains(var.name, "--") + error_message = "Use 4–20 lowercase letters, digits, or single hyphens, beginning with a letter." + } +} + +variable "location" { + type = string + description = "Azure region supporting Container Apps and PostgreSQL Flexible Server." +} + +variable "owner" { + type = string + description = "Team responsible for cost, releases, and recovery." + + validation { + condition = length(trimspace(var.owner)) > 0 + error_message = "An installation owner is required." + } +} + +variable "image" { + type = string + description = "Existing public or ACR image pinned to an immutable SHA-256 digest." + + validation { + condition = can(regex("^[a-z0-9][a-z0-9.:/-]+@sha256:[a-f0-9]{64}$", var.image)) + error_message = "Supply a registry/repository@sha256:<64 lowercase hex characters>, not a tag." + } +} + +variable "deploy_hub" { + type = bool + default = false + description = "Keep false for bootstrap. Only deploy-hub enables Hub after setup succeeds." +} + +variable "acr_id" { + type = string + default = null + description = "Optional existing ACR resource ID. No registry is created or deleted." +} + +variable "postgres_admin_password" { + type = string + sensitive = true + description = "Explicit PostgreSQL administrator password. Stored in protected Terraform state." + + validation { + condition = length(var.postgres_admin_password) >= 16 + error_message = "Use an independently generated PostgreSQL password of at least 16 characters." + } +} + +variable "postgres_sku" { + type = string + default = "B_Standard_B1ms" + description = "PostgreSQL Flexible Server SKU; assess capacity before production use." +} + +variable "postgres_version" { + type = string + default = "17" + description = "PostgreSQL major version supported by Hub and the selected region." +} + +variable "backup_retention_days" { + type = number + default = 14 + description = "PostgreSQL point-in-time recovery retention." + + validation { + condition = var.backup_retention_days >= 7 && var.backup_retention_days <= 35 + error_message = "PostgreSQL retention must be between 7 and 35 days." + } +} + +variable "runtime_secrets" { + type = map(string) + sensitive = true + default = {} + description = "Explicit additional runtime secret values keyed by uppercase environment name." + + validation { + condition = alltrue([ + for name in keys(var.runtime_secrets) : + can(regex("^[A-Z][A-Z0-9_]{0,40}$", name)) && + !contains(["DATABASE_URL", "NODE_ENV", "RUNTIME_MODE", "HOST", "PORT", "CLIENT_DIRECTORY", "SHUTDOWN_TIMEOUT_MS", "ARTIFACT_STORAGE_PROVIDER", "ARTIFACT_STORAGE_DIRECTORY", "AZURE_STORAGE_ACCOUNT", "AZURE_STORAGE_CONTAINER", "AZURE_CLIENT_ID"], name) + ]) + error_message = "Use uppercase environment names of at most 41 characters; deployment-owned settings cannot be overridden." + } +} + +variable "custom_hostname" { + type = string + default = null + description = "Optional DNS hostname already verified for this app." +} + +variable "certificate_id" { + type = string + default = null + description = "Existing Container Apps environment certificate ID for custom_hostname." + + validation { + condition = (var.custom_hostname == null) == (var.certificate_id == null) + error_message = "Supply both custom_hostname and certificate_id, or neither." + } +} diff --git a/infrastructure/azure/versions.tf b/infrastructure/azure/versions.tf new file mode 100644 index 00000000..a93abfa8 --- /dev/null +++ b/infrastructure/azure/versions.tf @@ -0,0 +1,24 @@ +terraform { + required_version = "~> 1.13.0" + + backend "azurerm" {} + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "= 4.50.0" + } + } +} + +provider "azurerm" { + subscription_id = var.subscription_id + storage_use_azuread = true + resource_provider_registrations = "none" + + features { + resource_group { + prevent_deletion_if_contains_resources = true + } + } +} diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index 99e58089..e839e4a9 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -10,6 +10,6 @@ "skipLibCheck": true, "types": ["node"] }, - "include": ["**/*.ts"], + "include": ["**/*.ts", "../infrastructure/**/*.ts"], "exclude": ["node_modules"] }