diff --git a/.devcontainer/postAttach.sh b/.devcontainer/postAttach.sh index 9793dc1..3cc039f 100755 --- a/.devcontainer/postAttach.sh +++ b/.devcontainer/postAttach.sh @@ -2,6 +2,3 @@ # postCreate.sh - Script executed after the dev container is attached to the host set -euo pipefail - -gcloud auth application-default login -gcloud auth application-default set-quota-project data-warehouse-492522 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8af6ed4..2d8328d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,7 +17,9 @@ updates: labels: - "dependencies" - package-ecosystem: "docker" - directory: "/.devcontainer" + directories: + - "/.devcontainer" + - "/metabase" schedule: interval: "weekly" commit-message: diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/deploy.yml similarity index 65% rename from .github/workflows/terraform-apply.yml rename to .github/workflows/deploy.yml index 010611d..45d429d 100644 --- a/.github/workflows/terraform-apply.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Terraform Apply +name: Deploy on: workflow_dispatch: @@ -21,6 +21,7 @@ concurrency: permissions: contents: write id-token: write + packages: write pull-requests: write jobs: @@ -61,15 +62,44 @@ jobs: terraform-apply: runs-on: ubuntu-latest needs: [setup] - # this convoluted syntax allows skipped and successful prerequisite runs and disallows failures - if: (always() && !cancelled() && !failure()) - environment: ${{ needs.setup.outputs.env_name }} env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Docker Login to GitHub Container Registry + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Cache Parameters + id: cache_params + working-directory: . + run: | + CACHE_SCOPE="compiler" + MAIN_BRANCH_REF="refs/heads/main" + echo "cache_from_args=type=gha,scope=${CACHE_SCOPE},ref=${MAIN_BRANCH_REF}" >> $GITHUB_OUTPUT + echo "cache_to_args=type=gha,scope=${CACHE_SCOPE},mode=max,ref=${MAIN_BRANCH_REF}" >> $GITHUB_OUTPUT + + - name: Build, tag, and push image to GitHub Container Registry + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + builder: ${{ steps.buildx.outputs.name }} + build-args: GIT-SHA=${{ github.sha }} + cache-from: ${{ steps.cache_params.outputs.cache_from_args }} + cache-to: ${{ steps.cache_params.outputs.cache_to_args }} + context: . + file: metabase/Dockerfile + push: true + tags: ghcr.io/${{ github.repository }}/metabase:${{ github.sha }} + - name: Install Terraform uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 with: @@ -85,9 +115,13 @@ jobs: run: ./init.sh "${{ needs.setup.outputs.env_name }}" - name: Terraform Plan + env: + TF_VAR_metabase_container_tag: ${{ github.sha }} run: terraform plan -out=tfplan -lock-timeout=5m - name: Terraform Apply + env: + TF_VAR_metabase_container_tag: ${{ github.sha }} run: terraform apply -lock-timeout=5m tfplan - name: Attach detailed summary to job @@ -98,13 +132,12 @@ jobs: skip-comment: true release: - needs: [terraform-apply] + needs: [setup, terraform-apply] if: ${{ needs.setup.outputs.is_release =='true' }} runs-on: ubuntu-latest permissions: # https://github.com/softprops/action-gh-release#permissions contents: write - steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -112,7 +145,7 @@ jobs: fetch-depth: 0 - name: Release - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 + uses: softprops/action-gh-release@e598afbe1493e6b1bafb1f389cabb956eab91231 # v3.0.3 with: prerelease: false generate_release_notes: true diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index 090f8cc..6a9dae6 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -7,27 +7,28 @@ on: - "terraform/**" - ".github/workflows/terraform-plan.yml" -defaults: - run: - shell: bash - working-directory: terraform - permissions: - id-token: write contents: read + id-token: write pull-requests: write jobs: terraform-plan: runs-on: ubuntu-latest - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + defaults: + run: + working-directory: terraform + steps: - - name: Checkout + - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + - name: Get head SHA of main + id: get_main_sha + run: | + git fetch origin main --depth=1 + MAIN_SHA=$(git rev-parse origin/main) + echo "sha=${MAIN_SHA}" >> $GITHUB_OUTPUT - name: Authenticate to GCP uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 @@ -35,11 +36,18 @@ jobs: workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} service_account: ${{ secrets.GCP_WIF_SERVICE_ACCOUNT }} - - name: Terraform Init - run: terraform init + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: 1.16.6 # should match TF_VERSION in .devcontainer/Dockerfile + + - name: Initialize Terraform and select workspace + run: ./init.sh prod - name: Terraform Plan - run: terraform plan -out=tfplan + env: + TF_VAR_metabase_container_tag: ${{ steps.get_main_sha.outputs.sha }} + run: terraform plan -out=tfplan -lock-timeout=5m - name: Setup tf-summarize uses: kishaningithub/setup-tf-summarize@1596bb06f9509eb36b71bc1b304407cc111059c0 # v3.0.0 @@ -49,12 +57,12 @@ jobs: - name: Check for existing comment uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + id: fc with: issue-number: ${{ github.event.pull_request.number }} comment-author: "github-actions[bot]" body-includes: CHANGE direction: last - id: fc - name: Create or update PR comment with course-grained plan summary uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 diff --git a/.gitignore b/.gitignore index fcef273..66b6c68 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.tfstate.* *.tfvars !terraform.tfvars.example +tfplan # Python __pycache__/ diff --git a/bin/gcloud-auth.sh b/bin/gcloud-auth.sh new file mode 100755 index 0000000..f9a1ae3 --- /dev/null +++ b/bin/gcloud-auth.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# auth to gcloud + +set -euo pipefail + +gcloud auth application-default login +gcloud auth application-default set-quota-project data-warehouse-492522 + +gcloud auth login +gcloud config set project data-warehouse-492522 diff --git a/compose.yml b/compose.yml index b40b076..b5b506a 100644 --- a/compose.yml +++ b/compose.yml @@ -29,8 +29,9 @@ services: volumes: - "./:/home/compiler/src" metabase: - image: "metabase/metabase:v0.65.x" - container_name: "metabase" + build: + context: "./metabase" + dockerfile: Dockerfile ports: - "3000:3000" environment: @@ -48,7 +49,6 @@ services: restart: "unless-stopped" postgres: image: "postgres:18-alpine" - container_name: "postgres" environment: POSTGRES_DB: "metabase" POSTGRES_USER: "metabase" diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index a601e9c..b12560b 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -32,7 +32,7 @@ flowchart TD end subgraph CloudRun ["Application Runtime"] - MB["Metabase OSS Container
(metabase/metabase:vX.Y.Z)
Min Instances: 1 | 2 GiB RAM"] + MB["Metabase Custom Container
(socat TCP bridge)
Min Instances: 1 | 2 GiB RAM"] end subgraph Storage_State ["Application State & Secrets"] @@ -58,7 +58,7 @@ flowchart TD GCS --> dbtRun dbtRun -->|SQL Transformation| BQ - MB -->|Cloud SQL Connection Unix Socket| CSQL + MB -->|TCP localhost:5432 → socat bridge| CSQL MB -.->|Fetch Secrets on Boot| SM MB -->|Native IAM Auth / Zero Keys| BQ SA -.->|Attached Identity| MB diff --git a/docs/reference/structure.md b/docs/reference/structure.md index df4e2fc..cc71680 100644 --- a/docs/reference/structure.md +++ b/docs/reference/structure.md @@ -15,7 +15,10 @@ ├── .env.example # Sample local environment variables ├── .pre-commit-config.yaml # Conventional commits & file hygiene hooks ├── plan.md # Full architectural design & implementation plan -├── dbt/ # dbt project (models, macros, snapshots, profiles) +├── dbt/ # dbt project (models, macros, snapshots, profiles) +├── metabase/ # Custom Metabase image (socat TCP bridge) +│ ├── Dockerfile +│ └── entrypoint.sh ├── terraform/ # Infrastructure as Code (GCP Cloud Run & Cloud SQL) └── README.md ``` diff --git a/metabase/Dockerfile b/metabase/Dockerfile new file mode 100644 index 0000000..a1c0224 --- /dev/null +++ b/metabase/Dockerfile @@ -0,0 +1,8 @@ +FROM metabase/metabase:v0.65.2 + +RUN apk add --no-cache socat + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/metabase/entrypoint.sh b/metabase/entrypoint.sh new file mode 100644 index 0000000..7fb207d --- /dev/null +++ b/metabase/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# Only start socat bridge if CLOUD_SQL_CONNECTION_NAME is set +if [ -n "$CLOUD_SQL_CONNECTION_NAME" ]; then + echo "Starting socat bridge: localhost:5432 → /cloudsql/$CLOUD_SQL_CONNECTION_NAME" + socat TCP-LISTEN:5432,fork,reuseaddr UNIX-CONNECT:/cloudsql/$CLOUD_SQL_CONNECTION_NAME & + sleep 1 +else + echo "CLOUD_SQL_CONNECTION_NAME not set, skipping socat bridge" +fi + +# Execute the original command (Metabase startup) +exec /app/run_metabase.sh "$@" diff --git a/plan.md b/plan.md index 816f5bb..b18542d 100644 --- a/plan.md +++ b/plan.md @@ -38,22 +38,9 @@ Refer to [docs/reference/structure.md](docs/reference/structure.md) for the deta --- -## 4. Phase 1: Dev Container & Local Development +## 4. Production Configuration Details -Following the `compilerla` pattern: - -- **`compose.yml`** orchestrates three services: - 1. `dev`: VS Code developer container mounting the repository at `/home/compiler/src`, running as user `compiler`. - 2. `metabase`: Official Metabase Docker container connected to local Postgres. - 3. `postgres`: PostgreSQL 16 backing Metabase application state. -- **`.devcontainer/devcontainer.json`** configures VS Code extensions (`hashicorp.terraform`, `gitlens`, `prettier`, etc.) and sets the default terminal profile. -- **`.devcontainer/Dockerfile`** installs developer tools: Terraform/OpenTofu, Google Cloud CLI (`gcloud`), `pre-commit`, and Postgres client utilities. - ---- - -## 5. Production Configuration Details - -### 5.1 Cloud Run Sizing & Environment Variables +### 4.1 Cloud Run Sizing & Environment Variables - **CPU & Memory**: 1 vCPU, 2 GiB RAM (can scale to 2 vCPU / 4 GiB if user concurrency grows). - **Scaling Rules**: `min_instances = 1`, `max_instances = 3`. @@ -68,7 +55,7 @@ Following the `compilerla` pattern: - `MB_SITE_URL`: `https://dashboards.compiler.la` - `JAVA_OPTS`: `"-Xmx1536m -Xms512m -XX:+UseG1GC"` (Allocates appropriate JVM heap within the 2 GiB container limit). -### 5.2 BigQuery Authentication +### 4.2 BigQuery Authentication - Dedicated Service Account: `metabase-runner@data-warehouse-492522.iam.gserviceaccount.com`. - IAM roles granted: @@ -76,7 +63,7 @@ Following the `compilerla` pattern: - `roles/bigquery.dataViewer` (permits reading analytical datasets/tables). - `roles/cloudsql.client` (permits connecting to Cloud SQL via unix domain socket). -### 5.3 Ingress & Cloudflare DNS +### 4.3 Ingress & Cloudflare DNS 1. In GCP Cloud Run, configure Custom Domain Mapping for `dashboards.compiler.la`. 2. In Cloudflare DNS: @@ -85,7 +72,7 @@ Following the `compilerla` pattern: --- -## 6. Phased Implementation Roadmap +## 5. Phased Implementation Roadmap 1. **Phase 1: Dev Container & Local Development**: - Set up `.devcontainer/devcontainer.json`, `.devcontainer/Dockerfile`, `compose.yml`, and `.env.example`. diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 76d1982..7db4243 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -2,39 +2,39 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/cloudflare/cloudflare" { - version = "5.28.0" + version = "5.28.1" constraints = "~> 5.28.0, ~> 5.28" hashes = [ - "h1:CdHJMkBNOZa+dyrho1BuYdjgCNaoqaBbSSOqlArqkD0=", - "zh:216e03e0dd702f4be29280584290388c7fed0d4beb0a998f3c1229cb74166b7b", - "zh:364b099957c7c093491591ecbdc8449f12d13c57d799597563b2b7a5ae9598b9", - "zh:5b160e70bbab47785a95c4b9066e4c8e561fefcc4fc40dbdc586940b57bdcc82", - "zh:6cd34eee027645eea6db31730bcc8c4d1d031d699ed361589105c65342baeb59", - "zh:886714cfd9ada7876dd39ed857661766d9b75ba205079c3b70c4d5b56e9955b8", - "zh:9c1893d4da37939a88b6286ae786d1293b160a63230b6dd396067593fd24eeb4", - "zh:d56c220ad4aea9cd8775d10cbb788c0450870533f3ccc605efdc254dc7186dd4", + "h1:9YL6g+iEfg3OzX6qSZOYsictVO0fNP09D6jRwIZKwQ0=", + "zh:219d464dd727c0e5c90a97b44aa5d7988acc4eec47f3684f7ecd006b17c46d9c", + "zh:2511f2380209c81121e6eb943d7e68d03602a4fcf4f0311dfc67e198179b9303", + "zh:2ae200482bf592cd703981c4d4fe87a52c0dfae81a70261295f59b1bb0d01a6c", + "zh:7df6fe6573379c2faf2e801fdbca58ac909266dbee604feaa993439b8616c9b3", + "zh:9fb5a295588351ce565515dc2a770047e8bab786a4a9665511ab9a9ca2db5f1b", + "zh:a182f4aec86c74e9db4affa22323c5a21f5d92c1fcf91b934570753d04f4daf8", + "zh:a39bb8624fb6c714e38bc0e632aae117a9fede5a18a248ab15dd1b60013c0324", + "zh:bef2ffbdb68569549fe432b550592e07456759a7288f84f7c37c9d933a493f54", "zh:f809ab383cca0a5f83072981c64208cbd7fa67e986a86ee02dd2c82333221e32", - "zh:f8e00ec634b0e97adb664d04ea1ceb116e37eaf4b7a7e581f01e3c95ed1eb83c", ] } provider "registry.terraform.io/hashicorp/google" { - version = "8.7.0" + version = "8.9.0" constraints = "~> 8.7" hashes = [ - "h1:r0evOKSNhYYj04qb6VCTVtm7/uWr67jMZ2TsgLubk6o=", - "zh:087771b1c22126084ec1ac26272f7e0bce1030bedd9d692b6eef38e6baf5db2e", - "zh:4e075872421717124f1a059f80a9cd2a742e868e94ee74546e52494a02f656d2", - "zh:5024e53d378a551d354244db25085c2db530dfdd7bdbd3a3df03abdc957a8a20", - "zh:52dd649ad0c39f51bf39218e896ced370d4c56d65876715b6c12e2ac17f16f71", - "zh:70d6c8f5371a354f8265fe4f2c279e0c2f6a06054538dbe2d92e428692192439", - "zh:7aceb5fe517d2dbd270644663a2b237fc3be3e06577a40d2009983b5a5088fcd", - "zh:bb3a6fd8f472080f3199e57e61cbbdd253c8210b2b70e6c4e30e06b08c39e7de", - "zh:cbd1a1de219397ba98f17203751300433766f5c36953e6aece4a1fb1817638a9", - "zh:d12886e54a30e3b6f174f879777616401f8749466443bc6db870eddd42985fa7", - "zh:d58a84d3a387c78e8ac58b200ceb51a3e332105a11420f33e03aa69b3ec4449f", - "zh:ed11511bc8ee66fbe2d0e6b59f694c127f140cba7ea9654e332c3ebfbdf8bea9", + "h1:1REt6EKYmjtrLK5PC/AHGZ2FFaHs0EMsvLOyv2eADeA=", + "zh:204c800cca7419c69aa219c5584029432e2add15a7159101fb259028d5db5684", + "zh:22cbb6c7ba3d3a91bbd63db158d65a594d1dab04d53fb37fffb3de9e28fccbc8", + "zh:3fe9bc4bcd8d6cdb277ad074ca71fd76d408b007aed915137807062e6b78bc59", + "zh:46cd3c139d1d24607f0310e01353f1c40cce69d30328b0339768dad428a81d5a", + "zh:5aa53cec75d0e2fc494b3a8ee53f74fece6bc2652bdb9b75e1bdc62a2a43b6f7", + "zh:8650863d8148062f6e22df62c331927e134ae9f83af556d265b17bcbd7b1ab51", + "zh:9ee9965d085f86263842f281257efc8394998c937c4c49eda649a61f26544813", + "zh:cc80028bd30af6182f29f0148e9469a9a49254c0f378caeb3d9fba050543c11a", + "zh:e2adf06a292d50dc58f52d2ba059f2361f1642dad3054ea839ab2e4763cceb8d", + "zh:f0afcbd33aaea01f6a62e7ced7ad5b31c86d5a7016da1965db1fcb2d7b926556", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:feeaffe1edb8ac1f3c0f431f11d21ef31c6b797762d8428e3c18a67ac641cf49", ] } diff --git a/terraform/main.tf b/terraform/main.tf index 87369f0..e96fa34 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -12,10 +12,19 @@ module "iam" { depends_on = [module.apis] } +module "wif" { + source = "./modules/wif" + + project_id = var.project_id + service_account_id = module.iam.github_actions_service_account_id + + depends_on = [module.iam] +} + module "bigquery" { source = "./modules/bigquery" - metabase_service_account_email = module.iam.service_account_email + metabase_service_account_email = module.iam.metabase_service_account_email project_id = var.project_id depends_on = [module.apis, module.iam] @@ -55,10 +64,11 @@ module "cloud_run" { database_name = module.database.database_name database_user = module.database.database_user db_password_secret_id = module.secrets.db_password_secret_id - encryption_secret_key_secret_id = module.secrets.encryption_secret_key_secret_id + encryption_secret_key_secret_id = module.secrets.metabase_encryption_secret_key_secret_id project_id = var.project_id region = var.region - service_account_email = module.iam.service_account_email + service_account_email = module.iam.metabase_service_account_email + container_tag = var.metabase_container_tag depends_on = [module.apis, module.iam, module.secrets, module.database] } diff --git a/terraform/modules/cloud_run/main.tf b/terraform/modules/cloud_run/main.tf index 3993277..024ba1a 100644 --- a/terraform/modules/cloud_run/main.tf +++ b/terraform/modules/cloud_run/main.tf @@ -1,14 +1,15 @@ resource "google_cloud_run_v2_service" "metabase" { - name = var.service_name - location = var.region - project = var.project_id - description = "Metabase OSS Container" - ingress = "INGRESS_TRAFFIC_ALL" + name = var.service_name + location = var.region + project = var.project_id + description = "Metabase OSS Container" + ingress = "INGRESS_TRAFFIC_ALL" + deletion_protection = false template { service_account = var.service_account_email containers { - image = var.container_image + image = "ghcr.io/compilerla/dashboards/metabase:${var.container_tag}" env { name = "JAVA_OPTS" value = var.java_opts @@ -19,10 +20,19 @@ resource "google_cloud_run_v2_service" "metabase" { } env { name = "MB_DB_HOST" - value = "/cloudsql/data-warehouse-492522:us-west1:metabase-db" + value = "localhost" + } + env { + name = "MB_DB_PORT" + value = "5432" + } + env { + name = "CLOUD_SQL_CONNECTION_NAME" + value = var.cloud_sql_connection_name } env { name = "MB_DB_PASS" + value_source { secret_key_ref { secret = "metabase-db-password" @@ -59,6 +69,16 @@ resource "google_cloud_run_v2_service" "metabase" { } } + startup_probe { + tcp_socket { + port = 5432 + } + initial_delay_seconds = 30 + period_seconds = 10 + timeout_seconds = 5 + failure_threshold = 60 + } + volume_mounts { mount_path = "/cloudsql" name = "cloudsql" @@ -88,11 +108,13 @@ resource "google_cloud_run_v2_service_iam_member" "public_access" { } resource "google_cloud_run_domain_mapping" "custom_domain" { - count = var.custom_domain_enabled ? 1 : 0 location = var.region name = "${var.subdomain}.${var.domain_name}" project = var.project_id + metadata { + namespace = var.project_id + } spec { certificate_mode = "AUTOMATIC" route_name = var.service_name diff --git a/terraform/modules/cloud_run/outputs.tf b/terraform/modules/cloud_run/outputs.tf index 460cc5d..7710747 100644 --- a/terraform/modules/cloud_run/outputs.tf +++ b/terraform/modules/cloud_run/outputs.tf @@ -1,6 +1,6 @@ output "custom_domain_resource_records" { description = "DNS resource records required to map the custom domain in Cloudflare." - value = try(google_cloud_run_domain_mapping.custom_domain[0].status[0].resource_records, []) + value = google_cloud_run_domain_mapping.custom_domain.status[0].resource_records } output "service_id" { diff --git a/terraform/modules/cloud_run/variables.tf b/terraform/modules/cloud_run/variables.tf index 7264e36..02b2be4 100644 --- a/terraform/modules/cloud_run/variables.tf +++ b/terraform/modules/cloud_run/variables.tf @@ -8,19 +8,12 @@ variable "domain_name" { type = string } -variable "custom_domain_enabled" { - description = "Whether to enable custom domain mapping. Set to false to skip." - type = bool - default = true -} - variable "cloud_sql_connection_name" { description = "The Cloud SQL instance connection name (project:region:instance)." type = string } -variable "container_image" { - default = "metabase/metabase:v0.65.x" +variable "container_tag" { description = "Docker image tag for Metabase." type = string } diff --git a/terraform/modules/database/variables.tf b/terraform/modules/database/variables.tf index c0cf8fa..7a09ee0 100644 --- a/terraform/modules/database/variables.tf +++ b/terraform/modules/database/variables.tf @@ -11,8 +11,8 @@ variable "database_password" { } variable "database_tier" { - default = "db-f1-micro" - description = "The machine tier for the Cloud SQL PostgreSQL instance (e.g. db-f1-micro, db-g1-small, db-custom-1-3840)." + default = "db-perf-optimized-N-2" + description = "The machine tier for the Cloud SQL PostgreSQL instance." type = string } diff --git a/terraform/modules/iam/main.tf b/terraform/modules/iam/main.tf index 3df9f09..9a7ab50 100644 --- a/terraform/modules/iam/main.tf +++ b/terraform/modules/iam/main.tf @@ -1,9 +1,3 @@ -resource "google_service_account" "metabase" { - account_id = var.service_account_id - description = "Dedicated runtime service account for Metabase on Cloud Run" - display_name = "Metabase Cloud Run Service Account" - project = var.project_id -} locals { # roles/bigquery.dataViewer is granted at the individual dataset level @@ -13,6 +7,27 @@ locals { "roles/cloudsql.client", "roles/secretmanager.secretAccessor" ] + + # Roles for the CI/CD runner to manage infrastructure + github_actions_roles = [ + "roles/editor", + "roles/resourcemanager.projectIamAdmin", + "roles/iam.serviceAccountTokenCreator" + ] +} + +resource "google_service_account" "metabase" { + account_id = var.service_account_id + description = "Dedicated runtime service account for Metabase on Cloud Run" + display_name = "Metabase Cloud Run Service Account" + project = var.project_id +} + +resource "google_service_account" "github_actions" { + account_id = "github-actions-runner" + description = "Service account for GitHub Actions CI/CD" + display_name = "GitHub Actions Runner" + project = var.project_id } resource "google_project_iam_member" "metabase_roles" { @@ -22,3 +37,11 @@ resource "google_project_iam_member" "metabase_roles" { project = var.project_id role = each.value } + +resource "google_project_iam_member" "github_actions_roles" { + for_each = toset(local.github_actions_roles) + + member = "serviceAccount:${google_service_account.github_actions.email}" + project = var.project_id + role = each.value +} diff --git a/terraform/modules/iam/outputs.tf b/terraform/modules/iam/outputs.tf index 22ac995..1eeda5f 100644 --- a/terraform/modules/iam/outputs.tf +++ b/terraform/modules/iam/outputs.tf @@ -1,9 +1,7 @@ -output "service_account_email" { - description = "The email address of the Metabase Cloud Run service account." - value = google_service_account.metabase.email +output "github_actions_service_account_id" { + value = google_service_account.github_actions.id } -output "service_account_id" { - description = "The fully qualified ID of the Metabase Cloud Run service account." - value = google_service_account.metabase.id +output "metabase_service_account_email" { + value = google_service_account.metabase.email } diff --git a/terraform/modules/secrets/main.tf b/terraform/modules/secrets/main.tf index 54bd050..e513ae7 100644 --- a/terraform/modules/secrets/main.tf +++ b/terraform/modules/secrets/main.tf @@ -3,14 +3,14 @@ resource "random_password" "db_password" { special = false } -resource "random_password" "encryption_secret_key" { +resource "random_password" "metabase_encryption_secret_key" { length = 32 special = false } locals { - resolved_db_password = var.db_password != "" ? var.db_password : random_password.db_password.result - resolved_encryption_key = var.encryption_secret_key != "" ? var.encryption_secret_key : random_password.encryption_secret_key.result + resolved_db_password = var.db_password != "" ? var.db_password : random_password.db_password.result + resolved_metabase_encryption_key = var.metabase_encryption_secret_key != "" ? var.metabase_encryption_secret_key : random_password.metabase_encryption_secret_key.result } # Database Password Secret @@ -29,7 +29,7 @@ resource "google_secret_manager_secret_version" "db_password" { } # Encryption Secret Key Secret -resource "google_secret_manager_secret" "encryption_key" { +resource "google_secret_manager_secret" "metabase_encryption_key" { project = var.project_id secret_id = "metabase-encryption-secret-key" @@ -38,9 +38,9 @@ resource "google_secret_manager_secret" "encryption_key" { } } -resource "google_secret_manager_secret_version" "encryption_key" { - secret = google_secret_manager_secret.encryption_key.id - secret_data = local.resolved_encryption_key +resource "google_secret_manager_secret_version" "metabase_encryption_key" { + secret = google_secret_manager_secret.metabase_encryption_key.id + secret_data = local.resolved_metabase_encryption_key } # Google OAuth Client Secret (Conditional) diff --git a/terraform/modules/secrets/outputs.tf b/terraform/modules/secrets/outputs.tf index d2bc47a..ba24220 100644 --- a/terraform/modules/secrets/outputs.tf +++ b/terraform/modules/secrets/outputs.tf @@ -9,23 +9,23 @@ output "db_password_secret_id" { value = google_secret_manager_secret.db_password.secret_id } -output "encryption_secret_key" { +output "metabase_encryption_secret_key" { description = "The resolved Metabase encryption key value." sensitive = true - value = local.resolved_encryption_key + value = local.resolved_metabase_encryption_key } -output "encryption_secret_key_secret_id" { +output "metabase_encryption_secret_key_secret_id" { description = "The Secret Manager secret ID for the Metabase encryption key." - value = google_secret_manager_secret.encryption_key.secret_id + value = google_secret_manager_secret.metabase_encryption_key.secret_id } -output "google_client_secret_id" { +output "metabase_google_client_secret_id" { description = "The Secret Manager secret ID for Google OAuth client secret, if created." value = try(google_secret_manager_secret.google_client_secret[0].secret_id, null) } -output "smtp_password_secret_id" { +output "metabase_smtp_password_secret_id" { description = "The Secret Manager secret ID for SMTP password, if created." value = try(google_secret_manager_secret.smtp_password[0].secret_id, null) } diff --git a/terraform/modules/secrets/variables.tf b/terraform/modules/secrets/variables.tf index 2a65690..79179a2 100644 --- a/terraform/modules/secrets/variables.tf +++ b/terraform/modules/secrets/variables.tf @@ -5,7 +5,7 @@ variable "db_password" { type = string } -variable "encryption_secret_key" { +variable "metabase_encryption_secret_key" { default = "" description = "Optional pre-defined 32+ character Metabase encryption key. If left empty, a secure random key is generated." sensitive = true diff --git a/terraform/modules/wif/main.tf b/terraform/modules/wif/main.tf new file mode 100644 index 0000000..e240111 --- /dev/null +++ b/terraform/modules/wif/main.tf @@ -0,0 +1,21 @@ +resource "google_iam_workload_identity_pool" "github_actions" { + workload_identity_pool_id = var.pool_id + description = "Pool for GitHub Actions" +} + +resource "google_iam_workload_identity_pool_provider" "github_actions" { + workload_identity_pool_id = google_iam_workload_identity_pool.github_actions.workload_identity_pool_id + workload_identity_pool_provider_id = var.provider_id + attribute_condition = var.attribute_condition + attribute_mapping = var.attribute_mapping + description = "GitHub Actions Provider" + oidc { + issuer_uri = var.issuer_uri + } +} + +resource "google_service_account_iam_member" "binding" { + service_account_id = var.service_account_id + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/projects/${var.project_id}/locations/global/workloadIdentityPools/${google_iam_workload_identity_pool.github_actions.workload_identity_pool_id}/providers/${google_iam_workload_identity_pool_provider.github_actions.workload_identity_pool_provider_id}" +} diff --git a/terraform/modules/wif/outputs.tf b/terraform/modules/wif/outputs.tf new file mode 100644 index 0000000..0ca3265 --- /dev/null +++ b/terraform/modules/wif/outputs.tf @@ -0,0 +1,7 @@ +output "pool_id" { + value = var.pool_id +} + +output "provider_id" { + value = var.provider_id +} diff --git a/terraform/modules/wif/variables.tf b/terraform/modules/wif/variables.tf new file mode 100644 index 0000000..8ce0df7 --- /dev/null +++ b/terraform/modules/wif/variables.tf @@ -0,0 +1,41 @@ +variable "project_id" { + description = "The GCP project ID" + type = string +} + +variable "service_account_id" { + description = "The ID of the service account to bind to the provider" + type = string +} + +variable "pool_id" { + description = "The ID of the Workload Identity Pool" + type = string + default = "github-actions-pool" +} + +variable "provider_id" { + description = "The ID of the Workload Identity Provider" + type = string + default = "github-actions-provider" +} + +variable "issuer_uri" { + description = "The OIDC issuer URI" + type = string + default = "https://token.actions.githubusercontent.com" +} + +variable "attribute_mapping" { + description = "Mapping of claims to Google attributes" + type = map(string) + default = { + "google.subject" = "assertion.sub" + } +} + +variable "attribute_condition" { + description = "Condition for the provider" + type = string + default = "assertion.sub == 'repo:compilerla/dashboards'" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 9ef566d..d72084b 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -28,7 +28,8 @@ output "custom_domain_dns_records" { value = module.cloud_run.custom_domain_resource_records } -output "service_account_email" { - description = "The email address of the Cloud Run runtime service account." - value = module.iam.service_account_email +output "metabase_service_account_email" { + description = "The email address of the Metabase Cloud Run runtime service account." + value = module.iam.metabase_service_account_email + } diff --git a/terraform/variables.tf b/terraform/variables.tf index b0490ac..a6a9a5d 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -26,3 +26,8 @@ variable "region" { description = "The GCP region to deploy resources into." type = string } + +variable "metabase_container_tag" { + description = "The Docker tag to deploy." + type = string +}