From 60ced83bf9b5982165d45303a5a5b1704c83ac69 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Fri, 18 Sep 2026 22:50:05 -0700 Subject: [PATCH 01/14] Add md plans --- LOCALSTACK.md | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++ PLAN.md | 55 ++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 LOCALSTACK.md create mode 100644 PLAN.md diff --git a/LOCALSTACK.md b/LOCALSTACK.md new file mode 100644 index 0000000..355aace --- /dev/null +++ b/LOCALSTACK.md @@ -0,0 +1,136 @@ +Stage 1 +Docker + ↓ +LocalStack + ↓ +Terraform + ↓ +S3 + +Stage 2 +S3 + DynamoDB + +Stage 3 +IAM + +Stage 4 +Lambda + +Stage 5 +VPC + subnets + security groups + +Stage 6 +SSM parameters + +Stage 7 +Terraform remote state + +Stage 8 +modules + +Stage 9 +CI/CD + +Stage 10 +ECR/ECS concepts + +Stage 1 — Local infrastructure foundation +Goal: get comfortable with the toolchain itself. Set up Docker, LocalStack, Terraform, and one S3 bucket. Learn what init, plan, apply, destroy, provider, resource, state, variable, and output mean. +Done when: you can start LocalStack, create S3 with Terraform, inspect it, destroy it, and explain every step. +Stage 2 — Application data layer +Goal: understand the difference between object storage and database storage. Keep S3 for files and add DynamoDB for structured application data. +Done when: your app can conceptually say “file goes to S3, metadata/record goes to DynamoDB,” and you understand keys, tables, items, and basic access patterns. +Stage 3 — Permissions and identity +Goal: stop thinking of cloud resources as globally accessible. Add IAM roles and least-privilege policies for the backend/Lambda. +Done when: you can explain which service is allowed to access which bucket/table and why you should avoid broad * permissions. +Stage 4 — Event-driven compute +Goal: understand serverless execution. Add a Lambda that performs one small useful task, ideally tied to your app. +Example: upload to S3 → Lambda processes metadata → writes to DynamoDB. +Done when: you understand events, handlers, execution roles, logs, and when Lambda makes sense versus a normal backend server. +Stage 5 — Networking +Goal: understand how AWS resources communicate securely. Add a VPC, subnets, route concepts, and security groups. +Done when: you can explain VPC vs subnet vs route table vs security group, and which resources should be public or private. +Stage 6 — Configuration and secrets +Goal: remove environment-specific values from source code. Add SSM Parameter Store for configuration and secret-like values. +Done when: your app reads configuration from the environment/SSM rather than hardcoded values, and you understand why secrets should not live in Git or Terraform state. +Stage 7 — Terraform state architecture +Goal: understand how Terraform itself is managed in a team environment. Move from local state toward an S3-backed state model and learn locking. +Done when: you understand why state matters, why teams need shared state, and why simultaneous applies are dangerous. +Stage 8 — Terraform structure and modules +Goal: refactor working infrastructure instead of building abstractions too early. Split networking, storage, IAM, and compute into reusable modules. +Done when: the root Terraform config is small and readable, and you understand module inputs, outputs, and dependencies. +Stage 9 — CI/CD and validation +Goal: make infrastructure changes repeatable and safer. Add automated terraform fmt, validate, tests, and optionally a LocalStack-based plan/test workflow in GitHub Actions. +Done when: every push can automatically verify that your infrastructure code is valid before merging. +Stage 10 — Containers and real deployment architecture +Goal: connect your application container to AWS-style deployment concepts. Dockerize the backend properly and learn ECR, ECS, task definitions, services, and Fargate. If LocalStack free tier limits this, document the architecture rather than forcing emulation. +Done when: you can explain the flow: code → Docker image → ECR → ECS task/service → Fargate. +Stage 11 — Production hardening +Goal: make the architecture look like something you would actually deploy. Add logging, tagging, environment separation, health checks, sensible IAM boundaries, backup considerations, and failure handling. +Done when: you can explain how dev/staging/prod would differ and what would need to change before using real AWS. +Stage 12 — Real AWS migration plan +Goal: prove that LocalStack was a development environment, not the final architecture. Document exactly what would change to deploy to AWS: real credentials, provider endpoints removed, remote state bucket, secrets handling, real ECS/ECR, networking, monitoring, and cost controls. +Done when: you could migrate the project to AWS without redesigning it from scratch. + +Stage 1 — your step list + +0. Install AWS CLI (needed for step 8 verification) +brew install awscli +Verify: +aws --version + +1. Install Terraform +The terraform formula was removed from homebrew-core (BUSL license change) — `brew install terraform` now errors with "No available formula". Use the official HashiCorp tap: +brew tap hashicorp/tap +brew install hashicorp/tap/terraform +Verify: +terraform -version +Expect: version output (1.16.3 or later), no error. +Later upgrades: brew upgrade hashicorp/tap/terraform + +2. Add LocalStack to docker-compose.yml +I'll propose the diff, you review/apply (say when ready — this is a file edit, want your go-ahead before I write it). + +3. Start LocalStack +docker compose up -d localstack +Verify container healthy: +docker compose ps +curl http://localhost:4566/_localstack/health +Expect: JSON showing "s3": "available" (or "running"). + +4. Create infra/ dir, provider.tf, main.tf +I'll write these once you say go — will explain each block before writing. +Provider block targets LocalStack manually (writing it by hand rather than using the `lstk terraform` wrapper, since the point of Stage 1 is understanding what `provider` does): + +provider "aws" { + access_key = "test" + secret_key = "test" + region = "us-east-1" + skip_credentials_validation = true + skip_metadata_api_check = true + + endpoints { + s3 = "http://s3.localhost.localstack.cloud:4566" + } +} + +Note: S3 needs the s3.localhost.localstack.cloud hostname for virtual-hosted-style addressing. Other services use plain http://localhost:4566. + +5. terraform init +Run inside infra/. Downloads AWS provider plugin. Expect: "Terraform has been successfully initialized!" + +6. terraform plan +Shows 1 resource to add (the S3 bucket). Nothing created yet — dry run. + +7. terraform apply +Type yes when prompted. Creates bucket inside LocalStack. + +8. Verify bucket exists — two ways +aws --endpoint-url=http://localhost:4566 s3 ls +(needs the aws CLI from step 0. The old `awslocal` wrapper is deprecated — as of 2026-09-08 LocalStack replaced the Python localstack CLI and all its wrappers (awslocal, tflocal, samlocal, cdklocal) with a single Go binary, `lstk`. If you install it, the equivalent is `lstk awslocal s3 ls`.) +or check state: +terraform show +Expect: bucket name listed either way. + +9. terraform destroy +Type yes. Confirm bucket gone via same aws s3 ls command — expect empty. \ No newline at end of file diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..219e3c4 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,55 @@ +# FirstWaveAI — Project Plan + +## Goal order + +1. Learn Terraform + AWS, free, resume-worthy — using LocalStack (no real AWS account spend, no deploy target needed). +2. After that, upgrade app itself (features, code quality). + +Rationale: app already dockerized (backend FastAPI, frontend Next.js, docker-compose) — used as the thing infra-as-code manages, but goal is the skill/resume line, not actually running this in production AWS. + +--- + +## Phase 1: Terraform + AWS stack (LocalStack — $0, resume-worthy) + +Use [LocalStack](https://localstack.cloud) (free community edition) in Docker. Point Terraform's `aws` provider at it via `endpoint` overrides + dummy credentials — same HCL as real AWS, zero cost, safe to break. + +Resume line target: "Terraform, AWS (VPC, ECS/Fargate, S3, IAM, Lambda, DynamoDB), IaC, CI/CD." + +- [ ] Run LocalStack via docker-compose, confirm `awslocal`/`tflocal` or provider endpoint override works +- [ ] Terraform CLI, core concepts: providers, resources, variables, outputs, state, modules +- [ ] Remote state: S3 backend + DynamoDB lock table +- [ ] IAM: roles, policies, least-privilege for a service +- [ ] Networking: VPC, subnets, security groups (community edition supports these) +- [ ] Storage/data: S3 buckets, DynamoDB tables +- [ ] Compute: Lambda function; ECS (Fargate) task/service if LocalStack community supports it well enough — else document as Pro-only gap +- [ ] Containerize this app's images conceptually as Terraform-managed ECR repos (LocalStack ECR support, or skip ECR and just show Dockerfile + task def wiring) +- [ ] Secrets: SSM Parameter Store for backend `.env` values +- [ ] Terraform modules: refactor above into a reusable module (shows structuring skill, not just single-file HCL) + +Known gaps in free LocalStack (don't chase these, not worth cost): ALB, Route53, ACM — networking/LB edge is weak/paid-tier. Skip; not needed since there's no real deploy target. + +Resources: Terraform docs (registry.terraform.io), LocalStack docs, "Terraform Up & Running" (optional book). + +--- + +## Phase 2: Ops polish (still $0, still resume-worthy) + +- [ ] CI/CD: GitHub Actions running `terraform plan`/`apply` against LocalStack in a CI job — real, showable pipeline, zero cost +- [ ] Environments: dev/prod split via Terraform workspaces or dir-per-env +- [ ] Write up: short README in a new `infra/` dir explaining the stack — this is often what resume/portfolio reviewers actually read + +--- + +## Phase 3: App upgrades (after infra done) + +- [ ] Review backend agents pipeline (`backend/agents`) for correctness/latency +- [ ] Frontend UX polish (`frontend/src`) +- [ ] Add tests (backend + frontend currently look untested) +- [ ] Revisit README's stated features vs actual implementation, close gaps + +--- + +## Notes + +- No real AWS account needed for Phase 1–2 — LocalStack covers the resume-worthy stack at $0. +- If later actually wanting a live deployed demo, that's a separate future decision (real AWS Free Tier) — not part of this plan. From a1a41c802155a173304005f61ae5e64382b93ba2 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Sun, 20 Sep 2026 19:25:47 -0700 Subject: [PATCH 02/14] Setup localstack --- LOCALSTACK.md | 19 ++++++++++++++++++- docker-compose.yml | 23 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/LOCALSTACK.md b/LOCALSTACK.md index 355aace..8926a40 100644 --- a/LOCALSTACK.md +++ b/LOCALSTACK.md @@ -89,7 +89,24 @@ Expect: version output (1.16.3 or later), no error. Later upgrades: brew upgrade hashicorp/tap/terraform 2. Add LocalStack to docker-compose.yml -I'll propose the diff, you review/apply (say when ready — this is a file edit, want your go-ahead before I write it). +Done. `localstack` service added, plus a top-level `volumes:` block. + +2b. LocalStack auth token (required as of 2026) +LocalStack merged the community and pro images into one as of 2026.3.0. `localstack/localstack:latest` +now refuses to start without a token — the container exits with code 55 and logs +"License activation failed! No credentials were found in the environment." +The free tier is still free, but it needs an account. + +Sign up at https://app.localstack.cloud, copy the auth token, and put it in a gitignored root .env: + +LOCALSTACK_AUTH_TOKEN=ls-xxxxxxxx + +docker-compose.yml reads it via ${LOCALSTACK_AUTH_TOKEN:?...}, so a missing token fails at +`docker compose config` time with a clear message instead of a mystery exit 55. +Never commit the token. Root .env is covered by .gitignore line 34. + +Alternative if you ever want to avoid the account: pin `localstack/localstack:4.4.0`, the last +pre-merge community release. Frozen 2025 feature set, and the `lstk` CLI notes below won't apply. 3. Start LocalStack docker compose up -d localstack diff --git a/docker-compose.yml b/docker-compose.yml index d3cda4d..f17d9cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,3 +18,26 @@ services: - "3000:3000" depends_on: - backend + + localstack: + image: localstack/localstack:latest + container_name: firstwave-localstack + ports: + - "4566:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN in .env} + SERVICES: s3 + DEBUG: "0" + AWS_DEFAULT_REGION: us-east-1 + volumes: + - localstack-data:/var/lib/localstack + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + +volumes: + localstack-data: From 65cae948ecbfa9ab143e886b108168f7b3e59c21 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Sun, 20 Sep 2026 20:30:00 -0700 Subject: [PATCH 03/14] Add Terraform S3 bucket against LocalStack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 of the LocalStack learning plan: provider targets LocalStack via endpoint overrides and dummy credentials, creating one S3 bucket. Region is us-west-2, matching AWS_DEFAULT_REGION in docker-compose.yml. Ignore Terraform state and .terraform/ — state can contain secrets and is environment-specific. The provider lock file is committed so init resolves the same versions everywhere. Co-Authored-By: Claude Opus 5 --- .gitignore | 6 ++++++ docker-compose.yml | 2 +- infra/.terraform.lock.hcl | 26 ++++++++++++++++++++++++++ infra/main.tf | 7 +++++++ infra/provider.tf | 22 ++++++++++++++++++++++ 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 infra/.terraform.lock.hcl create mode 100644 infra/main.tf create mode 100644 infra/provider.tf diff --git a/.gitignore b/.gitignore index aff9626..fa864c7 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,9 @@ next-env.d.ts *.egg-info/ .pytest_cache/ .mypy_cache/ + +# terraform +**/.terraform/ +*.tfstate +*.tfstate.* +*.tfvars diff --git a/docker-compose.yml b/docker-compose.yml index f17d9cb..68e3acf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN in .env} SERVICES: s3 DEBUG: "0" - AWS_DEFAULT_REGION: us-east-1 + AWS_DEFAULT_REGION: us-west-2 volumes: - localstack-data:/var/lib/localstack - /var/run/docker.sock:/var/run/docker.sock diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 0000000..d591841 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.65.0" + constraints = "~> 6.0" + hashes = [ + "h1:/VgIzAOR/v+p135IFsJjYT7q3pA24yE3lZGBV0Otqq0=", + "zh:15b5bd81119965363893197b3b6065bdf46888b93c536623fd113b5505c375be", + "zh:16409fd045116a31b28adce98fcaaa56a7c01487369713e4a2a04af1cfa96fa3", + "zh:422ea20ef4be8e5b942118d1da61cbde818cadb512ec1132306d65120f983917", + "zh:42cda6703a6a51585c2cb2b8b3ab3a7c80a3ee08838138be8ecaa6b97b1d74d8", + "zh:718a880d81bfd9af7e297ed3d7bf98d1febebe8b9ebe3333854a4c17f2c4de09", + "zh:74e538a8ff4ea27b2040be426cdd2b952725883f5b45c8c03b27eff7d82b40f8", + "zh:876bf62e56a41e0c7a514652e22a41246e7c1d67be3b2c1b68553fa3a8dae6d7", + "zh:8d571e06d78b91b28faa7fafee99dee920d4f7a50f07ec9cd21695a385bcc0d5", + "zh:93154e33f4cbd39825a92a230642082e7b2b8b058c27cf93588ee6824aa1c294", + "zh:935f9523c940dc5795ce8afac37aa8a2ed06c0012196ab39b1de2ea26acc129e", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9bffe18e907e04d98d7d6b9a6a4c2381448e365441df423e90e1eeaf3a79fd2f", + "zh:9ddfdefef226c8ff3c8de03d8df23ab3199fed9f0684618a62489e0e90a21cab", + "zh:9eab3abf041fe8e1ce5959fda9c20ddfaf331b7c29ff707e914451abce7841af", + "zh:ed749702f6c56b26a390a52bfd31d3bdf7f0af800bc16244276ca71d6f541e87", + "zh:f304df223a0bc3e840a806a5dffb75e6b2b75c879053dc4ebd0228484923ee59", + ] +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000..ad2a208 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,7 @@ +resource "aws_s3_bucket" "uploads" { + bucket = "firstwave-uploads" +} + +output "bucket_name" { + value = aws_s3_bucket.uploads.id +} diff --git a/infra/provider.tf b/infra/provider.tf new file mode 100644 index 0000000..ec1c1e6 --- /dev/null +++ b/infra/provider.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } +} + +provider "aws" { + access_key = "test" + secret_key = "test" + region = "us-west-2" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + + endpoints { + s3 = "http://s3.localhost.localstack.cloud:4566" + s3control = "http://localhost.localstack.cloud:4566" + } +} From 21040d38f0b5d31cddad2e5817695f30e4347dcd Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Sun, 20 Sep 2026 20:34:02 -0700 Subject: [PATCH 04/14] Update plans --- LOCALSTACK.md | 23 +++++++++++++++++++++-- PLAN.md | 3 ++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/LOCALSTACK.md b/LOCALSTACK.md index 8926a40..7643ed1 100644 --- a/LOCALSTACK.md +++ b/LOCALSTACK.md @@ -143,11 +143,30 @@ Shows 1 resource to add (the S3 bucket). Nothing created yet — dry run. Type yes when prompted. Creates bucket inside LocalStack. 8. Verify bucket exists — two ways -aws --endpoint-url=http://localhost:4566 s3 ls +The AWS CLI refuses to sign a request with no credentials at all, even though LocalStack +ignores the values. So pass dummy creds and a region (same reason provider.tf sets them): + +AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test \ + aws --endpoint-url=http://localhost:4566 --region us-west-2 s3 ls + +Without them: "An error occurred (NoCredentials): Unable to locate credentials." + +To stop prefixing every command, set up a named profile once (endpoint_url in a profile +needs AWS CLI v2.13+): + +aws configure set aws_access_key_id test --profile localstack +aws configure set aws_secret_access_key test --profile localstack +aws configure set region us-west-2 --profile localstack +aws configure set endpoint_url http://localhost:4566 --profile localstack + +then: aws --profile localstack s3 ls + (needs the aws CLI from step 0. The old `awslocal` wrapper is deprecated — as of 2026-09-08 LocalStack replaced the Python localstack CLI and all its wrappers (awslocal, tflocal, samlocal, cdklocal) with a single Go binary, `lstk`. If you install it, the equivalent is `lstk awslocal s3 ls`.) or check state: terraform show Expect: bucket name listed either way. 9. terraform destroy -Type yes. Confirm bucket gone via same aws s3 ls command — expect empty. \ No newline at end of file +Type yes. Confirm bucket gone via the same aws s3 ls command from step 8 — expect empty. +destroy deletes the bucket inside LocalStack and updates terraform.tfstate. It does NOT +touch provider.tf or main.tf — terraform apply rebuilds the bucket from them any time. \ No newline at end of file diff --git a/PLAN.md b/PLAN.md index 219e3c4..befd6e3 100644 --- a/PLAN.md +++ b/PLAN.md @@ -11,7 +11,8 @@ Rationale: app already dockerized (backend FastAPI, frontend Next.js, docker-com ## Phase 1: Terraform + AWS stack (LocalStack — $0, resume-worthy) -Use [LocalStack](https://localstack.cloud) (free community edition) in Docker. Point Terraform's `aws` provider at it via `endpoint` overrides + dummy credentials — same HCL as real AWS, zero cost, safe to break. +Use [LocalStack](https://localstack.cloud) (free tier — needs a free account and an +auth token since the community and pro images merged in 2026.3.0) in Docker. Point Terraform's `aws` provider at it via `endpoint` overrides + dummy credentials — same HCL as real AWS, zero cost, safe to break. Resume line target: "Terraform, AWS (VPC, ECS/Fargate, S3, IAM, Lambda, DynamoDB), IaC, CI/CD." From 85ae3b535652c6f71f9817b385ce26a6c68562d8 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Sun, 20 Sep 2026 23:02:20 -0700 Subject: [PATCH 05/14] Add dynamodb --- .gitignore | 4 + LOCALSTACK.md | 172 ----------------------------------- PLAN.md | 56 ------------ docker-compose.yml | 2 +- docs/reference/aws-cli.md | 26 ++++++ docs/reference/localstack.md | 35 +++++++ docs/reference/terraform.md | 31 +++++++ infra/main.tf | 15 +++ infra/provider.tf | 1 + 9 files changed, 113 insertions(+), 229 deletions(-) delete mode 100644 LOCALSTACK.md delete mode 100644 PLAN.md create mode 100644 docs/reference/aws-cli.md create mode 100644 docs/reference/localstack.md create mode 100644 docs/reference/terraform.md diff --git a/.gitignore b/.gitignore index fa864c7..ff9bb1b 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,7 @@ next-env.d.ts *.tfstate *.tfstate.* *.tfvars + +# planning docs (local guidance only) +docs/stages +PLAN.md diff --git a/LOCALSTACK.md b/LOCALSTACK.md deleted file mode 100644 index 7643ed1..0000000 --- a/LOCALSTACK.md +++ /dev/null @@ -1,172 +0,0 @@ -Stage 1 -Docker - ↓ -LocalStack - ↓ -Terraform - ↓ -S3 - -Stage 2 -S3 + DynamoDB - -Stage 3 -IAM - -Stage 4 -Lambda - -Stage 5 -VPC + subnets + security groups - -Stage 6 -SSM parameters - -Stage 7 -Terraform remote state - -Stage 8 -modules - -Stage 9 -CI/CD - -Stage 10 -ECR/ECS concepts - -Stage 1 — Local infrastructure foundation -Goal: get comfortable with the toolchain itself. Set up Docker, LocalStack, Terraform, and one S3 bucket. Learn what init, plan, apply, destroy, provider, resource, state, variable, and output mean. -Done when: you can start LocalStack, create S3 with Terraform, inspect it, destroy it, and explain every step. -Stage 2 — Application data layer -Goal: understand the difference between object storage and database storage. Keep S3 for files and add DynamoDB for structured application data. -Done when: your app can conceptually say “file goes to S3, metadata/record goes to DynamoDB,” and you understand keys, tables, items, and basic access patterns. -Stage 3 — Permissions and identity -Goal: stop thinking of cloud resources as globally accessible. Add IAM roles and least-privilege policies for the backend/Lambda. -Done when: you can explain which service is allowed to access which bucket/table and why you should avoid broad * permissions. -Stage 4 — Event-driven compute -Goal: understand serverless execution. Add a Lambda that performs one small useful task, ideally tied to your app. -Example: upload to S3 → Lambda processes metadata → writes to DynamoDB. -Done when: you understand events, handlers, execution roles, logs, and when Lambda makes sense versus a normal backend server. -Stage 5 — Networking -Goal: understand how AWS resources communicate securely. Add a VPC, subnets, route concepts, and security groups. -Done when: you can explain VPC vs subnet vs route table vs security group, and which resources should be public or private. -Stage 6 — Configuration and secrets -Goal: remove environment-specific values from source code. Add SSM Parameter Store for configuration and secret-like values. -Done when: your app reads configuration from the environment/SSM rather than hardcoded values, and you understand why secrets should not live in Git or Terraform state. -Stage 7 — Terraform state architecture -Goal: understand how Terraform itself is managed in a team environment. Move from local state toward an S3-backed state model and learn locking. -Done when: you understand why state matters, why teams need shared state, and why simultaneous applies are dangerous. -Stage 8 — Terraform structure and modules -Goal: refactor working infrastructure instead of building abstractions too early. Split networking, storage, IAM, and compute into reusable modules. -Done when: the root Terraform config is small and readable, and you understand module inputs, outputs, and dependencies. -Stage 9 — CI/CD and validation -Goal: make infrastructure changes repeatable and safer. Add automated terraform fmt, validate, tests, and optionally a LocalStack-based plan/test workflow in GitHub Actions. -Done when: every push can automatically verify that your infrastructure code is valid before merging. -Stage 10 — Containers and real deployment architecture -Goal: connect your application container to AWS-style deployment concepts. Dockerize the backend properly and learn ECR, ECS, task definitions, services, and Fargate. If LocalStack free tier limits this, document the architecture rather than forcing emulation. -Done when: you can explain the flow: code → Docker image → ECR → ECS task/service → Fargate. -Stage 11 — Production hardening -Goal: make the architecture look like something you would actually deploy. Add logging, tagging, environment separation, health checks, sensible IAM boundaries, backup considerations, and failure handling. -Done when: you can explain how dev/staging/prod would differ and what would need to change before using real AWS. -Stage 12 — Real AWS migration plan -Goal: prove that LocalStack was a development environment, not the final architecture. Document exactly what would change to deploy to AWS: real credentials, provider endpoints removed, remote state bucket, secrets handling, real ECS/ECR, networking, monitoring, and cost controls. -Done when: you could migrate the project to AWS without redesigning it from scratch. - -Stage 1 — your step list - -0. Install AWS CLI (needed for step 8 verification) -brew install awscli -Verify: -aws --version - -1. Install Terraform -The terraform formula was removed from homebrew-core (BUSL license change) — `brew install terraform` now errors with "No available formula". Use the official HashiCorp tap: -brew tap hashicorp/tap -brew install hashicorp/tap/terraform -Verify: -terraform -version -Expect: version output (1.16.3 or later), no error. -Later upgrades: brew upgrade hashicorp/tap/terraform - -2. Add LocalStack to docker-compose.yml -Done. `localstack` service added, plus a top-level `volumes:` block. - -2b. LocalStack auth token (required as of 2026) -LocalStack merged the community and pro images into one as of 2026.3.0. `localstack/localstack:latest` -now refuses to start without a token — the container exits with code 55 and logs -"License activation failed! No credentials were found in the environment." -The free tier is still free, but it needs an account. - -Sign up at https://app.localstack.cloud, copy the auth token, and put it in a gitignored root .env: - -LOCALSTACK_AUTH_TOKEN=ls-xxxxxxxx - -docker-compose.yml reads it via ${LOCALSTACK_AUTH_TOKEN:?...}, so a missing token fails at -`docker compose config` time with a clear message instead of a mystery exit 55. -Never commit the token. Root .env is covered by .gitignore line 34. - -Alternative if you ever want to avoid the account: pin `localstack/localstack:4.4.0`, the last -pre-merge community release. Frozen 2025 feature set, and the `lstk` CLI notes below won't apply. - -3. Start LocalStack -docker compose up -d localstack -Verify container healthy: -docker compose ps -curl http://localhost:4566/_localstack/health -Expect: JSON showing "s3": "available" (or "running"). - -4. Create infra/ dir, provider.tf, main.tf -I'll write these once you say go — will explain each block before writing. -Provider block targets LocalStack manually (writing it by hand rather than using the `lstk terraform` wrapper, since the point of Stage 1 is understanding what `provider` does): - -provider "aws" { - access_key = "test" - secret_key = "test" - region = "us-east-1" - skip_credentials_validation = true - skip_metadata_api_check = true - - endpoints { - s3 = "http://s3.localhost.localstack.cloud:4566" - } -} - -Note: S3 needs the s3.localhost.localstack.cloud hostname for virtual-hosted-style addressing. Other services use plain http://localhost:4566. - -5. terraform init -Run inside infra/. Downloads AWS provider plugin. Expect: "Terraform has been successfully initialized!" - -6. terraform plan -Shows 1 resource to add (the S3 bucket). Nothing created yet — dry run. - -7. terraform apply -Type yes when prompted. Creates bucket inside LocalStack. - -8. Verify bucket exists — two ways -The AWS CLI refuses to sign a request with no credentials at all, even though LocalStack -ignores the values. So pass dummy creds and a region (same reason provider.tf sets them): - -AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test \ - aws --endpoint-url=http://localhost:4566 --region us-west-2 s3 ls - -Without them: "An error occurred (NoCredentials): Unable to locate credentials." - -To stop prefixing every command, set up a named profile once (endpoint_url in a profile -needs AWS CLI v2.13+): - -aws configure set aws_access_key_id test --profile localstack -aws configure set aws_secret_access_key test --profile localstack -aws configure set region us-west-2 --profile localstack -aws configure set endpoint_url http://localhost:4566 --profile localstack - -then: aws --profile localstack s3 ls - -(needs the aws CLI from step 0. The old `awslocal` wrapper is deprecated — as of 2026-09-08 LocalStack replaced the Python localstack CLI and all its wrappers (awslocal, tflocal, samlocal, cdklocal) with a single Go binary, `lstk`. If you install it, the equivalent is `lstk awslocal s3 ls`.) -or check state: -terraform show -Expect: bucket name listed either way. - -9. terraform destroy -Type yes. Confirm bucket gone via the same aws s3 ls command from step 8 — expect empty. -destroy deletes the bucket inside LocalStack and updates terraform.tfstate. It does NOT -touch provider.tf or main.tf — terraform apply rebuilds the bucket from them any time. \ No newline at end of file diff --git a/PLAN.md b/PLAN.md deleted file mode 100644 index befd6e3..0000000 --- a/PLAN.md +++ /dev/null @@ -1,56 +0,0 @@ -# FirstWaveAI — Project Plan - -## Goal order - -1. Learn Terraform + AWS, free, resume-worthy — using LocalStack (no real AWS account spend, no deploy target needed). -2. After that, upgrade app itself (features, code quality). - -Rationale: app already dockerized (backend FastAPI, frontend Next.js, docker-compose) — used as the thing infra-as-code manages, but goal is the skill/resume line, not actually running this in production AWS. - ---- - -## Phase 1: Terraform + AWS stack (LocalStack — $0, resume-worthy) - -Use [LocalStack](https://localstack.cloud) (free tier — needs a free account and an -auth token since the community and pro images merged in 2026.3.0) in Docker. Point Terraform's `aws` provider at it via `endpoint` overrides + dummy credentials — same HCL as real AWS, zero cost, safe to break. - -Resume line target: "Terraform, AWS (VPC, ECS/Fargate, S3, IAM, Lambda, DynamoDB), IaC, CI/CD." - -- [ ] Run LocalStack via docker-compose, confirm `awslocal`/`tflocal` or provider endpoint override works -- [ ] Terraform CLI, core concepts: providers, resources, variables, outputs, state, modules -- [ ] Remote state: S3 backend + DynamoDB lock table -- [ ] IAM: roles, policies, least-privilege for a service -- [ ] Networking: VPC, subnets, security groups (community edition supports these) -- [ ] Storage/data: S3 buckets, DynamoDB tables -- [ ] Compute: Lambda function; ECS (Fargate) task/service if LocalStack community supports it well enough — else document as Pro-only gap -- [ ] Containerize this app's images conceptually as Terraform-managed ECR repos (LocalStack ECR support, or skip ECR and just show Dockerfile + task def wiring) -- [ ] Secrets: SSM Parameter Store for backend `.env` values -- [ ] Terraform modules: refactor above into a reusable module (shows structuring skill, not just single-file HCL) - -Known gaps in free LocalStack (don't chase these, not worth cost): ALB, Route53, ACM — networking/LB edge is weak/paid-tier. Skip; not needed since there's no real deploy target. - -Resources: Terraform docs (registry.terraform.io), LocalStack docs, "Terraform Up & Running" (optional book). - ---- - -## Phase 2: Ops polish (still $0, still resume-worthy) - -- [ ] CI/CD: GitHub Actions running `terraform plan`/`apply` against LocalStack in a CI job — real, showable pipeline, zero cost -- [ ] Environments: dev/prod split via Terraform workspaces or dir-per-env -- [ ] Write up: short README in a new `infra/` dir explaining the stack — this is often what resume/portfolio reviewers actually read - ---- - -## Phase 3: App upgrades (after infra done) - -- [ ] Review backend agents pipeline (`backend/agents`) for correctness/latency -- [ ] Frontend UX polish (`frontend/src`) -- [ ] Add tests (backend + frontend currently look untested) -- [ ] Revisit README's stated features vs actual implementation, close gaps - ---- - -## Notes - -- No real AWS account needed for Phase 1–2 — LocalStack covers the resume-worthy stack at $0. -- If later actually wanting a live deployed demo, that's a separate future decision (real AWS Free Tier) — not part of this plan. diff --git a/docker-compose.yml b/docker-compose.yml index 68e3acf..110e058 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: - "4566:4566" environment: LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN in .env} - SERVICES: s3 + SERVICES: s3,dynamodb DEBUG: "0" AWS_DEFAULT_REGION: us-west-2 volumes: diff --git a/docs/reference/aws-cli.md b/docs/reference/aws-cli.md new file mode 100644 index 0000000..041d92b --- /dev/null +++ b/docs/reference/aws-cli.md @@ -0,0 +1,26 @@ +# AWS CLI reference (against LocalStack) + +LocalStack ignores credential values but the CLI still refuses to sign a request with none at all. Pass dummy creds + a region: + +``` +AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test \ + aws --endpoint-url=http://localhost:4566 --region us-west-2 s3 ls +``` + +Without them: `An error occurred (NoCredentials): Unable to locate credentials.` + +## Named profile (avoids prefixing every command) + +Needs AWS CLI v2.13+ (for `endpoint_url` in a profile): + +``` +aws configure set aws_access_key_id test --profile localstack +aws configure set aws_secret_access_key test --profile localstack +aws configure set region us-west-2 --profile localstack +aws configure set endpoint_url http://localhost:4566 --profile localstack +``` + +Then: +``` +aws --profile localstack s3 ls +``` diff --git a/docs/reference/localstack.md b/docs/reference/localstack.md new file mode 100644 index 0000000..bb3e8af --- /dev/null +++ b/docs/reference/localstack.md @@ -0,0 +1,35 @@ +# LocalStack reference + +## Auth token (required as of 2026) + +LocalStack merged the community and pro images into one as of 2026.3.0. `localstack/localstack:latest` refuses to start without a token — container exits with code 55 and logs "License activation failed! No credentials were found in the environment." Free tier is still free, but needs an account. + +Sign up at https://app.localstack.cloud, copy the auth token, put it in a gitignored root `.env`: + +``` +LOCALSTACK_AUTH_TOKEN=ls-xxxxxxxx +``` + +`docker-compose.yml` reads it via `${LOCALSTACK_AUTH_TOKEN:?...}`, so a missing token fails at `docker compose config` time with a clear message instead of a mystery exit 55. Never commit the token — root `.env` is gitignored. + +Alternative if you want to avoid the account: pin `localstack/localstack:4.4.0`, the last pre-merge community release. Frozen 2025 feature set — the `lstk` CLI notes below won't apply. + +## Health check + +``` +curl http://localhost:4566/_localstack/health +``` +Expect JSON with each service's status, e.g. `"s3": "available"`. + +## CLI tooling + +The old `awslocal`/`tflocal`/`samlocal`/`cdklocal` Python wrappers are deprecated — as of 2026-09-08 LocalStack replaced them with a single Go binary, `lstk`. Equivalent of `awslocal s3 ls` is `lstk awslocal s3 ls`. + +## Known free-tier gaps + +ALB, Route53, ACM — networking/LB edge is weak or paid-tier. Not worth chasing since there's no real deploy target. + +## Endpoints + +- Most services: `http://localhost:4566` +- S3 (virtual-hosted-style): `http://s3.localhost.localstack.cloud:4566` diff --git a/docs/reference/terraform.md b/docs/reference/terraform.md new file mode 100644 index 0000000..7b438b1 --- /dev/null +++ b/docs/reference/terraform.md @@ -0,0 +1,31 @@ +# Terraform reference + +## Core commands + +- `terraform init` — downloads provider plugins, sets up backend. Run once per config dir (and after provider/backend changes). +- `terraform plan` — dry run, shows what would change. Creates nothing. +- `terraform apply` — applies the plan, prompts `yes`. Creates/updates real (or LocalStack) resources. +- `terraform destroy` — tears down everything Terraform manages in the current state. Prompts `yes`. +- `terraform show` — prints current state in human-readable form. + +## Provider block against LocalStack + +```hcl +provider "aws" { + access_key = "test" + secret_key = "test" + region = "us-east-1" + skip_credentials_validation = true + skip_metadata_api_check = true + + endpoints { + s3 = "http://s3.localhost.localstack.cloud:4566" + } +} +``` + +Add more `endpoints { ... }` entries per service as new resources are introduced (`dynamodb`, `iam`, `lambda`, etc.) — see `docs/reference/localstack.md` for endpoint conventions. + +## State + +`terraform.tfstate` is the source of truth Terraform diffs against — not the `.tf` files. `destroy` updates state; it never edits `.tf` files, so `apply` can always rebuild from them. Stage 7 covers moving state to a remote (S3) backend. diff --git a/infra/main.tf b/infra/main.tf index ad2a208..4ad7398 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -5,3 +5,18 @@ resource "aws_s3_bucket" "uploads" { output "bucket_name" { value = aws_s3_bucket.uploads.id } + +resource "aws_dynamodb_table" "uploads_metadata" { + name = "uploads-metadata" + billing_mode = "PAY_PER_REQUEST" + hash_key = "id" + + attribute { + name = "id" + type = "S" + } +} + +output "table_name" { + value = aws_dynamodb_table.uploads_metadata.name +} diff --git a/infra/provider.tf b/infra/provider.tf index ec1c1e6..b8cb611 100644 --- a/infra/provider.tf +++ b/infra/provider.tf @@ -18,5 +18,6 @@ provider "aws" { endpoints { s3 = "http://s3.localhost.localstack.cloud:4566" s3control = "http://localhost.localstack.cloud:4566" + dynamodb = "http://localhost:4566" } } From d0d1fcc1ed46d116920ccf82e0585736ecfbd905 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Mon, 21 Sep 2026 22:45:45 -0700 Subject: [PATCH 06/14] Add Lambda wiring S3 uploads to DynamoDB metadata table --- .gitignore | 1 + docker-compose.yml | 2 +- infra/.terraform.lock.hcl | 55 ++++++++++++++++------- infra/lambda/handler.py | 44 ++++++++++++++++++ infra/main.tf | 94 +++++++++++++++++++++++++++++++++++++++ infra/provider.tf | 6 +++ 6 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 infra/lambda/handler.py diff --git a/.gitignore b/.gitignore index ff9bb1b..75827c5 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ next-env.d.ts *.tfstate *.tfstate.* *.tfvars +**/lambda/handler.zip # planning docs (local guidance only) docs/stages diff --git a/docker-compose.yml b/docker-compose.yml index 110e058..3979327 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: - "4566:4566" environment: LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN in .env} - SERVICES: s3,dynamodb + SERVICES: s3,dynamodb,iam,lambda,logs DEBUG: "0" AWS_DEFAULT_REGION: us-west-2 volumes: diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index d591841..13af87d 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -1,26 +1,47 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.1" + constraints = "~> 2.4" + hashes = [ + "h1:aLNmq6dc3cDcqZc8s/8eKtn0I+UQXyJMGrmo4rRFtNw=", + "zh:03de290604114a89fcd45c2e5bc7787d5a1ebfc5f964fb5989306bea7a4c79ec", + "zh:0a7d69dc9fbbc48960bc2f04588c8fb1bd92c78a8f306566b7fb17fc4a4f2058", + "zh:4df1f3981379c35f1757da957470f7f7724496b57219da3485177cf1647bdf59", + "zh:50f0e72ba53bfe6e11b03b7fc899e1f72536354381d302a743a274ae2a3f45f4", + "zh:5c4e15a04c98e2a8cafb1cd9632b48ad318051e8462d105b1153006277985c35", + "zh:66069e604bcf5c4af0278e15997d9e6bd755c54fb3801d78885838b889729c5f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:979765db3f42601870ab377104ba70befb029547b278337ec4ada980e3582de6", + "zh:b165254da774f49945a73fbccc3ef1b63d70ea00a98fa9e14716665ba80ecaad", + "zh:c0bb2697b525da9fec4511f569ed2bd2b42f25d5c1f9fa00f8f3645b50cfc2e9", + "zh:c48f6695d12df0d0fa5231f7c1b8d518a4feae91a733fdd35a5804af3a303831", + "zh:d589954c93f075180c9f4e2ce91780d5edcb56dfd0d3cda8ba08e13c10b52249", + "zh:f5792ed06da65d0daf7ca3711f5399ff78c7cb4400afe53fbce9a926fbde4477", + ] +} + provider "registry.terraform.io/hashicorp/aws" { - version = "6.65.0" + version = "6.66.0" constraints = "~> 6.0" hashes = [ - "h1:/VgIzAOR/v+p135IFsJjYT7q3pA24yE3lZGBV0Otqq0=", - "zh:15b5bd81119965363893197b3b6065bdf46888b93c536623fd113b5505c375be", - "zh:16409fd045116a31b28adce98fcaaa56a7c01487369713e4a2a04af1cfa96fa3", - "zh:422ea20ef4be8e5b942118d1da61cbde818cadb512ec1132306d65120f983917", - "zh:42cda6703a6a51585c2cb2b8b3ab3a7c80a3ee08838138be8ecaa6b97b1d74d8", - "zh:718a880d81bfd9af7e297ed3d7bf98d1febebe8b9ebe3333854a4c17f2c4de09", - "zh:74e538a8ff4ea27b2040be426cdd2b952725883f5b45c8c03b27eff7d82b40f8", - "zh:876bf62e56a41e0c7a514652e22a41246e7c1d67be3b2c1b68553fa3a8dae6d7", - "zh:8d571e06d78b91b28faa7fafee99dee920d4f7a50f07ec9cd21695a385bcc0d5", - "zh:93154e33f4cbd39825a92a230642082e7b2b8b058c27cf93588ee6824aa1c294", - "zh:935f9523c940dc5795ce8afac37aa8a2ed06c0012196ab39b1de2ea26acc129e", + "h1:OnLj4nhqJnEcUzyyRKUjp1FgWG00Y8maikJEYSf9Zjw=", + "zh:156fe7164a3d26ef6b35734c43e99fb198df90575ed897d1182b8e930b8cd523", + "zh:1af52b22b35be00f8d16e3ebebff9fa699ec4db2ef69e6032ba5c536f80c03d9", + "zh:2545a8478bd551fdc9694f6cc1a1ad24617f6736f8bde0ad6cae90987c65380f", + "zh:4070db1ee369ccb41cb610bfd887386bc0a9b9ecad60aeb4dbce58443d2519dd", + "zh:53da7d3c1840ef875c7d34e967732502a64fe677af0e78824773d4c15a8fe740", + "zh:576a93a28bf611a4de2a2e6ced697a41d5126b8fd31d30782b16797e410a9706", + "zh:58fed5fa9a033355b9d4f3092c817b70d934100e0d8678d6e4c93f3c9493d4e4", + "zh:6a9ca2f24e2ee9156dd785d159a850b35d190e9cf7eca21cb9582970c2db80cd", + "zh:729edd30f99cc16009deba5c013265b0c81eda261a3d0821cbd011d3287fd230", + "zh:7ae460049b75bd4aefee465ef7c53a01ac2df46d4d3e3ac00824afa8b5cb83fb", + "zh:9051fa85c8034ade8a57a5c6f232fd33da28f3800bb5aa40bc8625dbc5e27632", + "zh:906547e4319805e7acf7fbdf2bac28a4b1a7370790a2a430c7adb1b29bb934eb", + "zh:998f27410a66158a35ee5ed142c27e5b21fe8601941da55da2157f8042d6dcca", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9bffe18e907e04d98d7d6b9a6a4c2381448e365441df423e90e1eeaf3a79fd2f", - "zh:9ddfdefef226c8ff3c8de03d8df23ab3199fed9f0684618a62489e0e90a21cab", - "zh:9eab3abf041fe8e1ce5959fda9c20ddfaf331b7c29ff707e914451abce7841af", - "zh:ed749702f6c56b26a390a52bfd31d3bdf7f0af800bc16244276ca71d6f541e87", - "zh:f304df223a0bc3e840a806a5dffb75e6b2b75c879053dc4ebd0228484923ee59", + "zh:9c1804eff1dda0446dc2d215231015bb65a2fc6c3b7ba24584fe45f1ddd3fa9f", + "zh:b03ff5efdee310502aaaeb460144dc059bce72a0d8217e6b989099ef8aef9283", ] } diff --git a/infra/lambda/handler.py b/infra/lambda/handler.py new file mode 100644 index 0000000..6ca8a2e --- /dev/null +++ b/infra/lambda/handler.py @@ -0,0 +1,44 @@ +import os +import urllib.parse +from datetime import datetime, timezone + +import boto3 + +dynamodb = boto3.resource("dynamodb", endpoint_url=os.environ.get("DYNAMODB_ENDPOINT_URL")) +s3 = boto3.client("s3", endpoint_url=os.environ.get("S3_ENDPOINT_URL")) + +TABLE_NAME = os.environ.get("TABLE_NAME", "uploads-metadata") + + +def handler(event, context): + table = dynamodb.Table(TABLE_NAME) + + for record in event.get("Records", []): + s3_info = record["s3"] + bucket = s3_info["bucket"]["name"] + key = urllib.parse.unquote_plus(s3_info["object"]["key"]) + size_bytes = s3_info["object"].get("size") + uploaded_at = record.get("eventTime", datetime.now(timezone.utc).isoformat()) + + item = { + "id": key, + "filename": key, + "size_bytes": size_bytes, + "uploaded_at": uploaded_at, + } + + try: + head = s3.head_object(Bucket=bucket, Key=key) + content_type = head.get("ContentType") + if content_type: + item["content_type"] = content_type + if size_bytes is None: + item["size_bytes"] = head.get("ContentLength") + except Exception: + pass + + item = {k: v for k, v in item.items() if v is not None} + + table.put_item(Item=item) + + return {"statusCode": 200} diff --git a/infra/main.tf b/infra/main.tf index 4ad7398..96dd5fd 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -20,3 +20,97 @@ resource "aws_dynamodb_table" "uploads_metadata" { output "table_name" { value = aws_dynamodb_table.uploads_metadata.name } + +resource "aws_iam_role" "lambda_exec" { + name = "uploads-metadata-lambda-exec" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_role_policy" "lambda_exec" { + name = "uploads-metadata-lambda-exec-policy" + role = aws_iam_role.lambda_exec.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "dynamodb:PutItem" + Resource = aws_dynamodb_table.uploads_metadata.arn + }, + { + Effect = "Allow" + Action = "s3:GetObject" + Resource = "${aws_s3_bucket.uploads.arn}/*" + }, + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "arn:aws:logs:*:*:*" + } + ] + }) +} + +data "archive_file" "lambda_zip" { + type = "zip" + source_file = "${path.module}/lambda/handler.py" + output_path = "${path.module}/lambda/handler.zip" +} + +resource "aws_lambda_function" "uploads_metadata" { + function_name = "uploads-metadata-writer" + role = aws_iam_role.lambda_exec.arn + filename = data.archive_file.lambda_zip.output_path + source_code_hash = data.archive_file.lambda_zip.output_base64sha256 + handler = "handler.handler" + runtime = "python3.12" + timeout = 10 + + environment { + variables = { + TABLE_NAME = aws_dynamodb_table.uploads_metadata.name + DYNAMODB_ENDPOINT_URL = "http://host.docker.internal:4566" + S3_ENDPOINT_URL = "http://host.docker.internal:4566" + } + } +} + +output "lambda_function_name" { + value = aws_lambda_function.uploads_metadata.function_name +} + +resource "aws_lambda_permission" "allow_s3" { + statement_id = "AllowS3Invoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.uploads_metadata.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.uploads.arn +} + +resource "aws_s3_bucket_notification" "uploads" { + bucket = aws_s3_bucket.uploads.id + + lambda_function { + lambda_function_arn = aws_lambda_function.uploads_metadata.arn + events = ["s3:ObjectCreated:*"] + } + + depends_on = [aws_lambda_permission.allow_s3] +} diff --git a/infra/provider.tf b/infra/provider.tf index b8cb611..57d1b03 100644 --- a/infra/provider.tf +++ b/infra/provider.tf @@ -4,6 +4,10 @@ terraform { source = "hashicorp/aws" version = "~> 6.0" } + archive = { + source = "hashicorp/archive" + version = "~> 2.4" + } } } @@ -19,5 +23,7 @@ provider "aws" { s3 = "http://s3.localhost.localstack.cloud:4566" s3control = "http://localhost.localstack.cloud:4566" dynamodb = "http://localhost:4566" + iam = "http://localhost:4566" + lambda = "http://localhost:4566" } } From d129fbbd842853cdcb0b7b6d729e797ea89a9d4d Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Tue, 22 Sep 2026 23:54:59 -0700 Subject: [PATCH 07/14] Implement VPC and subnets --- docker-compose.yml | 2 +- infra/network.tf | 56 ++++++++++++++++++++++++++++++++++++++++++++++ infra/provider.tf | 2 ++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 infra/network.tf diff --git a/docker-compose.yml b/docker-compose.yml index 3979327..b937346 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: - "4566:4566" environment: LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN in .env} - SERVICES: s3,dynamodb,iam,lambda,logs + SERVICES: s3,dynamodb,iam,lambda,logs,ec2,ssm DEBUG: "0" AWS_DEFAULT_REGION: us-west-2 volumes: diff --git a/infra/network.tf b/infra/network.tf new file mode 100644 index 0000000..71e7628 --- /dev/null +++ b/infra/network.tf @@ -0,0 +1,56 @@ +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" + + tags = { + Name = "firstwave-main" + } +} + +resource "aws_subnet" "public" { + vpc_id = aws_vpc.main.id + cidr_block = "10.0.1.0/24" + map_public_ip_on_launch = true + + tags = { + Name = "firstwave-public" + } +} + +resource "aws_subnet" "private" { + vpc_id = aws_vpc.main.id + cidr_block = "10.0.2.0/24" + + tags = { + Name = "firstwave-private" + } +} + +resource "aws_internet_gateway" "main" { + vpc_id = aws_vpc.main.id + + tags = { + Name = "firstwave-igw" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.main.id + } + + tags = { + Name = "firstwave-public-rt" + } +} + +resource "aws_route_table_association" "public" { + subnet_id = aws_subnet.public.id + route_table_id = aws_route_table.public.id +} + +# Private subnet intentionally gets no route table association — no outbound +# internet access. NAT gateway deferred until something in the private +# subnet actually needs it. diff --git a/infra/provider.tf b/infra/provider.tf index 57d1b03..ac4fa34 100644 --- a/infra/provider.tf +++ b/infra/provider.tf @@ -25,5 +25,7 @@ provider "aws" { dynamodb = "http://localhost:4566" iam = "http://localhost:4566" lambda = "http://localhost:4566" + ec2 = "http://localhost:4566" + ssm = "http://localhost:4566" } } From 8993454e22d8736fc7f5b5b07ba6b4f8f72cb17e Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Wed, 23 Sep 2026 00:00:04 -0700 Subject: [PATCH 08/14] Add security to network --- infra/network.tf | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/infra/network.tf b/infra/network.tf index 71e7628..f2dea7d 100644 --- a/infra/network.tf +++ b/infra/network.tf @@ -54,3 +54,29 @@ resource "aws_route_table_association" "public" { # Private subnet intentionally gets no route table association — no outbound # internet access. NAT gateway deferred until something in the private # subnet actually needs it. + +resource "aws_security_group" "app" { + name = "firstwave-app" + description = "Backend app traffic (port 8000), scoped to the VPC" + vpc_id = aws_vpc.main.id + + ingress { + description = "Backend API port, VPC-internal only" + from_port = 8000 + to_port = 8000 + protocol = "tcp" + cidr_blocks = [aws_vpc.main.cidr_block] + } + + egress { + description = "Outbound within the VPC" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = [aws_vpc.main.cidr_block] + } + + tags = { + Name = "firstwave-app" + } +} From 2bc540b7c33df40c00d5ebc0efb1722ff39ecfa7 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Wed, 23 Sep 2026 14:52:27 -0700 Subject: [PATCH 09/14] Migrate to SSM and implement VPC --- backend/agents/graph.py | 33 ++++++++++++++++++++++---------- backend/main.py | 26 ++++++++++++++++--------- backend/requirements.txt | 1 + docker-compose.yml | 1 + infra/ssm.tf | 35 ++++++++++++++++++++++++++++++++++ infra/terraform.tfvars.example | 3 +++ 6 files changed, 80 insertions(+), 19 deletions(-) create mode 100644 infra/ssm.tf create mode 100644 infra/terraform.tfvars.example diff --git a/backend/agents/graph.py b/backend/agents/graph.py index add211d..17139e6 100644 --- a/backend/agents/graph.py +++ b/backend/agents/graph.py @@ -9,12 +9,11 @@ import json import logging import traceback -from pathlib import Path from typing import Literal, Annotated, Callable, TypeVar -from functools import wraps +from functools import wraps, lru_cache from typing_extensions import TypedDict -from dotenv import load_dotenv +import boto3 from langgraph.graph import StateGraph, START, END from langgraph.graph.message import add_messages from langchain_groq import ChatGroq @@ -24,9 +23,23 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -# Load environment variables from .env file -env_path = Path(__file__).parent.parent / ".env" -load_dotenv(env_path) +# SSM (LocalStack in dev; real AWS SSM in a deployed environment) +SSM_ENDPOINT_URL = os.getenv("SSM_ENDPOINT_URL", "http://localhost:4566") + +_ssm_client = boto3.client( + "ssm", + region_name="us-west-2", + endpoint_url=SSM_ENDPOINT_URL, + aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID", "test"), + aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", "test"), +) + + +@lru_cache(maxsize=None) +def get_ssm_parameter(name: str) -> str: + """Fetch and cache a config value from SSM Parameter Store.""" + response = _ssm_client.get_parameter(Name=name, WithDecryption=True) + return response["Parameter"]["Value"] # ============================================================================= @@ -123,12 +136,12 @@ def safe_json_parse(content: str, agent_name: str) -> dict | None: def get_model(): """Get the Groq model instance.""" - api_key = os.getenv("GROQ_API_KEY") + api_key = get_ssm_parameter("/firstwave/backend/GROQ_API_KEY") if not api_key: - raise ValueError("GROQ_API_KEY not found. Please set it in backend/.env") - + raise ValueError("GROQ_API_KEY not found in SSM at /firstwave/backend/GROQ_API_KEY") + return ChatGroq( - model="llama-3.3-70b-versatile", # Free tier model + model="openai/gpt-oss-120b", # llama-3.3-70b-versatile was decommissioned by Groq temperature=0.1, api_key=api_key, timeout=30, # Add timeout to prevent hanging diff --git a/backend/main.py b/backend/main.py index 1c4c8d1..3d97622 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,10 +11,9 @@ from pydantic import BaseModel, field_validator from agents import dispatcher_graph +from agents.graph import get_ssm_parameter -# Fish Audio TTS configuration -FISH_AUDIO_API_KEY = os.getenv("FISH_AUDIO_API_KEY") -FISH_AUDIO_VOICE_ID = os.getenv("FISH_AUDIO_VOICE_ID", "") # Optional: specific voice +# Fish Audio TTS configuration (keys/voice ID come from SSM, see agents/graph.py) FISH_AUDIO_API_URL = "https://api.fish.audio/v1/tts" # Configure logging @@ -159,18 +158,27 @@ async def text_to_speech(request: TTSRequest): Returns audio bytes (MP3 format) that can be played directly. """ - if not FISH_AUDIO_API_KEY: + try: + fish_audio_api_key = get_ssm_parameter("/firstwave/backend/FISH_AUDIO_API_KEY") + except Exception: + fish_audio_api_key = None + + if not fish_audio_api_key: logger.error("Fish Audio API key not configured") raise APIError( status_code=500, message="TTS service not configured", - details="FISH_AUDIO_API_KEY environment variable is not set" + details="FISH_AUDIO_API_KEY not found in SSM at /firstwave/backend/FISH_AUDIO_API_KEY" ) - + logger.info(f"TTS request (text length: {len(request.text)})") - + # Use provided voice_id or fall back to configured default - voice_id = request.voice_id or FISH_AUDIO_VOICE_ID + try: + default_voice_id = get_ssm_parameter("/firstwave/backend/FISH_AUDIO_VOICE_ID") + except Exception: + default_voice_id = "" + voice_id = request.voice_id or default_voice_id try: async with httpx.AsyncClient(timeout=30.0) as client: @@ -187,7 +195,7 @@ async def text_to_speech(request: TTSRequest): response = await client.post( FISH_AUDIO_API_URL, headers={ - "Authorization": f"Bearer {FISH_AUDIO_API_KEY}", + "Authorization": f"Bearer {fish_audio_api_key}", "Content-Type": "application/json", }, json=payload, diff --git a/backend/requirements.txt b/backend/requirements.txt index 4a6e228..c0a4912 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -6,3 +6,4 @@ langchain-anthropic==1.7.2 langchain-groq==1.1.3 langchain-core==1.6.3 httpx==0.28.1 +boto3==1.40.30 diff --git a/docker-compose.yml b/docker-compose.yml index b937346..e419a6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: - ./backend/.env environment: ALLOWED_ORIGINS: http://localhost:3000 + SSM_ENDPOINT_URL: http://host.docker.internal:4566 frontend: build: diff --git a/infra/ssm.tf b/infra/ssm.tf new file mode 100644 index 0000000..2d971db --- /dev/null +++ b/infra/ssm.tf @@ -0,0 +1,35 @@ +variable "groq_api_key" { + description = "Groq API key" + type = string + sensitive = true +} + +variable "fish_audio_api_key" { + description = "Fish Audio API key" + type = string + sensitive = true +} + +variable "fish_audio_voice_id" { + description = "Fish Audio voice ID (not secret)" + type = string + default = "" +} + +resource "aws_ssm_parameter" "groq_api_key" { + name = "/firstwave/backend/GROQ_API_KEY" + type = "SecureString" + value = var.groq_api_key +} + +resource "aws_ssm_parameter" "fish_audio_api_key" { + name = "/firstwave/backend/FISH_AUDIO_API_KEY" + type = "SecureString" + value = var.fish_audio_api_key +} + +resource "aws_ssm_parameter" "fish_audio_voice_id" { + name = "/firstwave/backend/FISH_AUDIO_VOICE_ID" + type = "String" + value = var.fish_audio_voice_id +} diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example new file mode 100644 index 0000000..a0cdfd1 --- /dev/null +++ b/infra/terraform.tfvars.example @@ -0,0 +1,3 @@ +groq_api_key = "your-groq-api-key-here" +fish_audio_api_key = "your_api_key_here" +fish_audio_voice_id = "optional_voice_id" From 0084c50a2386005576fef5c030f937c841dcb449 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Thu, 24 Sep 2026 22:17:40 -0700 Subject: [PATCH 10/14] Create infra bootstrap to initiate bucket --- infra-bootstrap/.terraform.lock.hcl | 26 ++++++++++++++++++++++++++ infra-bootstrap/main.tf | 15 +++++++++++++++ infra-bootstrap/provider.tf | 22 ++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 infra-bootstrap/.terraform.lock.hcl create mode 100644 infra-bootstrap/main.tf create mode 100644 infra-bootstrap/provider.tf diff --git a/infra-bootstrap/.terraform.lock.hcl b/infra-bootstrap/.terraform.lock.hcl new file mode 100644 index 0000000..66a073f --- /dev/null +++ b/infra-bootstrap/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.66.0" + constraints = "~> 6.0" + hashes = [ + "h1:OnLj4nhqJnEcUzyyRKUjp1FgWG00Y8maikJEYSf9Zjw=", + "zh:156fe7164a3d26ef6b35734c43e99fb198df90575ed897d1182b8e930b8cd523", + "zh:1af52b22b35be00f8d16e3ebebff9fa699ec4db2ef69e6032ba5c536f80c03d9", + "zh:2545a8478bd551fdc9694f6cc1a1ad24617f6736f8bde0ad6cae90987c65380f", + "zh:4070db1ee369ccb41cb610bfd887386bc0a9b9ecad60aeb4dbce58443d2519dd", + "zh:53da7d3c1840ef875c7d34e967732502a64fe677af0e78824773d4c15a8fe740", + "zh:576a93a28bf611a4de2a2e6ced697a41d5126b8fd31d30782b16797e410a9706", + "zh:58fed5fa9a033355b9d4f3092c817b70d934100e0d8678d6e4c93f3c9493d4e4", + "zh:6a9ca2f24e2ee9156dd785d159a850b35d190e9cf7eca21cb9582970c2db80cd", + "zh:729edd30f99cc16009deba5c013265b0c81eda261a3d0821cbd011d3287fd230", + "zh:7ae460049b75bd4aefee465ef7c53a01ac2df46d4d3e3ac00824afa8b5cb83fb", + "zh:9051fa85c8034ade8a57a5c6f232fd33da28f3800bb5aa40bc8625dbc5e27632", + "zh:906547e4319805e7acf7fbdf2bac28a4b1a7370790a2a430c7adb1b29bb934eb", + "zh:998f27410a66158a35ee5ed142c27e5b21fe8601941da55da2157f8042d6dcca", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c1804eff1dda0446dc2d215231015bb65a2fc6c3b7ba24584fe45f1ddd3fa9f", + "zh:b03ff5efdee310502aaaeb460144dc059bce72a0d8217e6b989099ef8aef9283", + ] +} diff --git a/infra-bootstrap/main.tf b/infra-bootstrap/main.tf new file mode 100644 index 0000000..6d69c89 --- /dev/null +++ b/infra-bootstrap/main.tf @@ -0,0 +1,15 @@ +resource "aws_s3_bucket" "tf_state" { + bucket = "firstwave-tf-state" +} + +resource "aws_s3_bucket_versioning" "tf_state" { + bucket = aws_s3_bucket.tf_state.id + + versioning_configuration { + status = "Enabled" + } +} + +output "state_bucket_name" { + value = aws_s3_bucket.tf_state.id +} diff --git a/infra-bootstrap/provider.tf b/infra-bootstrap/provider.tf new file mode 100644 index 0000000..ec1c1e6 --- /dev/null +++ b/infra-bootstrap/provider.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } +} + +provider "aws" { + access_key = "test" + secret_key = "test" + region = "us-west-2" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + + endpoints { + s3 = "http://s3.localhost.localstack.cloud:4566" + s3control = "http://localhost.localstack.cloud:4566" + } +} From ba98f89f2ed5f66ac312f1157376622cfe555fee Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Thu, 24 Sep 2026 22:25:45 -0700 Subject: [PATCH 11/14] Wire in backend to s3 bucket --- infra/provider.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/infra/provider.tf b/infra/provider.tf index ac4fa34..30cc3dd 100644 --- a/infra/provider.tf +++ b/infra/provider.tf @@ -9,6 +9,24 @@ terraform { version = "~> 2.4" } } + + backend "s3" { + bucket = "firstwave-tf-state" + key = "firstwave/terraform.tfstate" + region = "us-west-2" + use_lockfile = true + + access_key = "test" + secret_key = "test" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + skip_region_validation = true + + endpoints = { + s3 = "http://s3.localhost.localstack.cloud:4566" + } + } } provider "aws" { From b8dd0574af6875d4ee4666683d929d27bda51508 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Thu, 24 Sep 2026 22:34:44 -0700 Subject: [PATCH 12/14] Change folder structure to modules --- infra/main.tf | 121 ++++-------------- infra/modules/compute/main.tf | 60 +++++++++ infra/modules/compute/outputs.tf | 3 + infra/modules/compute/variables.tf | 37 ++++++ infra/modules/iam/main.tf | 46 +++++++ infra/modules/iam/outputs.tf | 3 + infra/modules/iam/variables.tf | 9 ++ .../networking/main.tf} | 6 +- infra/modules/networking/outputs.tf | 15 +++ infra/modules/networking/variables.tf | 17 +++ infra/modules/storage/main.tf | 14 ++ infra/modules/storage/outputs.tf | 15 +++ infra/moved.tf | 84 ++++++++++++ infra/ssm.tf | 35 ----- infra/variables.tf | 17 +++ 15 files changed, 345 insertions(+), 137 deletions(-) create mode 100644 infra/modules/compute/main.tf create mode 100644 infra/modules/compute/outputs.tf create mode 100644 infra/modules/compute/variables.tf create mode 100644 infra/modules/iam/main.tf create mode 100644 infra/modules/iam/outputs.tf create mode 100644 infra/modules/iam/variables.tf rename infra/{network.tf => modules/networking/main.tf} (92%) create mode 100644 infra/modules/networking/outputs.tf create mode 100644 infra/modules/networking/variables.tf create mode 100644 infra/modules/storage/main.tf create mode 100644 infra/modules/storage/outputs.tf create mode 100644 infra/moved.tf delete mode 100644 infra/ssm.tf create mode 100644 infra/variables.tf diff --git a/infra/main.tf b/infra/main.tf index 96dd5fd..c4ac428 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,116 +1,39 @@ -resource "aws_s3_bucket" "uploads" { - bucket = "firstwave-uploads" +module "networking" { + source = "./modules/networking" } -output "bucket_name" { - value = aws_s3_bucket.uploads.id +module "storage" { + source = "./modules/storage" } -resource "aws_dynamodb_table" "uploads_metadata" { - name = "uploads-metadata" - billing_mode = "PAY_PER_REQUEST" - hash_key = "id" - - attribute { - name = "id" - type = "S" - } -} +module "iam" { + source = "./modules/iam" -output "table_name" { - value = aws_dynamodb_table.uploads_metadata.name + dynamodb_arn = module.storage.table_arn + bucket_arn = module.storage.bucket_arn } -resource "aws_iam_role" "lambda_exec" { - name = "uploads-metadata-lambda-exec" +module "compute" { + source = "./modules/compute" - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Principal = { - Service = "lambda.amazonaws.com" - } - Action = "sts:AssumeRole" - } - ] - }) -} - -resource "aws_iam_role_policy" "lambda_exec" { - name = "uploads-metadata-lambda-exec-policy" - role = aws_iam_role.lambda_exec.id + table_name = module.storage.table_name + bucket_name = module.storage.bucket_name + bucket_arn = module.storage.bucket_arn + role_arn = module.iam.role_arn - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = "dynamodb:PutItem" - Resource = aws_dynamodb_table.uploads_metadata.arn - }, - { - Effect = "Allow" - Action = "s3:GetObject" - Resource = "${aws_s3_bucket.uploads.arn}/*" - }, - { - Effect = "Allow" - Action = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - Resource = "arn:aws:logs:*:*:*" - } - ] - }) + groq_api_key = var.groq_api_key + fish_audio_api_key = var.fish_audio_api_key + fish_audio_voice_id = var.fish_audio_voice_id } -data "archive_file" "lambda_zip" { - type = "zip" - source_file = "${path.module}/lambda/handler.py" - output_path = "${path.module}/lambda/handler.zip" +output "bucket_name" { + value = module.storage.bucket_name } -resource "aws_lambda_function" "uploads_metadata" { - function_name = "uploads-metadata-writer" - role = aws_iam_role.lambda_exec.arn - filename = data.archive_file.lambda_zip.output_path - source_code_hash = data.archive_file.lambda_zip.output_base64sha256 - handler = "handler.handler" - runtime = "python3.12" - timeout = 10 - - environment { - variables = { - TABLE_NAME = aws_dynamodb_table.uploads_metadata.name - DYNAMODB_ENDPOINT_URL = "http://host.docker.internal:4566" - S3_ENDPOINT_URL = "http://host.docker.internal:4566" - } - } +output "table_name" { + value = module.storage.table_name } output "lambda_function_name" { - value = aws_lambda_function.uploads_metadata.function_name -} - -resource "aws_lambda_permission" "allow_s3" { - statement_id = "AllowS3Invoke" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.uploads_metadata.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.uploads.arn -} - -resource "aws_s3_bucket_notification" "uploads" { - bucket = aws_s3_bucket.uploads.id - - lambda_function { - lambda_function_arn = aws_lambda_function.uploads_metadata.arn - events = ["s3:ObjectCreated:*"] - } - - depends_on = [aws_lambda_permission.allow_s3] + value = module.compute.lambda_function_name } diff --git a/infra/modules/compute/main.tf b/infra/modules/compute/main.tf new file mode 100644 index 0000000..7dcbb35 --- /dev/null +++ b/infra/modules/compute/main.tf @@ -0,0 +1,60 @@ +data "archive_file" "lambda_zip" { + type = "zip" + source_file = "${path.root}/lambda/handler.py" + output_path = "${path.root}/lambda/handler.zip" +} + +resource "aws_lambda_function" "uploads_metadata" { + function_name = "uploads-metadata-writer" + role = var.role_arn + filename = data.archive_file.lambda_zip.output_path + source_code_hash = data.archive_file.lambda_zip.output_base64sha256 + handler = "handler.handler" + runtime = "python3.12" + timeout = 10 + + environment { + variables = { + TABLE_NAME = var.table_name + DYNAMODB_ENDPOINT_URL = "http://host.docker.internal:4566" + S3_ENDPOINT_URL = "http://host.docker.internal:4566" + } + } +} + +resource "aws_lambda_permission" "allow_s3" { + statement_id = "AllowS3Invoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.uploads_metadata.function_name + principal = "s3.amazonaws.com" + source_arn = var.bucket_arn +} + +resource "aws_s3_bucket_notification" "uploads" { + bucket = var.bucket_name + + lambda_function { + lambda_function_arn = aws_lambda_function.uploads_metadata.arn + events = ["s3:ObjectCreated:*"] + } + + depends_on = [aws_lambda_permission.allow_s3] +} + +resource "aws_ssm_parameter" "groq_api_key" { + name = "/firstwave/backend/GROQ_API_KEY" + type = "SecureString" + value = var.groq_api_key +} + +resource "aws_ssm_parameter" "fish_audio_api_key" { + name = "/firstwave/backend/FISH_AUDIO_API_KEY" + type = "SecureString" + value = var.fish_audio_api_key +} + +resource "aws_ssm_parameter" "fish_audio_voice_id" { + name = "/firstwave/backend/FISH_AUDIO_VOICE_ID" + type = "String" + value = var.fish_audio_voice_id +} diff --git a/infra/modules/compute/outputs.tf b/infra/modules/compute/outputs.tf new file mode 100644 index 0000000..dd08abf --- /dev/null +++ b/infra/modules/compute/outputs.tf @@ -0,0 +1,3 @@ +output "lambda_function_name" { + value = aws_lambda_function.uploads_metadata.function_name +} diff --git a/infra/modules/compute/variables.tf b/infra/modules/compute/variables.tf new file mode 100644 index 0000000..744ad60 --- /dev/null +++ b/infra/modules/compute/variables.tf @@ -0,0 +1,37 @@ +variable "table_name" { + description = "Name of the DynamoDB table the lambda writes to" + type = string +} + +variable "bucket_name" { + description = "Name of the S3 bucket that triggers the lambda" + type = string +} + +variable "bucket_arn" { + description = "ARN of the S3 bucket that triggers the lambda" + type = string +} + +variable "role_arn" { + description = "ARN of the IAM role the lambda assumes" + type = string +} + +variable "groq_api_key" { + description = "Groq API key" + type = string + sensitive = true +} + +variable "fish_audio_api_key" { + description = "Fish Audio API key" + type = string + sensitive = true +} + +variable "fish_audio_voice_id" { + description = "Fish Audio voice ID (not secret)" + type = string + default = "" +} diff --git a/infra/modules/iam/main.tf b/infra/modules/iam/main.tf new file mode 100644 index 0000000..e892afa --- /dev/null +++ b/infra/modules/iam/main.tf @@ -0,0 +1,46 @@ +resource "aws_iam_role" "lambda_exec" { + name = "uploads-metadata-lambda-exec" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_role_policy" "lambda_exec" { + name = "uploads-metadata-lambda-exec-policy" + role = aws_iam_role.lambda_exec.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "dynamodb:PutItem" + Resource = var.dynamodb_arn + }, + { + Effect = "Allow" + Action = "s3:GetObject" + Resource = "${var.bucket_arn}/*" + }, + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "arn:aws:logs:*:*:*" + } + ] + }) +} diff --git a/infra/modules/iam/outputs.tf b/infra/modules/iam/outputs.tf new file mode 100644 index 0000000..9ef1db2 --- /dev/null +++ b/infra/modules/iam/outputs.tf @@ -0,0 +1,3 @@ +output "role_arn" { + value = aws_iam_role.lambda_exec.arn +} diff --git a/infra/modules/iam/variables.tf b/infra/modules/iam/variables.tf new file mode 100644 index 0000000..be6cf8f --- /dev/null +++ b/infra/modules/iam/variables.tf @@ -0,0 +1,9 @@ +variable "dynamodb_arn" { + description = "ARN of the DynamoDB table the lambda writes to" + type = string +} + +variable "bucket_arn" { + description = "ARN of the S3 bucket the lambda reads from" + type = string +} diff --git a/infra/network.tf b/infra/modules/networking/main.tf similarity index 92% rename from infra/network.tf rename to infra/modules/networking/main.tf index f2dea7d..d1a3fb4 100644 --- a/infra/network.tf +++ b/infra/modules/networking/main.tf @@ -1,5 +1,5 @@ resource "aws_vpc" "main" { - cidr_block = "10.0.0.0/16" + cidr_block = var.vpc_cidr tags = { Name = "firstwave-main" @@ -8,7 +8,7 @@ resource "aws_vpc" "main" { resource "aws_subnet" "public" { vpc_id = aws_vpc.main.id - cidr_block = "10.0.1.0/24" + cidr_block = var.public_subnet_cidr map_public_ip_on_launch = true tags = { @@ -18,7 +18,7 @@ resource "aws_subnet" "public" { resource "aws_subnet" "private" { vpc_id = aws_vpc.main.id - cidr_block = "10.0.2.0/24" + cidr_block = var.private_subnet_cidr tags = { Name = "firstwave-private" diff --git a/infra/modules/networking/outputs.tf b/infra/modules/networking/outputs.tf new file mode 100644 index 0000000..f4a8d40 --- /dev/null +++ b/infra/modules/networking/outputs.tf @@ -0,0 +1,15 @@ +output "vpc_id" { + value = aws_vpc.main.id +} + +output "public_subnet_id" { + value = aws_subnet.public.id +} + +output "private_subnet_id" { + value = aws_subnet.private.id +} + +output "security_group_id" { + value = aws_security_group.app.id +} diff --git a/infra/modules/networking/variables.tf b/infra/modules/networking/variables.tf new file mode 100644 index 0000000..edc0fa0 --- /dev/null +++ b/infra/modules/networking/variables.tf @@ -0,0 +1,17 @@ +variable "vpc_cidr" { + description = "CIDR block for the VPC" + type = string + default = "10.0.0.0/16" +} + +variable "public_subnet_cidr" { + description = "CIDR block for the public subnet" + type = string + default = "10.0.1.0/24" +} + +variable "private_subnet_cidr" { + description = "CIDR block for the private subnet" + type = string + default = "10.0.2.0/24" +} diff --git a/infra/modules/storage/main.tf b/infra/modules/storage/main.tf new file mode 100644 index 0000000..c1741b5 --- /dev/null +++ b/infra/modules/storage/main.tf @@ -0,0 +1,14 @@ +resource "aws_s3_bucket" "uploads" { + bucket = "firstwave-uploads" +} + +resource "aws_dynamodb_table" "uploads_metadata" { + name = "uploads-metadata" + billing_mode = "PAY_PER_REQUEST" + hash_key = "id" + + attribute { + name = "id" + type = "S" + } +} diff --git a/infra/modules/storage/outputs.tf b/infra/modules/storage/outputs.tf new file mode 100644 index 0000000..4701986 --- /dev/null +++ b/infra/modules/storage/outputs.tf @@ -0,0 +1,15 @@ +output "bucket_name" { + value = aws_s3_bucket.uploads.id +} + +output "bucket_arn" { + value = aws_s3_bucket.uploads.arn +} + +output "table_name" { + value = aws_dynamodb_table.uploads_metadata.name +} + +output "table_arn" { + value = aws_dynamodb_table.uploads_metadata.arn +} diff --git a/infra/moved.tf b/infra/moved.tf new file mode 100644 index 0000000..085518f --- /dev/null +++ b/infra/moved.tf @@ -0,0 +1,84 @@ +moved { + from = aws_vpc.main + to = module.networking.aws_vpc.main +} + +moved { + from = aws_subnet.public + to = module.networking.aws_subnet.public +} + +moved { + from = aws_subnet.private + to = module.networking.aws_subnet.private +} + +moved { + from = aws_internet_gateway.main + to = module.networking.aws_internet_gateway.main +} + +moved { + from = aws_route_table.public + to = module.networking.aws_route_table.public +} + +moved { + from = aws_route_table_association.public + to = module.networking.aws_route_table_association.public +} + +moved { + from = aws_security_group.app + to = module.networking.aws_security_group.app +} + +moved { + from = aws_s3_bucket.uploads + to = module.storage.aws_s3_bucket.uploads +} + +moved { + from = aws_dynamodb_table.uploads_metadata + to = module.storage.aws_dynamodb_table.uploads_metadata +} + +moved { + from = aws_iam_role.lambda_exec + to = module.iam.aws_iam_role.lambda_exec +} + +moved { + from = aws_iam_role_policy.lambda_exec + to = module.iam.aws_iam_role_policy.lambda_exec +} + +moved { + from = aws_lambda_function.uploads_metadata + to = module.compute.aws_lambda_function.uploads_metadata +} + +moved { + from = aws_lambda_permission.allow_s3 + to = module.compute.aws_lambda_permission.allow_s3 +} + +moved { + from = aws_s3_bucket_notification.uploads + to = module.compute.aws_s3_bucket_notification.uploads +} + +moved { + from = aws_ssm_parameter.groq_api_key + to = module.compute.aws_ssm_parameter.groq_api_key +} + +moved { + from = aws_ssm_parameter.fish_audio_api_key + to = module.compute.aws_ssm_parameter.fish_audio_api_key +} + +moved { + from = aws_ssm_parameter.fish_audio_voice_id + to = module.compute.aws_ssm_parameter.fish_audio_voice_id +} diff --git a/infra/ssm.tf b/infra/ssm.tf deleted file mode 100644 index 2d971db..0000000 --- a/infra/ssm.tf +++ /dev/null @@ -1,35 +0,0 @@ -variable "groq_api_key" { - description = "Groq API key" - type = string - sensitive = true -} - -variable "fish_audio_api_key" { - description = "Fish Audio API key" - type = string - sensitive = true -} - -variable "fish_audio_voice_id" { - description = "Fish Audio voice ID (not secret)" - type = string - default = "" -} - -resource "aws_ssm_parameter" "groq_api_key" { - name = "/firstwave/backend/GROQ_API_KEY" - type = "SecureString" - value = var.groq_api_key -} - -resource "aws_ssm_parameter" "fish_audio_api_key" { - name = "/firstwave/backend/FISH_AUDIO_API_KEY" - type = "SecureString" - value = var.fish_audio_api_key -} - -resource "aws_ssm_parameter" "fish_audio_voice_id" { - name = "/firstwave/backend/FISH_AUDIO_VOICE_ID" - type = "String" - value = var.fish_audio_voice_id -} diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 0000000..35a7108 --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,17 @@ +variable "groq_api_key" { + description = "Groq API key" + type = string + sensitive = true +} + +variable "fish_audio_api_key" { + description = "Fish Audio API key" + type = string + sensitive = true +} + +variable "fish_audio_voice_id" { + description = "Fish Audio voice ID (not secret)" + type = string + default = "" +} From 9511ccd7bb9eb46418b327930590ff6c45af2f6f Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Fri, 25 Sep 2026 10:39:10 -0700 Subject: [PATCH 13/14] Add terraform ci --- .github/workflows/terraform-ci.yml | 79 ++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/terraform-ci.yml diff --git a/.github/workflows/terraform-ci.yml b/.github/workflows/terraform-ci.yml new file mode 100644 index 0000000..a5b53fc --- /dev/null +++ b/.github/workflows/terraform-ci.yml @@ -0,0 +1,79 @@ +name: Terraform CI + +on: + pull_request: + paths: + - 'infra/**' + - 'infra-bootstrap/**' + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: terraform-ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + terraform: + runs-on: ubuntu-latest + + services: + localstack: + image: localstack/localstack + ports: + - 4566:4566 + env: + SERVICES: s3,dynamodb,iam,lambda,ec2,ssm + + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-west-2 + TF_VAR_groq_api_key: dummy + TF_VAR_fish_audio_api_key: dummy + TF_VAR_fish_audio_voice_id: dummy + + steps: + - uses: actions/checkout@v5 + + - uses: hashicorp/setup-terraform@v3 + + - name: terraform fmt check + run: terraform fmt -check -recursive + + - name: terraform validate (infra-bootstrap) + working-directory: infra-bootstrap + run: | + terraform init -backend=false + terraform validate + + - name: terraform validate (infra) + working-directory: infra + run: | + terraform init -backend=false + terraform validate + + - name: Wait for LocalStack + run: | + timeout 60 sh -c 'until curl -sf http://localhost:4566/_localstack/health; do sleep 2; done' + + - name: bootstrap state bucket + working-directory: infra-bootstrap + run: | + terraform init + terraform apply -auto-approve + + - name: terraform plan (infra) + working-directory: infra + run: | + terraform init -reconfigure + set +e + terraform plan -detailed-exitcode + exit_code=$? + set -e + if [ $exit_code -eq 1 ]; then + exit 1 + fi + exit 0 From b754a8cc3a63ea2d50086083da951aaf95bb82e4 Mon Sep 17 00:00:00 2001 From: Darren Shen Date: Fri, 25 Sep 2026 11:02:44 -0700 Subject: [PATCH 14/14] Update terraform ci --- .github/workflows/terraform-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/terraform-ci.yml b/.github/workflows/terraform-ci.yml index a5b53fc..6ffee90 100644 --- a/.github/workflows/terraform-ci.yml +++ b/.github/workflows/terraform-ci.yml @@ -26,6 +26,7 @@ jobs: - 4566:4566 env: SERVICES: s3,dynamodb,iam,lambda,ec2,ssm + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} env: AWS_ACCESS_KEY_ID: test