From 9068a8a768c21c0765889c863d1ce00e9f645144 Mon Sep 17 00:00:00 2001 From: Alexanderamiri Date: Mon, 26 Oct 2026 15:23:44 +0100 Subject: [PATCH] feat(app-yaml): adopt an existing RDS instance and Route53 record databases[].adopt imports an existing Postgres instance through the new service-rds-adopted module (tags, deletion protection, final-snapshot settings only; prevent_destroy), host_env/port_env expose address and port, and routing.adopt_dns takes over an existing A record. Each adoption must be listed in ADOPTIONS in scripts/registry.py. Rebased onto routing.login (#158): routing accepts both login and adopt_dns, and service-routing takes login_rules and allow_dns_overwrite. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 2 + docs/app-yaml-reference.md | 31 ++++- docs/reusable-modules.md | 24 +++- scripts/generate-modules.py | 88 +++++++++++++- scripts/registry.py | 15 +++ scripts/tests/test_generate_modules.py | 107 ++++++++++++++++- terraform/modules/README.md | 2 + terraform/modules/service-rds-adopted/main.tf | 109 ++++++++++++++++++ .../modules/service-rds-adopted/outputs.tf | 24 ++++ .../modules/service-rds-adopted/variables.tf | 27 +++++ .../modules/service-rds-adopted/versions.tf | 13 +++ terraform/modules/service-rds/outputs.tf | 5 + terraform/modules/service-routing/main.tf | 11 +- .../modules/service-routing/variables.tf | 6 + 14 files changed, 451 insertions(+), 13 deletions(-) create mode 100644 terraform/modules/service-rds-adopted/main.tf create mode 100644 terraform/modules/service-rds-adopted/outputs.tf create mode 100644 terraform/modules/service-rds-adopted/variables.tf create mode 100644 terraform/modules/service-rds-adopted/versions.tf diff --git a/CLAUDE.md b/CLAUDE.md index 9f066cb..411ea5b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -131,6 +131,8 @@ terraform/state/ | `terraform/modules/ecs-service/` | ECS Fargate service | | `terraform/modules/service-bucket/` | S3 bucket with IAM policy output | | `terraform/modules/service-database/` | DynamoDB table with IAM policy output | +| `terraform/modules/service-rds/` | RDS PostgreSQL instance with generated master password | +| `terraform/modules/service-rds-adopted/` | Existing RDS instance imported as-is (`databases[].adopt`); manages tags and deletion protection only | | `terraform/modules/service-secret/` | SSM Parameter Store SecureString with IAM policy output | | `terraform/modules/service-queue/` | SQS queue + DLQ with IAM policy output | | `terraform/modules/service-alarm/` | CloudWatch alarms for ECS service | diff --git a/docs/app-yaml-reference.md b/docs/app-yaml-reference.md index 82876ef..3b2b010 100644 --- a/docs/app-yaml-reference.md +++ b/docs/app-yaml-reference.md @@ -107,11 +107,13 @@ routing: host: moresleep.javazone.no # required if routing is specified; also java.no / javabin.no priority: 100 # required, unique 1-50000 deregistration_delay: 30 # default: 30 (seconds) + adopt_dns: false # default: false — take over an existing record for host ``` - `host` must be exactly one label under `javazone.no`, `java.no` or `javabin.no` (e.g. `moresleep.javazone.no`, `medlem.java.no`, `styret.javabin.no`). Those are the domains with a wildcard cert on the platform ALB; deeper names like `a.b.java.no` are rejected because the wildcard cert does not cover them. - `host` creates a Route53 A record (in the zone matching the host's domain) pointing to the platform ALB and an ALB listener rule matching the Host header. - `priority` determines listener rule evaluation order. Must be unique across all apps. Coordinate via the registry. +- `adopt_dns: true` takes over an A record for `host` that already exists (for example a hand-made record of an app moving onto the platform). It must be approved first: the host has to be listed for your team in `ADOPTIONS` in `scripts/registry.py` (a javaBin/platform PR), otherwise generation fails. Route53 records are not protected by team permissions, so this is what stops one team from taking over a hostname another app serves. Without `adopt_dns`, the apply fails because the record exists. With it, Route53 overwrites the record (UPSERT) to point at the platform ALB, and from then on Terraform manages it. The plan shows the record as a *create*. Changing `host` later replaces the record, and removing routing deletes it, as for any managed record. See [Adopting existing resources](#adopting-existing-resources). #### routing.login — sign-in at the load balancer @@ -239,13 +241,40 @@ resources: backup_retention_period: 7 # days, RDS only, default: 7 multi_az: false # RDS only, default: false deletion_protection: true # RDS only, default: true - env: DATABASE_URL + env: DATABASE_URL # host:port + host_env: DB_HOST # optional, postgres only: hostname alone + port_env: DB_PORT # optional, postgres only: port alone ``` DynamoDB and PostgreSQL entries can coexist in the same `databases` list. Entries without `engine` (or with `engine: dynamodb`) use the DynamoDB module. Entries with `engine: postgres` or `engine: postgresql` use the RDS module. RDS instances generate a master password at creation time, stored in SSM Parameter Store at `/{project}/apps/{name}/db-master-password`. The ECS task role automatically receives IAM policies for `rds-db:connect` and `ssm:GetParameter` on the password parameter. +`env` gets the endpoint as `host:port`. Apps that read host and port separately use `host_env` and `port_env` (PostgreSQL only, created or adopted). + +##### Adopting an existing PostgreSQL instance + +An app that moves onto the platform can keep its existing RDS instance instead of getting a new one: + +```yaml +resources: + databases: + - name: main + engine: postgres + adopt: dbmoresleep # DB instance identifier of the existing instance + host_env: DBHOST + port_env: DBPORT + deletion_protection: true # default: true + backup_retention_period: 14 # optional; unset = keep the instance's current value +``` + +- The generated `main.tf` gets an `import {}` block, and the instance is imported into the app's Terraform state on the first apply. Nothing is created or moved. Once the instance is in state, Terraform skips the block, so it stays in the generated file. +- The `service-rds-adopted` module manages only tags, `deletion_protection` and (when set) `backup_retention_period`. Engine, version, class, storage, network placement, security groups, parameter group, password and monitoring stay as they are in AWS; `instance_class`, `allocated_storage`, `engine_version` and `multi_az` are rejected with `adopt`. The first plan shows `1 to import`, an in-place update of tags and deletion protection, one `random_id` (the suffix of the final snapshot name), and nothing replaced. +- The app keeps connecting the way it did, with its own credentials (put them in `secrets:`). No IAM policy or platform-managed password is added. +- The instance has `prevent_destroy`. To stop managing it without deleting it, add a hand-written file next to the generated ones, e.g. `terraform/release-db.tf`, with `removed { from = module.rds_main.aws_db_instance.this lifecycle { destroy = false } }`, and remove the entry from `app.yaml` in the same change. +- Approval: each adoption is approved in a javaBin/platform PR that lists the identifier for your team in `ADOPTIONS` in `scripts/registry.py`; without it generation fails. Legacy instances are often untagged, and the team permission boundary only protects resources tagged with another team, so an unapproved `adopt` could take over (and tag away) another team's database. After the first apply the instance carries `team=`. +- Network: platform tasks run in the platform VPC's private subnets and reach the internet through its NAT gateway (all database traffic then pays NAT data processing). An adopted instance in another VPC must be reachable from there. If that means a public endpoint, its security group should allow only the NAT address (and any legacy clients that still need it), never `0.0.0.0/0`. The platform does not change its security group, so tighten it yourself. + #### secrets SSM Parameter Store SecureString parameters. The platform creates each one with the value `PLACEHOLDER`; set the real value with the CLI from the app repo, signed in with your team's SSO permission set (`javabin-team-`): diff --git a/docs/reusable-modules.md b/docs/reusable-modules.md index 6bacbbc..cd30dd0 100644 --- a/docs/reusable-modules.md +++ b/docs/reusable-modules.md @@ -39,7 +39,7 @@ ECR repository with 4-rule lifecycle policy. ALB target group + listener rule (host-header routing) + Route53 A record (alias to ALB). **Required inputs:** `name`, `team`, `vpc_id`, `https_listener_arn`, `listener_rule_priority`, `host_header`, `route53_zone_id`, `alb_dns_name`, `alb_zone_id` -**Optional inputs:** `project`, `login_rules` (app.yaml `routing.login`), `login_session_timeout` (default 8 h), `login_client_wait` (default `60s`) +**Optional inputs:** `project`, `login_rules` (app.yaml `routing.login`), `login_session_timeout` (default 8 h), `login_client_wait` (default `60s`), `allow_dns_overwrite` (default false; take over an existing A record for `host_header` with a Route53 UPSERT, set by `routing.adopt_dns` in app.yaml) **Outputs:** `target_group_arn`, `target_group_arn_suffix`, `dns_name` `login_rules` (map keyed `p1`, `p2`, ...) adds one listener rule per entry, matching host + paths at the entry's priority (below `listener_rule_priority`). Protected entries get an `authenticate-oidc` action against the chosen Cognito pool, with a per-rule client `{team}-{name}.login-N` requested through a nested `cognito-app-client` (slot `login-N`); group restrictions are enforced by Cognito (auth-claims). `public` entries forward without sign-in. The host rule `depends_on` them, so a new app serves nothing until they exist. `authenticate-cognito` is not used: it needs `cognito-idp:DescribeUserPoolClient` on the platform pools, which the developer boundary denies. Requires the `hashicorp/time` provider (for the wait on the async client creation). @@ -119,7 +119,7 @@ RDS PostgreSQL instance in private subnets. **Password:** Generated once via `random_password`, stored in SSM at `/{project}/apps/{name}/db-master-password`. -**Outputs:** `endpoint`, `port`, `db_name`, `password_parameter_arn`, `access_policy_json`, `security_group_id` +**Outputs:** `endpoint`, `address`, `port`, `db_name`, `password_parameter_arn`, `access_policy_json`, `security_group_id` **Auto-wiring:** `access_policy_json` grants `rds-db:connect` + `ssm:GetParameter` on the password parameter. Auto-attached to task role via `collect:access_policy_json`. @@ -133,6 +133,26 @@ databases: engine_version: "16" ``` + +## service-rds-adopted + +An existing RDS instance brought under Terraform without changing it (`databases[].adopt` in app.yaml). The generated root module imports it with an `import {}` block. + +**Inputs:** + +| Input | Default | +|-------|---------| +| `identifier` | required — DB instance identifier | +| `deletion_protection` | true | +| `backup_retention_period` | null (keep the live value) | +| `tags` | `{}` | + +**Manages:** tags, `deletion_protection`, `backup_retention_period` (when set), final-snapshot settings (`-final-`, so a later delete/re-adopt cycle never collides with an old snapshot). Needs AWS provider 5.40 or later. Every other argument is read from the live instance or listed in `ignore_changes`, so importing never plans a replacement. `prevent_destroy` is set. + +**Outputs:** `endpoint`, `address`, `port` (string), `arn`, `resource_id` + +**Auto-wiring:** none. The app keeps its own credentials (`secrets:`); `env`, `host_env` and `port_env` map to `endpoint`, `address` and `port`. + ## service-alarm CloudWatch alarms for ECS services: CPU high, memory high, unhealthy targets, 5xx errors. diff --git a/scripts/generate-modules.py b/scripts/generate-modules.py index 26eceef..b37a572 100755 --- a/scripts/generate-modules.py +++ b/scripts/generate-modules.py @@ -26,7 +26,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from registry import ( - PROJECT, DOMAIN, APP_DOMAINS, MODULE_SOURCES, + PROJECT, DOMAIN, APP_DOMAINS, MODULE_SOURCES, ADOPTIONS, BACKEND_TEMPLATE, PROVIDERS_TEMPLATE, OUTPUTS_TEMPLATE, GITIGNORE_TEMPLATE, ) @@ -61,7 +61,7 @@ class AppYamlError(ValueError): "team": None, "compute": {"cpu", "memory", "port", "desired_count", "health_check", "health_check_matcher", "user", "readonly_root_filesystem"}, - "routing": {"host", "priority", "deregistration_delay", "login"}, + "routing": {"host", "priority", "deregistration_delay", "login", "adopt_dns"}, "domain": None, "resources": {"buckets", "databases", "secrets", "queues"}, "auth": {"pool", "client", "callback_urls", "logout_urls", "scopes", "groups", "env", "extra_hosts"}, @@ -73,7 +73,8 @@ class AppYamlError(ValueError): "buckets": {"name", "env", "versioning", "expire_days"}, "databases": {"name", "env", "engine", "hash_key", "hash_key_type", "range_key", "range_key_type", "ttl_attribute", "instance_class", "allocated_storage", - "engine_version", "backup_retention_period", "multi_az", "deletion_protection"}, + "engine_version", "backup_retention_period", "multi_az", "deletion_protection", + "adopt", "host_env", "port_env"}, "secrets": {"name", "env", "description"}, "queues": {"name", "env", "visibility_timeout", "retention_seconds", "max_receive_count"}, } @@ -104,6 +105,12 @@ class AppYamlError(ValueError): HOSTNAME_RE = re.compile(r"^(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$") LOCAL_HOSTS = ("localhost", "127.0.0.1") +RDS_IDENTIFIER_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9-]{0,62}$") + +POSTGRES_ENGINES = ("postgres", "postgresql") +# An adopted database keeps its live shape; these would suggest otherwise. +ADOPT_REJECTED = ("instance_class", "allocated_storage", "engine_version", "multi_az") + def host_zone(host): """The app domain a host lives under, or None if it is not one label under one.""" @@ -219,6 +226,38 @@ def login_warnings(rules): return [] +def _check_adoption(team, kind, identifier, where): + if identifier not in ADOPTIONS.get(team, {}).get(kind, set()): + raise AppYamlError( + f"{where}: team {team!r} is not approved to adopt {identifier!r}. Adopting takes over a " + "resource the platform did not create, so each one is approved in a javaBin/platform PR " + f"that adds it to ADOPTIONS[{team!r}][{kind!r}] in scripts/registry.py" + ) + + +def _validate_database(where, db, team): + postgres = str(db.get("engine") or "dynamodb").lower() in POSTGRES_ENGINES + for key in ("host_env", "port_env"): + if key in db: + if not postgres: + raise AppYamlError(f"{where}.{key}: only supported for engine: postgres") + if not ENV_RE.match(str(db[key])): + raise AppYamlError(f"{where}.{key}: {db[key]!r} is not a valid variable name") + if "adopt" not in db: + return + if not postgres: + raise AppYamlError(f"{where}.adopt: only supported for engine: postgres") + if not RDS_IDENTIFIER_RE.match(str(db["adopt"])): + raise AppYamlError(f"{where}.adopt: {db['adopt']!r} is not an RDS DB instance identifier") + _check_adoption(team, "rds", db["adopt"], f"{where}.adopt") + rejected = [k for k in ADOPT_REJECTED if k in db] + if rejected: + raise AppYamlError( + f"{where}: {', '.join(rejected)} cannot be set with adopt — an adopted database keeps " + "its live engine, class, storage and placement; change those outside app.yaml" + ) + + def validate(app): """Raise AppYamlError if app.yaml uses unknown fields or invalid values.""" if not isinstance(app, dict): @@ -250,6 +289,10 @@ def validate(app): if not isinstance(routing.get("priority"), int) or not 1 <= routing["priority"] <= 50000: raise AppYamlError("routing.priority is required: a unique integer 1-50000") normalize_login(routing) + if "adopt_dns" in routing and not isinstance(routing["adopt_dns"], bool): + raise AppYamlError("routing.adopt_dns must be true or false") + if routing.get("adopt_dns"): + _check_adoption(app["team"], "dns", host, "routing.adopt_dns") for rtype, items in (app.get("resources") or {}).items(): if not isinstance(items, list): @@ -260,6 +303,8 @@ def validate(app): raise AppYamlError(f"resources.{rtype}[{i}].name: {item.get('name')!r} is not a valid name") if item.get("env") and not ENV_RE.match(item["env"]): raise AppYamlError(f"resources.{rtype}[{i}].env: {item['env']!r} is not a valid variable name") + if rtype == "databases": + _validate_database(f"resources.databases[{i}]", item, app["team"]) env = app.get("environment") if env is not None: @@ -346,6 +391,18 @@ def module_block(name, source, attrs): return "\n".join(lines) +def import_block(to, import_id): + """An `import {}` block adopting an existing resource. Once the resource is + in state Terraform skips the block, so it can stay in the generated file.""" + return f"import {{\n to = {to}\n id = {hcl_string(import_id)}\n}}" + + +def _wire_postgres_env(env_map, mod_name, db): + for key, output in (("env", "endpoint"), ("host_env", "address"), ("port_env", "port")): + if db.get(key): + env_map[db[key]] = Ref(f"module.{mod_name}.{output}") + + # --------------------------------------------------------------------------- # Auth # --------------------------------------------------------------------------- @@ -461,6 +518,8 @@ def generate(app, account_id, region, platform_ref): raise AppYamlError("domain: requires routing.priority (unique 1-50000) to create the ALB rule") routing["host"] = app["domain"] + imports = [] # (resource address, import id) for existing resources to adopt + def src(mod): path = MODULE_SOURCES[mod] return f"git::https://github.com/javaBin/platform//{path}?ref={platform_ref}" @@ -513,6 +572,9 @@ def src(mod): "alb_dns_name": Ref("module.platform.alb_dns_name"), "alb_zone_id": Ref("module.platform.alb_zone_id"), "deregistration_delay": routing.get("deregistration_delay", 30), + # adopt_dns: take over an existing record (Route53 UPSERT). Only + # emitted when set, so other apps' generated files don't change. + **({"allow_dns_overwrite": True} if routing.get("adopt_dns") else {}), } # routing.login: one ALB rule per entry, at the priorities directly # below routing.priority so they are evaluated before the host rule. @@ -556,7 +618,20 @@ def src(mod): for db in resources.get("databases", []): dname = db["name"] engine = (db.get("engine") or "dynamodb").lower() - if engine in ("postgres", "postgresql"): + if engine in POSTGRES_ENGINES and db.get("adopt"): + # Existing instance: imported, never created. Password auth with + # the app's own credentials, so no task-role policy. + mod_name = f"rds_{dname}" + attrs = { + "identifier": db["adopt"], + "deletion_protection": db.get("deletion_protection", True), + } + if db.get("backup_retention_period") is not None: + attrs["backup_retention_period"] = db["backup_retention_period"] + blocks.append(module_block(mod_name, src("service-rds-adopted"), attrs)) + imports.append((f"module.{mod_name}.aws_db_instance.this", db["adopt"])) + _wire_postgres_env(env_map, mod_name, db) + elif engine in POSTGRES_ENGINES: mod_name = f"rds_{dname}" blocks.append(module_block(mod_name, src("service-rds"), { "name": dname, @@ -573,8 +648,7 @@ def src(mod): "deletion_protection": db.get("deletion_protection", True), })) policy_map[mod_name] = Ref(f"module.{mod_name}.access_policy_json") - if db.get("env"): - env_map[db["env"]] = Ref(f"module.{mod_name}.endpoint") + _wire_postgres_env(env_map, mod_name, db) elif engine == "dynamodb": mod_name = f"database_{dname}" attrs = { @@ -739,6 +813,8 @@ def src(mod): name = "{PROJECT}-alerts" }}""") + blocks.extend(import_block(to, id_) for to, id_ in imports) + files["main.tf"] = GENERATED + "\n\n".join(blocks) + "\n" # -- outputs.tf -- diff --git a/scripts/registry.py b/scripts/registry.py index 2604994..72e620d 100644 --- a/scripts/registry.py +++ b/scripts/registry.py @@ -15,6 +15,20 @@ # cert on the platform ALB, so a host must be exactly one label under one. APP_DOMAINS = ("javazone.no", "java.no", "javabin.no") +# Existing resources a team may adopt with app.yaml (databases[].adopt and +# routing.adopt_dns). Adopting takes over a resource the platform did not +# create: the team's Terraform imports or overwrites it and then owns it. +# Neither Route53 records nor untagged legacy resources are protected by the +# team permission boundary, so each adoption is approved here, in a platform +# PR, instead of being a one-line app.yaml change. Remove an entry once the +# adoption is applied if the resource should never be adopted again. +ADOPTIONS = { + "moresleep": { + "rds": {"dbmoresleep"}, + "dns": {"sleepingpill.javazone.no"}, + }, +} + # --------------------------------------------------------------------------- # Module source paths (relative to platform repo root) # --------------------------------------------------------------------------- @@ -29,6 +43,7 @@ "service-bucket": "terraform/modules/service-bucket", "service-database": "terraform/modules/service-database", "service-rds": "terraform/modules/service-rds", + "service-rds-adopted": "terraform/modules/service-rds-adopted", "service-secret": "terraform/modules/service-secret", "service-queue": "terraform/modules/service-queue", "cognito-app-client": "terraform/modules/cognito-app-client", diff --git a/scripts/tests/test_generate_modules.py b/scripts/tests/test_generate_modules.py index e484de5..6a9b3e6 100644 --- a/scripts/tests/test_generate_modules.py +++ b/scripts/tests/test_generate_modules.py @@ -58,6 +58,24 @@ "alarms": {"cpu_threshold": 70}, } +MORESLEEP = { # javaBin/moresleep: existing RDS instance and hand-made DNS record + "name": "moresleep", + "team": "moresleep", + "compute": {"cpu": 256, "memory": 512, "port": 5000, "desired_count": 2, "health_check": "/"}, + "routing": {"host": "sleepingpill.javazone.no", "priority": 120, "adopt_dns": True}, + "resources": { + "databases": [{"name": "main", "engine": "postgres", "adopt": "dbmoresleep", + "host_env": "DBHOST", "port_env": "DBPORT"}], + "secrets": [{"name": "db-password", "env": "DBPASSWORD"}], + }, + "environment": {"DATABASE_TYPE": "POSTGRES", "ALL_OPEN_MODE": "false"}, +} + + +def with_db(app, **fields): + db = {**app["resources"]["databases"][0], **fields} + return {**app, "resources": {**app["resources"], "databases": [db]}} + def generate(app): return gen.generate(dict(app), ACCOUNT, REGION, REF) @@ -370,6 +388,92 @@ def test_invalid_rules_are_rejected(self): self.assertRejected([{"paths": ["/a"]}, {"paths": ["/b"]}], "use at least 3", priority=2) +class AdoptTest(unittest.TestCase): + def assertRejected(self, app, fragment): + with self.assertRaises(gen.AppYamlError) as cm: + generate(app) + self.assertIn(fragment, str(cm.exception)) + + def test_adopted_database_is_imported_not_created(self): + main = generate(MORESLEEP)["main.tf"] + self.assertIn('module "rds_main"', main) + self.assertIn("//terraform/modules/service-rds-adopted?ref=main", main) + self.assertIn('identifier = "dbmoresleep"', main) + self.assertIn('import {\n to = module.rds_main.aws_db_instance.this\n id = "dbmoresleep"\n}', main) + self.assertNotIn("modules/service-rds?", main) + + def test_adopted_database_defaults_to_deletion_protection(self): + self.assertRegex(generate(MORESLEEP)["main.tf"], r"deletion_protection\s+= true") + main = generate(with_db(MORESLEEP, deletion_protection=False))["main.tf"] + self.assertRegex(main, r"deletion_protection\s+= false") + + def test_backup_retention_is_only_managed_when_set(self): + self.assertNotIn("backup_retention_period", generate(MORESLEEP)["main.tf"]) + main = generate(with_db(MORESLEEP, backup_retention_period=14))["main.tf"] + self.assertRegex(main, r"backup_retention_period\s+= 14") + + def test_adopted_database_gets_no_task_role_policy(self): + main = generate(MORESLEEP)["main.tf"] + self.assertNotIn("module.rds_main.access_policy_json", main) + + def test_host_and_port_env_are_wired(self): + main = generate(MORESLEEP)["main.tf"] + self.assertRegex(main, r"DBHOST\s+= module.rds_main.address") + self.assertRegex(main, r"DBPORT\s+= module.rds_main.port") + + def test_host_env_works_for_created_postgres_too(self): + app = with_db(MORESLEEP, adopt=None, host_env="PGHOST", env="DATABASE_URL") + app["resources"]["databases"][0].pop("adopt") + main = generate(app)["main.tf"] + self.assertIn("modules/service-rds?ref=main", main) + self.assertRegex(main, r"PGHOST\s+= module.rds_main.address") + self.assertRegex(main, r"DATABASE_URL\s+= module.rds_main.endpoint") + self.assertNotIn("import {", main) + + def test_adopt_rejects_fields_mirrored_from_the_live_instance(self): + for field, value in (("instance_class", "db.t4g.small"), ("allocated_storage", 50), + ("engine_version", "17"), ("multi_az", True)): + self.assertRejected(with_db(MORESLEEP, **{field: value}), "cannot be set with adopt") + + def test_adopt_is_postgres_only(self): + app = with_db(MORESLEEP, engine="dynamodb") + for key in ("host_env", "port_env"): + app["resources"]["databases"][0].pop(key) + self.assertRejected(app, "adopt: only supported for engine: postgres") + + def test_adopt_needs_a_valid_identifier(self): + self.assertRejected(with_db(MORESLEEP, adopt="db_more sleep"), "not an RDS DB instance identifier") + + def test_host_env_is_postgres_only_and_a_valid_name(self): + self.assertRejected(with_db(MORESLEEP, engine="dynamodb", adopt=None), "only supported for engine: postgres") + self.assertRejected(with_db(MORESLEEP, host_env="DB-HOST"), "not a valid variable name") + + def test_adopt_dns_takes_over_the_existing_record(self): + main = generate(MORESLEEP)["main.tf"] + self.assertRegex(main, r"allow_dns_overwrite\s+= true") + self.assertIn('module.platform.route53_zone_ids["javazone.no"]', main) + + def test_without_adopt_dns_routing_output_is_unchanged(self): + self.assertNotIn("allow_dns_overwrite", generate(CUPCAKE)["main.tf"]) + + def test_adopting_a_database_needs_platform_approval(self): + self.assertRejected(with_db(MORESLEEP, adopt="dbcakeredux"), "not approved to adopt 'dbcakeredux'") + other_team = {**MORESLEEP, "team": "cupcake", "routing": {"host": "x.javazone.no", "priority": 120}} + self.assertRejected(other_team, "team 'cupcake' is not approved to adopt 'dbmoresleep'") + + def test_adopting_a_dns_record_needs_platform_approval(self): + app = {**MORESLEEP, "routing": {**MORESLEEP["routing"], "host": "submit.javazone.no"}} + self.assertRejected(app, "routing.adopt_dns: team 'moresleep' is not approved to adopt 'submit.javazone.no'") + + def test_unapproved_host_is_fine_without_adopt_dns(self): + routing = {"host": "submit.javazone.no", "priority": 120} + self.assertNotIn("allow_dns_overwrite", generate({**MORESLEEP, "routing": routing})["main.tf"]) + + def test_adopt_dns_must_be_a_bool(self): + app = {**MORESLEEP, "routing": {**MORESLEEP["routing"], "adopt_dns": "yes"}} + self.assertRejected(app, "routing.adopt_dns must be true or false") + + @unittest.skipUnless(shutil.which("terraform"), "terraform not installed") class FmtTest(unittest.TestCase): """tf-plan runs `terraform fmt -check`; generated files must pass it.""" @@ -377,7 +481,8 @@ class FmtTest(unittest.TestCase): def test_generated_files_are_fmt_clean(self): public = {**FROSTING, "auth": {"pool": "internal", "client": "public", "groups": ["helter"], "env": {"COGNITO_CLIENT_ID": "NUXT_PUBLIC_OIDC_CLIENT_ID"}}} - for app in (CUPCAKE, FROSTING, FULL, public, REAL_FROSTING, REAL_CUPCAKE, WIKI, {"name": "worker", "team": "core"}): + for app in (CUPCAKE, FROSTING, FULL, MORESLEEP, public, REAL_FROSTING, REAL_CUPCAKE, WIKI, + {"name": "worker", "team": "core"}): with tempfile.TemporaryDirectory() as d: for name, content in generate(app).items(): with open(os.path.join(d, name), "w") as f: diff --git a/terraform/modules/README.md b/terraform/modules/README.md index 7655f5d..0c6a8ee 100644 --- a/terraform/modules/README.md +++ b/terraform/modules/README.md @@ -21,6 +21,8 @@ module "app" { | [ecs-service](ecs-service/) | ECS Fargate service + task def + logs | `service_name` | | [service-bucket](service-bucket/) | S3 bucket with standard security config | `bucket_name`, `access_policy_json` | | [service-database](service-database/) | DynamoDB table | `table_name`, `access_policy_json` | +| [service-rds](service-rds/) | RDS PostgreSQL instance | `endpoint`, `address`, `port` | +| [service-rds-adopted](service-rds-adopted/) | Existing RDS instance imported as-is | `endpoint`, `address`, `port` | | [service-secret](service-secret/) | SSM Parameter Store SecureString | `parameter_arn`, `access_policy_json` | | [service-queue](service-queue/) | SQS queue + DLQ | `queue_url`, `access_policy_json` | | [service-alarm](service-alarm/) | CloudWatch alarms for ECS | `cpu_alarm_arn` | diff --git a/terraform/modules/service-rds-adopted/main.tf b/terraform/modules/service-rds-adopted/main.tf new file mode 100644 index 0000000..85f3e2a --- /dev/null +++ b/terraform/modules/service-rds-adopted/main.tf @@ -0,0 +1,109 @@ +################################################################################ +# Adopted RDS instance — an existing database brought under Terraform +# +# For apps that move onto the platform with a database that already exists +# and must not move. The generated root module imports the instance into +# aws_db_instance.this with an `import {}` block; nothing is created. +# +# What this module manages: tags (via provider default_tags + var.tags), +# deletion_protection, backup_retention_period +# (only when set), and the final-snapshot settings +# Terraform uses if the instance is ever deleted. +# What it never touches: everything else. Engine, class, storage, +# network, security groups, parameter group, +# password, monitoring... stay exactly as they are +# in AWS and are changed outside this module (or +# the app moves to service-rds later). +# +# instance_class is required by the provider, so it is read from the live +# instance. All other unmanaged arguments are in ignore_changes, so an import +# never plans an in-place change or a replacement for them. +# +# To stop managing the instance without deleting it, add a hand-written file +# next to the generated ones (e.g. terraform/release-db.tf): +# +# removed { +# from = module.rds_.aws_db_instance.this +# lifecycle { destroy = false } +# } +# +# and remove the database from app.yaml in the same change. +################################################################################ + +data "aws_db_instance" "live" { + db_instance_identifier = var.identifier +} + +# Suffix for the final snapshot name, fixed for the life of this state, so a +# later delete/restore/re-adopt cycle does not collide with an old snapshot. +resource "random_id" "final_snapshot" { + byte_length = 4 + keepers = { + identifier = var.identifier + } +} + +resource "aws_db_instance" "this" { + identifier = var.identifier + instance_class = data.aws_db_instance.live.db_instance_class + + deletion_protection = var.deletion_protection + backup_retention_period = coalesce(var.backup_retention_period, data.aws_db_instance.live.backup_retention_period) + + # Terraform-only settings (no API call): keep a final snapshot and the + # automated backups if the instance is ever deleted through Terraform. + skip_final_snapshot = false + final_snapshot_identifier = "${var.identifier}-final-${random_id.final_snapshot.hex}" + delete_automated_backups = false + + tags = var.tags + + lifecycle { + prevent_destroy = true + + # Mirrored from AWS, never changed by this module. + ignore_changes = [ + instance_class, + engine, + engine_version, + allocated_storage, + max_allocated_storage, + storage_type, + iops, + storage_throughput, + storage_encrypted, + kms_key_id, + username, + password, + manage_master_user_password, + master_user_secret_kms_key_id, + db_name, + port, + publicly_accessible, + multi_az, + availability_zone, + db_subnet_group_name, + vpc_security_group_ids, + network_type, + parameter_group_name, + option_group_name, + ca_cert_identifier, + iam_database_authentication_enabled, + performance_insights_enabled, + performance_insights_kms_key_id, + performance_insights_retention_period, + monitoring_interval, + monitoring_role_arn, + enabled_cloudwatch_logs_exports, + copy_tags_to_snapshot, + auto_minor_version_upgrade, + allow_major_version_upgrade, + backup_window, + maintenance_window, + apply_immediately, + license_model, + customer_owned_ip_enabled, + dedicated_log_volume, + ] + } +} diff --git a/terraform/modules/service-rds-adopted/outputs.tf b/terraform/modules/service-rds-adopted/outputs.tf new file mode 100644 index 0000000..84ebb55 --- /dev/null +++ b/terraform/modules/service-rds-adopted/outputs.tf @@ -0,0 +1,24 @@ +output "endpoint" { + description = "RDS instance endpoint (host:port)" + value = aws_db_instance.this.endpoint +} + +output "address" { + description = "RDS instance hostname" + value = aws_db_instance.this.address +} + +output "port" { + description = "RDS instance port, as a string for environment variables" + value = tostring(aws_db_instance.this.port) +} + +output "arn" { + description = "RDS instance ARN" + value = aws_db_instance.this.arn +} + +output "resource_id" { + description = "RDS instance resource ID (dbi-resource-id, used in rds-db:connect ARNs)" + value = aws_db_instance.this.resource_id +} diff --git a/terraform/modules/service-rds-adopted/variables.tf b/terraform/modules/service-rds-adopted/variables.tf new file mode 100644 index 0000000..90c44e5 --- /dev/null +++ b/terraform/modules/service-rds-adopted/variables.tf @@ -0,0 +1,27 @@ +variable "identifier" { + description = "DB instance identifier of the existing RDS instance to adopt" + type = string + + validation { + condition = can(regex("^[a-zA-Z][a-zA-Z0-9-]{0,62}$", var.identifier)) + error_message = "identifier must be a valid RDS DB instance identifier." + } +} + +variable "deletion_protection" { + description = "Enable RDS deletion protection on the adopted instance" + type = bool + default = true +} + +variable "backup_retention_period" { + description = "Automated backup retention in days. null keeps the instance's current value." + type = number + default = null +} + +variable "tags" { + description = "Additional tags (provider default_tags are applied as well)" + type = map(string) + default = {} +} diff --git a/terraform/modules/service-rds-adopted/versions.tf b/terraform/modules/service-rds-adopted/versions.tf new file mode 100644 index 0000000..cf9e12f --- /dev/null +++ b/terraform/modules/service-rds-adopted/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + # dedicated_log_volume (in ignore_changes) needs 5.40 or later. + version = ">= 5.40, < 6.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } +} diff --git a/terraform/modules/service-rds/outputs.tf b/terraform/modules/service-rds/outputs.tf index f3ba129..cf0dc6d 100644 --- a/terraform/modules/service-rds/outputs.tf +++ b/terraform/modules/service-rds/outputs.tf @@ -27,3 +27,8 @@ output "security_group_id" { description = "Security group ID for the RDS instance" value = aws_security_group.this.id } + +output "address" { + description = "RDS instance hostname" + value = aws_db_instance.this.address +} diff --git a/terraform/modules/service-routing/main.tf b/terraform/modules/service-routing/main.tf index 5e2351c..b884937 100644 --- a/terraform/modules/service-routing/main.tf +++ b/terraform/modules/service-routing/main.tf @@ -229,10 +229,15 @@ resource "aws_lb_listener_rule" "public" { # Route53 DNS Record (alias to ALB) ################################################################################ +# allow_overwrite (routing.adopt_dns in app.yaml) lets Terraform take over a +# record that already exists, e.g. a hand-made record for an app that moves +# onto the platform: Route53 UPSERTs it instead of failing on CREATE. From +# then on the record is managed here, and destroying it deletes it. resource "aws_route53_record" "this" { - zone_id = var.route53_zone_id - name = var.host_header - type = "A" + zone_id = var.route53_zone_id + name = var.host_header + type = "A" + allow_overwrite = var.allow_dns_overwrite ? true : null # null = unchanged for existing apps alias { name = var.alb_dns_name diff --git a/terraform/modules/service-routing/variables.tf b/terraform/modules/service-routing/variables.tf index 66ecc67..0555862 100644 --- a/terraform/modules/service-routing/variables.tf +++ b/terraform/modules/service-routing/variables.tf @@ -126,3 +126,9 @@ variable "login_client_wait" { type = string default = "60s" } + +variable "allow_dns_overwrite" { + description = "Take over an existing A record for host_header (Route53 UPSERT) instead of failing because it exists" + type = bool + default = false +}