diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 611bed06..6ab3f66a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -191,7 +191,7 @@ "serverless", "postgresql" ], - "version": "1.8.1" + "version": "1.9.0" }, { "category": "deployment", diff --git a/plugins/databases-on-aws/.claude-plugin/plugin.json b/plugins/databases-on-aws/.claude-plugin/plugin.json index 37364129..900230e5 100644 --- a/plugins/databases-on-aws/.claude-plugin/plugin.json +++ b/plugins/databases-on-aws/.claude-plugin/plugin.json @@ -22,5 +22,5 @@ "license": "Apache-2.0", "name": "databases-on-aws", "repository": "https://github.com/awslabs/agent-plugins", - "version": "1.8.1" + "version": "1.9.0" } diff --git a/plugins/databases-on-aws/.codex-plugin/plugin.json b/plugins/databases-on-aws/.codex-plugin/plugin.json index 9e3ad5c4..d011a94f 100644 --- a/plugins/databases-on-aws/.codex-plugin/plugin.json +++ b/plugins/databases-on-aws/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "databases-on-aws", - "version": "1.8.1", + "version": "1.9.0", "description": "Expert database guidance for the AWS database portfolio. Design schemas, execute queries, handle migrations, and choose the right database for your workload.", "author": { "name": "Amazon Web Services", diff --git a/plugins/databases-on-aws/skills/dsql/SKILL.md b/plugins/databases-on-aws/skills/dsql/SKILL.md index 91a3dc4a..0bc01c00 100644 --- a/plugins/databases-on-aws/skills/dsql/SKILL.md +++ b/plugins/databases-on-aws/skills/dsql/SKILL.md @@ -65,9 +65,9 @@ Load these files as needed for detailed guidance: ### ORM Guides: -| Reference | When to Load | Contains | -| ----------------------------------------------------------- | ------------------------- | ------------------------------------------------------------------------ | -| [orm-guides/overview.md](references/orm-guides/overview.md) | Migrating any ORM to DSQL | Adapter names, key gotchas for Django/EF Core/Hibernate/Rails/SQLAlchemy | +| Reference | When to Load | Contains | +| ----------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------------------------------------- | +| [orm-guides/overview.md](references/orm-guides/overview.md) | Migrating an ORM or using ORM locking with DSQL | Adapter names, locking, and key gotchas for Django/EF Core/Hibernate/Rails/SQLAlchemy | ### Data Loading: @@ -145,18 +145,18 @@ See [mcp-tools.md](mcp/mcp-tools.md) for detailed usage and examples. Consult for verifying DSQL service limits before advising users. The numeric limits below are defaults that may change — when a user's decision depends on an exact limit, verify it first: -| Limit | Default | Verify query | -| ------------------------------ | ------------- | ---------------------------------- | -| Max rows per transaction | 3,000 | `aurora dsql transaction limits` | -| Max data size per transaction | 10 MiB | `aurora dsql transaction limits` | -| Max transaction duration | 5 minutes | `aurora dsql transaction limits` | -| Max connections per cluster | 10,000 | `aurora dsql connection limits` | -| Auth token expiry | 15 minutes | `aurora dsql authentication token` | -| Max connection duration | 60 minutes | `aurora dsql connection limits` | -| Max indexes per table | 24 | `aurora dsql index limits` | -| Max columns per index | 8 | `aurora dsql index limits` | -| IDENTITY/SEQUENCE CACHE values | 1 or >= 65536 | `aurora dsql sequence cache` | -| Supported column data types | See docs | `aurora dsql supported data types` | +| Limit | Default | Verify query | +| ------------------------------------- | ------------- | ---------------------------------- | +| Max row modifications per transaction | 3,000 | `aurora dsql transaction limits` | +| Max data size per transaction | 10 MiB | `aurora dsql transaction limits` | +| Max transaction duration | 5 minutes | `aurora dsql transaction limits` | +| Max connections per cluster | 10,000 | `aurora dsql connection limits` | +| Auth token expiry | 15 minutes | `aurora dsql authentication token` | +| Max connection duration | 60 minutes | `aurora dsql connection limits` | +| Max indexes per table | 24 | `aurora dsql index limits` | +| Max columns per index | 8 | `aurora dsql index limits` | +| IDENTITY/SEQUENCE CACHE values | 1 or >= 65536 | `aurora dsql sequence cache` | +| Supported column data types | See docs | `aurora dsql supported data types` | **When to verify:** Before recommending batch sizes, connection pool settings, or schema designs where hitting a limit would cause failures; any time the exact number can affect user decision. @@ -271,9 +271,9 @@ MUST load [query-plan/workflow.md](references/query-plan/workflow.md) at entry MUST load [pg-migrations/type-mapping.md](references/pg-migrations/type-mapping.md), [pg-migrations/schema-objects.md](references/pg-migrations/schema-objects.md), and [foreign-keys.md](references/foreign-keys.md). Run `dsql_lint(fix=true)` first for mechanical fixes, preserve foreign-key relationships, translate unsupported source syntax or options, then apply semantic conversions from the pg-migrations references for unfixable diagnostics and patterns the linter cannot handle. Re-lint the final output before deploying. -### Workflow 11: ORM Migration (Django/EF Core/Hibernate/Rails/SQLAlchemy) +### Workflow 11: ORM Migration and Locking (Django/EF Core/Hibernate/Rails/SQLAlchemy) -Load [orm-guides/overview.md](references/orm-guides/overview.md) for adapter names and framework-specific gotchas. +Load [orm-guides/overview.md](references/orm-guides/overview.md) for adapter names, locking guidance, and framework-specific gotchas. ### Workflow 12: System Diagnostics (CloudWatch AAS) @@ -290,7 +290,7 @@ MUST load [system-diagnostics/workflow.md](references/system-diagnostics/workflo - **OCC serialization error:** Retry the transaction. If persistent, check for hot-key contention — see [troubleshooting.md](references/troubleshooting.md). - **Foreign key violation (`23503`):** Correct the relationship or referential action; **MUST NOT** send it through the `40001` retry loop — see [troubleshooting.md](references/troubleshooting.md). -- **Transaction exceeds limits:** Split into batches under 3,000 rows — see [batched-migration.md](references/ddl-migrations/batched-migration.md). +- **Transaction exceeds limits:** Split write batches to stay under 3,000 row modifications and 10 MiB; locked-row primary keys count toward the size limit — see [batched-migration.md](references/ddl-migrations/batched-migration.md). - **Token expiration mid-operation:** Generate a fresh IAM token — see [authentication-guide.md](references/auth/authentication-guide.md). See [troubleshooting.md](references/troubleshooting.md) for other issues. ## Additional Resources diff --git a/plugins/databases-on-aws/skills/dsql/mcp/tools/database-tools.md b/plugins/databases-on-aws/skills/dsql/mcp/tools/database-tools.md index 97ae17a3..fcfdafcf 100644 --- a/plugins/databases-on-aws/skills/dsql/mcp/tools/database-tools.md +++ b/plugins/databases-on-aws/skills/dsql/mcp/tools/database-tools.md @@ -118,7 +118,7 @@ transact([ **Important Notes:** - Each ALTER TABLE must be in its own transaction (DSQL limitation) -- Keep transactions under 3,000 rows and 10 MiB +- Keep transactions under 3,000 row modifications and 10 MiB - For large batch operations, split into multiple transact calls - **MUST** build every statement with [`safe_query.build()`](safe_query.py). Write mode disables all server-side injection filters diff --git a/plugins/databases-on-aws/skills/dsql/references/auth/scaling-guide.md b/plugins/databases-on-aws/skills/dsql/references/auth/scaling-guide.md index d1ca4357..9fb1af10 100644 --- a/plugins/databases-on-aws/skills/dsql/references/auth/scaling-guide.md +++ b/plugins/databases-on-aws/skills/dsql/references/auth/scaling-guide.md @@ -18,9 +18,9 @@ Aurora DSQL is designed for massive horizontal scale without latency degradation ### Batch Size Optimization -- **PREFER batches of 500-1,000 rows** - Balance throughput and transaction limits (3,000 rows, 10 MiB, 5 minutes max — verify via `awsknowledge`: `aurora dsql transaction limits`) +- **PREFER batches of 500-1,000 rows** - Balance throughput and transaction limits (3,000 row modifications, 10 MiB, 5 minutes max — verify via `awsknowledge`: `aurora dsql transaction limits`) - **SHOULD process batches concurrently** - Use multiple connections; consider multiple threads for bulk loading -- **Smaller batches reduce** lock contention, enable better concurrency, fail faster, distribute load evenly +- **Smaller batches reduce** OCC conflict surface, enable better concurrency, fail faster, distribute load evenly ### AVOID Hot Keys diff --git a/plugins/databases-on-aws/skills/dsql/references/ddl-migrations/batched-migration.md b/plugins/databases-on-aws/skills/dsql/references/ddl-migrations/batched-migration.md index 7d38310d..5ae7eb03 100644 --- a/plugins/databases-on-aws/skills/dsql/references/ddl-migrations/batched-migration.md +++ b/plugins/databases-on-aws/skills/dsql/references/ddl-migrations/batched-migration.md @@ -9,7 +9,7 @@ For the full Table Recreation Pattern and verify & swap steps, see [overview.md] ## Batch Size Rules - **PREFER batches of 500-1,000 rows** for optimal performance -- Smaller batches reduce lock contention and enable better concurrency +- Smaller batches reduce OCC conflict surface and enable better concurrency --- diff --git a/plugins/databases-on-aws/skills/dsql/references/development-guide.md b/plugins/databases-on-aws/skills/dsql/references/development-guide.md index 5f726992..48885d67 100644 --- a/plugins/databases-on-aws/skills/dsql/references/development-guide.md +++ b/plugins/databases-on-aws/skills/dsql/references/development-guide.md @@ -159,7 +159,7 @@ PRIMARY KEY, UNIQUE, FOREIGN KEY, NOT NULL, CHECK, DEFAULT (CREATE TABLE or dire Verify current limits via `awsknowledge`: `aurora dsql transaction limits` ``` -Rows: 3,000 max +Row modifications: 3,000 max Size: 10 MiB max Duration: 5 minutes max Isolation: Repeatable Read (fixed) diff --git a/plugins/databases-on-aws/skills/dsql/references/examples/data-operations.md b/plugins/databases-on-aws/skills/dsql/references/examples/data-operations.md index 080035d2..59847185 100644 --- a/plugins/databases-on-aws/skills/dsql/references/examples/data-operations.md +++ b/plugins/databases-on-aws/skills/dsql/references/examples/data-operations.md @@ -33,7 +33,7 @@ DELETE FROM owner WHERE city = 'Portland'; **Transaction Limits** (verify current limits via `awsknowledge`: `aurora dsql transaction limits`)**:** -- Maximum 3,000 rows per transaction +- Maximum 3,000 row modifications per transaction - Maximum 10 MiB data size per transaction - Maximum 5 minutes per transaction diff --git a/plugins/databases-on-aws/skills/dsql/references/orm-guides/overview.md b/plugins/databases-on-aws/skills/dsql/references/orm-guides/overview.md index ea2bd742..790bf3b1 100644 --- a/plugins/databases-on-aws/skills/dsql/references/orm-guides/overview.md +++ b/plugins/databases-on-aws/skills/dsql/references/orm-guides/overview.md @@ -26,18 +26,32 @@ Django and Rails; EF Core, Hibernate, and SQLAlchemy provide composite relations | Rails | Standard `pg` gem + `aws-sdk-dsql` | `gem 'pg'` + `gem 'aws-sdk-dsql'` | | SQLAlchemy | `aurora_dsql_sqlalchemy` | `pip install aurora-dsql-sqlalchemy boto3` | +## SELECT FOR UPDATE + +Aurora DSQL does not require equality predicates on every primary-key column for +`SELECT ... FOR UPDATE`; non-key predicates and queries that join multiple tables are supported. +It does not take blocking row locks. Instead, rows targeted by the locking clause participate in +optimistic commit-time conflict checks. If a concurrent transaction changes a targeted row, the +transaction that loses the commit race fails with SQLSTATE `40001`; retry the whole transaction +with backoff. Keep external side effects outside the retried callback or make them idempotent. + +The primary key of each row targeted by the locking clause counts toward the 10 MiB +transaction-size limit. + +`FOR UPDATE` and `FOR KEY SHARE` are supported. `FOR NO KEY UPDATE` and `FOR SHARE` are not supported. + ## Key Gotchas Per Framework ### Django -| Issue | Fix | -| ----------------- | ------------------------------------------------------------------------------- | -| ENGINE | `'aurora_dsql_django'` (not `django.db.backends.postgresql`) | -| CONN_MAX_AGE | ≤ 1800 (DSQL timeout is 1 hour) | -| Migrations | Each DDL in its own migration; `RunSQL("CREATE INDEX ASYNC ...")` | -| SELECT FOR UPDATE | Use when a write depends on rows read; retain whole-transaction OCC retry | -| AutoField | Replace with `UUIDField(primary_key=True, default=uuid.uuid4)` | -| ForeignKey | Keep `ForeignKey`; the DSQL backend creates database constraints for new tables | +| Issue | Fix | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| ENGINE | `'aurora_dsql_django'` (not `django.db.backends.postgresql`) | +| CONN_MAX_AGE | ≤ 1800 (DSQL timeout is 1 hour) | +| Migrations | Each DDL in its own migration; `RunSQL("CREATE INDEX ASYNC ...")` | +| SELECT FOR UPDATE | `select_for_update()` supports non-key filters and joined querysets that use inner joins; retry the whole transaction on SQLSTATE `40001` | +| AutoField | Replace with `UUIDField(primary_key=True, default=uuid.uuid4)` | +| ForeignKey | Keep `ForeignKey`; the DSQL backend creates database constraints for new tables | ### EF Core (.NET) @@ -60,19 +74,20 @@ Requires .NET 8.0+, EF Core 9.0.7+, and `Amazon.AuroraDsql.Npgsql` 1.1.0+. | Dialect | Provided by `aurora-dsql-hibernate-dialect` (auto-registered) | | ID generation | `@GeneratedValue(strategy = GenerationType.UUID)` | | OCC retry | Prefer the [aurora-dsql-jdbc-connector](https://github.com/awslabs/aurora-dsql-connectors/tree/main/java/jdbc) — built-in retry for SQLSTATE 40001. For manual `@Retryable`, match on `SQLException` and check `getSQLState() == "40001"` (Hibernate's class-40 mapping varies by version). | +| Locking | `PESSIMISTIC_WRITE` supports non-key predicates and inner-joined queries; it adds commit-time OCC checks rather than blocking row locks | | FK constraints | Keep normal relationship mappings; the DSQL dialect exports foreign key constraints | | DDL generation | `hibernate.hbm2ddl.auto = none` — manage DDL manually | ### Rails -| Issue | Fix | -| ---------- | ------------------------------------------------------------------------------------------------------------------- | -| adapter | `postgresql` (standard pg gem) | -| Auth | Custom connection handler generating IAM tokens via `aws-sdk-dsql` | -| Migrations | `disable_ddl_transaction!` in each migration | -| PKs | `id: :uuid` in `create_table` | -| FKs | Use `add_foreign_key ..., validate: false`, then run `ALTER TABLE ASYNC ... VALIDATE CONSTRAINT` and verify the job | -| Locking | Use `lock!` / `with_lock` when a decision depends on rows read; retain OCC retry in `ApplicationRecord` | +| Issue | Fix | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| adapter | `postgresql` (standard pg gem) | +| Auth | Custom connection handler generating IAM tokens via `aws-sdk-dsql` | +| Migrations | `disable_ddl_transaction!` in each migration | +| PKs | `id: :uuid` in `create_table` | +| FKs | Use `add_foreign_key ..., validate: false`, then run `ALTER TABLE ASYNC ... VALIDATE CONSTRAINT` and verify the job | +| Locking | Use `Relation#lock` for non-key filters and joins inside a transaction; use `lock!` or `with_lock` for individual persisted records; retry the whole transaction on SQLSTATE `40001` | ### SQLAlchemy diff --git a/tools/evals/databases-on-aws/README.md b/tools/evals/databases-on-aws/README.md index 54e19399..d904fccc 100644 --- a/tools/evals/databases-on-aws/README.md +++ b/tools/evals/databases-on-aws/README.md @@ -14,7 +14,7 @@ their recorded results, and runner files: tools/evals/databases-on-aws/ ├── README.md # This file — top-level index └── dsql/ # Aurora DSQL skill evals - ├── evals.json # Tier 2: functional evals (21 prompts, 85 assertions) + ├── evals.json # Tier 2: functional evals (22 prompts, 93 assertions) ├── dsql_lint_evals.json # dsql_lint workflow (4 prompts, 20 assertions) ├── pg_migration_evals.json # PostgreSQL migrations (17 prompts, 90 assertions) ├── pg_migration_hallucination_evals.json # Migration hallucinations (3 prompts, 14 assertions) @@ -96,31 +96,32 @@ mise exec -- python tools/evals/databases-on-aws/dsql/scripts/run_functional_eva --verbose ``` -**What it checks** (21 eval prompts, 85 assertions total): - -| Eval | Focus | Grader | Key assertions | -| ------------------------------ | --------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------- | -| 1. Transaction limits | MCP delegation | regex | Calls `awsknowledge`, cites 3,000 row limit, recommends batching | -| 2. Multi-tenant schema | Correctness | LLM judge | Uses non-null tenant keys, tenant-scoped foreign keys, `CREATE INDEX ASYNC`, separate DDL txns | -| 3. Index limits | MCP delegation | regex | Calls `awsknowledge`, cites 24 index limit, suggests alternatives | -| 4. Python connection | Language routing | regex | Recommends DSQL Python Connector, IAM auth, 15-min token expiry, SSL | -| 5. Column type change | DDL migration routing | LLM judge | Table Recreation Pattern, dependency gate, batching, user confirmation | -| 6. JSON column storage | Type guidance | LLM judge | Recommends `JSONB` (or `JSON`) as the column type for queryable structured data | -| 7. Array storage | Type guidance | LLM judge | Flags `TEXT[]` / array column as unsupported, recommends storing the array as `JSONB` | -| 8. INACTIVE cluster error | Troubleshooting | LLM judge | Identifies INACTIVE state, uses `aws dsql get-cluster` to poll until `ACTIVE`, retries afterwards | -| 9. Backup on IDLE/INACTIVE | Troubleshooting | LLM judge | Identifies `FailedPrecondition`, connects to wake cluster to ACTIVE, retries backup | -| 10. Loader stuck at 3K rec/s | Data loading | LLM judge | Identifies partition-constrained fresh table, advises to keep running, does NOT recommend more workers | -| 11. Loader crash lost manifest | Data loading | LLM judge | Identifies /tmp as tmpfs, recommends --on-conflict do-nothing for recovery, --manifest-dir for prevention | -| 12. Header row parse error | Data loading | LLM judge | Identifies missing --header flag, explains default behavior, recommends fix | -| 13. EF Core data layer setup | ORM routing (.NET) | LLM judge | Recommends Amazon.AuroraDsql.EntityFrameworkCore, Guid keys w/ gen_random_uuid(), foreign key constraints, DsqlExecutionStrategy | -| 14. .NET / C# support | Language routing | LLM judge | Confirms .NET support, recommends Npgsql connector for IAM auth and EF Core adapter | -| 15. System diagnostics (W12) | AAS interpretation | LLM judge | Identifies the shifted wait event vs baseline, rules out load growth, no absolute-AAS claim, defers to Workflow 9 | -| 16. Wait-event ≠ plan (W12) | Observe-only boundary | LLM judge | High SequentialScanRead = concurrency not full scan; does NOT claim a full/seq scan or missing/building index; routes to Workflow 9 | -| 17. Shared reference FK | Multi-tenant design | LLM judge | Uses an ordinary foreign key to a global/shared parent and keeps authorization separate | -| 18. Add UNIQUE constraint | Constraint migration | LLM judge | Async unique index, readiness verification, `UNIQUE USING INDEX`, no table recreation | -| 19. Modify referenced PK | Constraint migration | LLM judge | Inbound-FK preflight, retained unique target, approval or abort when relationships would be lost | -| 20. Direct constraint changes | Constraint migration | LLM judge | Direct `DROP CONSTRAINT`, CHECK `NOT VALID`, async validation, no table recreation | -| 21. Direct column options | Column migration | LLM judge | Direct `DROP NOT NULL` and default changes, no table recreation | +**What it checks** (22 eval prompts, 93 assertions total): + +| Eval | Focus | Grader | Key assertions | +| ------------------------------ | --------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1. Transaction limits | MCP delegation | regex | Calls `awsknowledge`, cites 3,000 row-modification limit, recommends batching | +| 2. Multi-tenant schema | Correctness | LLM judge | Uses non-null tenant keys, tenant-scoped foreign keys, `CREATE INDEX ASYNC`, separate DDL txns | +| 3. Index limits | MCP delegation | regex | Calls `awsknowledge`, cites 24 index limit, suggests alternatives | +| 4. Python connection | Language routing | regex | Recommends DSQL Python Connector, IAM auth, 15-min token expiry, SSL | +| 5. Column type change | DDL migration routing | LLM judge | Table Recreation Pattern, dependency gate, batching, user confirmation | +| 6. JSON column storage | Type guidance | LLM judge | Recommends `JSONB` (or `JSON`) as the column type for queryable structured data | +| 7. Array storage | Type guidance | LLM judge | Flags `TEXT[]` / array column as unsupported, recommends storing the array as `JSONB` | +| 8. INACTIVE cluster error | Troubleshooting | LLM judge | Identifies INACTIVE state, uses `aws dsql get-cluster` to poll until `ACTIVE`, retries afterwards | +| 9. Backup on IDLE/INACTIVE | Troubleshooting | LLM judge | Identifies `FailedPrecondition`, connects to wake cluster to ACTIVE, retries backup | +| 10. Loader stuck at 3K rec/s | Data loading | LLM judge | Identifies partition-constrained fresh table, advises to keep running, does NOT recommend more workers | +| 11. Loader crash lost manifest | Data loading | LLM judge | Identifies /tmp as tmpfs, recommends --on-conflict do-nothing for recovery, --manifest-dir for prevention | +| 12. Header row parse error | Data loading | LLM judge | Identifies missing --header flag, explains default behavior, recommends fix | +| 13. EF Core data layer setup | ORM routing (.NET) | LLM judge | Recommends Amazon.AuroraDsql.EntityFrameworkCore, Guid keys w/ gen_random_uuid(), foreign key constraints, DsqlExecutionStrategy | +| 14. .NET / C# support | Language routing | LLM judge | Confirms .NET support, recommends Npgsql connector for IAM auth and EF Core adapter | +| 15. System diagnostics (W12) | AAS interpretation | LLM judge | Identifies the shifted wait event vs baseline, rules out load growth, no absolute-AAS claim, defers to Workflow 9 | +| 16. Wait-event ≠ plan (W12) | Observe-only boundary | LLM judge | High SequentialScanRead = concurrency not full scan; does NOT claim a full/seq scan or missing/building index; routes to Workflow 9 | +| 17. Shared reference FK | Multi-tenant design | LLM judge | Uses an ordinary foreign key to a global/shared parent and keeps authorization separate | +| 18. Add UNIQUE constraint | Constraint migration | LLM judge | Async unique index, readiness verification, `UNIQUE USING INDEX`, no table recreation | +| 19. Modify referenced PK | Constraint migration | LLM judge | Inbound-FK preflight, retained unique target, approval or abort when relationships would be lost | +| 20. Direct constraint changes | Constraint migration | LLM judge | Direct `DROP CONSTRAINT`, CHECK `NOT VALID`, async validation, no table recreation | +| 21. Direct column options | Column migration | LLM judge | Direct `DROP NOT NULL` and default changes, no table recreation | +| 22. Joined SELECT FOR UPDATE | ORM routing (Django) | LLM judge | Keeps joined/non-key locking query, explains commit-time OCC checks, uses targeted-row primary-key accounting toward the 10 MiB transaction-size limit, covers lock clauses, retries SQLSTATE 40001 | ### Grader modes diff --git a/tools/evals/databases-on-aws/dsql/evals.json b/tools/evals/databases-on-aws/dsql/evals.json index f2afe4b5..bfac8e8f 100644 --- a/tools/evals/databases-on-aws/dsql/evals.json +++ b/tools/evals/databases-on-aws/dsql/evals.json @@ -5,15 +5,17 @@ { "id": 1, "prompt": "How many rows can I modify in a single DSQL transaction? I'm planning a data migration of about 10k rows and need to know how to batch it.", - "expected_output": "Explains the 3,000 row limit, recommends batching strategy with 500-1000 row batches, and verifies the limit via awsknowledge MCP", + "expected_output": "Explains the 3,000 row-modification limit, recommends batching strategy with 500-1000 row batches, and verifies the limit via awsknowledge MCP", "expectations": [ "Calls awsknowledge search_documentation with a transaction-related query", - "Mentions the 3,000 row per transaction limit", + "Mentions the 3,000 row-modification per transaction limit", "Recommends a batching strategy for the 10k row migration", "Mentions the 10 MiB data size limit per transaction" ], "grader": "regex", - "required_mcp_servers": ["awsknowledge"] + "required_mcp_servers": [ + "awsknowledge" + ] }, { "id": 2, @@ -39,7 +41,9 @@ "Suggests alternatives such as composite indexes or reducing index count" ], "grader": "regex", - "required_mcp_servers": ["awsknowledge"] + "required_mcp_servers": [ + "awsknowledge" + ] }, { "id": 4, @@ -256,6 +260,22 @@ "Does NOT recommend the Table Recreation Pattern" ], "grader": "llm_judge" + }, + { + "id": 22, + "prompt": "My Django transaction uses a non-nullable customer relationship and Order.objects.select_related(\"customer\").filter(tenant_id=tenant_id, status=\"pending\").select_for_update() before updating an order. Can Aurora DSQL run this joined SELECT FOR UPDATE even though the filter is not an equality predicate on the primary key? Include supported and unsupported lock clauses, transaction-size accounting, concurrency behavior, and retry guidance.", + "expected_output": "Keeps the joined non-key select_for_update query, states that equality on every primary-key column is not required, explains that rows targeted by the locking clause participate in optimistic commit-time conflict checks instead of blocking locks, states that each targeted row's primary key counts toward the 10 MiB transaction-size limit, identifies the supported and unsupported lock clauses, and retries the whole transaction on SQLSTATE 40001", + "expectations": [ + "States that SELECT FOR UPDATE supports non-primary-key predicates and does not require equality on every primary-key column", + "States that SELECT FOR UPDATE supports queries that join multiple tables", + "Keeps Django select_for_update() instead of replacing it solely because the query uses a joined non-key predicate", + "Explains that rows targeted by the locking clause participate in optimistic commit-time conflict checks rather than taking blocking row locks", + "States that the primary key of each row targeted by the locking clause counts toward the 10 MiB transaction-size limit", + "States that FOR UPDATE and FOR KEY SHARE are supported", + "States that FOR NO KEY UPDATE and FOR SHARE are unsupported", + "Recommends retrying the whole transaction with backoff when a conflict returns SQLSTATE 40001" + ], + "grader": "llm_judge" } ] } diff --git a/tools/evals/databases-on-aws/dsql/scripts/run_functional_evals.py b/tools/evals/databases-on-aws/dsql/scripts/run_functional_evals.py index 05b63350..8b45e2cf 100644 --- a/tools/evals/databases-on-aws/dsql/scripts/run_functional_evals.py +++ b/tools/evals/databases-on-aws/dsql/scripts/run_functional_evals.py @@ -3185,6 +3185,8 @@ def _build_judge_evidence(run_result: dict) -> str: AssertionRule.AWSKNOWLEDGE_INDEX, "mentions the 3,000 row per transaction limit": AssertionRule.TRANSACTION_ROW_LIMIT, + "mentions the 3,000 row-modification per transaction limit": + AssertionRule.TRANSACTION_ROW_LIMIT, "recommends a batching strategy for the 10k row migration": AssertionRule.BATCHING, "recommends a batching strategy": AssertionRule.BATCHING, @@ -4885,7 +4887,7 @@ def grade_eval( if _has_positive_window( text, r"\bbatch(?:es|ed|ing)?\b", - r"\b3[,.]?000[\s-]+(?:rows?|records?)\b", + r"\b3[,.]?000[\s-]+(?:rows?|records?|row[\s-]+modifications?)\b", r"\b(?:exceed|over|more\s+than|under|fewer|limit|" r"maximum|max|transaction|chunk)", ): @@ -4897,7 +4899,7 @@ def grade_eval( # --- Assertion: mentions 3,000 row limit --- elif rule is AssertionRule.TRANSACTION_ROW_LIMIT: row_limit_patterns = ( - r"\b3[,.]?000[\s-]+(?:rows?|records?)\b", + r"\b3[,.]?000[\s-]+(?:rows?|records?|row[\s-]+modifications?)\b", r"\btransactions?\b", r"\b(?:limit|maximum|max|at\s+most|up\s+to|" r"cannot\s+exceed|can't\s+exceed|" diff --git a/tools/evals/databases-on-aws/dsql/scripts/test_run_functional_evals.py b/tools/evals/databases-on-aws/dsql/scripts/test_run_functional_evals.py index fa80e0c8..eea56d35 100644 --- a/tools/evals/databases-on-aws/dsql/scripts/test_run_functional_evals.py +++ b/tools/evals/databases-on-aws/dsql/scripts/test_run_functional_evals.py @@ -1883,12 +1883,12 @@ def capture_judge_evidence(**kwargs): deterministic_eval = _eval_document()["evals"][0] deterministic_eval["expectations"] = [ - "Mentions the 3,000 row per transaction limit", + "Mentions the 3,000 row-modification per transaction limit", "Recommends a batching strategy", ] negated_result = { **successful_result, - "result_text": "3,000 rows is not the limit. Batching is unnecessary.", + "result_text": "3,000 row modifications is not the limit. Batching is unnecessary.", } negated_grading = RUNNER.grade_eval( deterministic_eval,