From db257562d94aa46c0b893e30edc639f56e902836 Mon Sep 17 00:00:00 2001 From: Alex Shapalov Date: Sun, 18 Oct 2026 18:07:39 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20`pgbot=20sql=20explain`=20=E2=80=94=20a?= =?UTF-8?q?=20query=20plan,=20read=20in=20plain=20English?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `sql` command group with `explain`: the plan-only form of EXPLAIN for one SELECT (GENERIC_PLAN on PG16+), run inside a READ ONLY transaction and never executed, rendered as a tree with a plain-English gloss per node, the planner's estimates, and a deterministic "What stands out" list (large seq scans, nested loops repeating a scan, sorts an index could avoid, filters applied after an index lookup). `--json` prints the raw plan; `--ai` appends a model's prose reading behind a disclosure prompt that names what is sent. The new internal/plan package (Prepare/Fetch/Parse/Gloss/Observe) is shared with the MCP explain_plan tool, which now also returns the observations. Cobra command matching is case-insensitive so `pgbot SQL EXPLAIN` works. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 20 ++ README.md | 64 ++++- cmd/pgbot/explain.go | 17 +- cmd/pgbot/main.go | 6 + cmd/pgbot/mcp_tools.go | 51 ++-- cmd/pgbot/sql.go | 202 ++++++++++++++++ cmd/pgbot/sql_test.go | 63 +++++ internal/ai/plan.go | 46 ++++ internal/ai/plan_test.go | 25 ++ internal/plan/fetch.go | 43 ++++ internal/plan/fetch_integration_test.go | 84 +++++++ internal/plan/observe.go | 89 +++++++ internal/plan/plan.go | 243 +++++++++++++++++++ internal/plan/plan_test.go | 297 ++++++++++++++++++++++++ internal/render/plan.go | 110 +++++++++ internal/render/plan_test.go | 99 ++++++++ 16 files changed, 1423 insertions(+), 36 deletions(-) create mode 100644 cmd/pgbot/sql.go create mode 100644 cmd/pgbot/sql_test.go create mode 100644 internal/ai/plan.go create mode 100644 internal/ai/plan_test.go create mode 100644 internal/plan/fetch.go create mode 100644 internal/plan/fetch_integration_test.go create mode 100644 internal/plan/observe.go create mode 100644 internal/plan/plan.go create mode 100644 internal/plan/plan_test.go create mode 100644 internal/render/plan.go create mode 100644 internal/render/plan_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0077b5f..62f4edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ separately by `model.SchemaVersion` (currently 1.5.0). ## [Unreleased] +### Added +- **`pgbot sql explain ""` | one query's plan, read in plain English — plan-only EXPLAIN, never executed (`--ai` for a prose reading) | | `ask "…"` · `explain` | a plain-language AI reading of the same deterministic findings | | `explain-finding ` | the catalogue page for a finding, offline | | `mcp` | serve the findings to an AI agent over MCP | @@ -603,6 +604,9 @@ pgbot tables # largest tables + row counts + seq-vs-index pgbot vacuum # autovacuum health per table — dead tuples + whether it's due pgbot tune # config-tuning recommendations from the workload pgbot explain # inspect, then have an AI explain the findings +pgbot sql explain "SELECT …" # one query's plan as a tree + plain English ($DATABASE_URL) + --json the raw EXPLAIN (FORMAT JSON) instead + --ai append a model's prose reading of the plan pgbot ask "why is it slow?" # AI answer grounded on the findings ($DATABASE_URL) --yes skip the data-disclosure confirmation prompt pgbot mcp # run as an MCP server over stdio (for AI agents) @@ -624,7 +628,8 @@ on stdio, so an AI agent can call pgbot as a read-only tool. It exposes your repo; it says what to search for. Also `pgbot indexes --correlate [--json]`. - `record_index_verdict` — store what the repo search found, so a later run over a longer window carries strengthening evidence (no DB connection needed) -- `explain_plan` — the planner's plan for a SELECT (plain EXPLAIN, never executed) +- `explain_plan` — the planner's plan for a SELECT (plain EXPLAIN, never executed), + with the same deterministic `observations` that `pgbot sql explain` prints - `schema_of` — a table's columns/indexes/constraints + row estimate, **no data** - `compare_to_baseline` — the `diff`, with its interval-honesty and reset caveats - `why` — the causal chains from stored history (symptom ← mechanism ← @@ -697,6 +702,63 @@ The plugin drives the `pgbot` binary, so install that first (`curl -fsSL https://pgbot.dev/install | sh`); set `DATABASE_URL` or pass a connection string per call, then ask *"is my Postgres healthy?"* +### `sql explain` — read a query plan in plain English + +```sh +pgbot sql explain "SELECT c.name, count(*) FROM orders o JOIN customers c ON c.id = o.customer_id + WHERE o.status = $1 GROUP BY c.name ORDER BY 2 DESC" +cat slow.sql | pgbot sql explain # the query on stdin +pgbot sql explain "$DATABASE_URL" "SELECT …" # connection first, then the query +``` + +`pgbot sql explain` runs the **plan-only** form of EXPLAIN for one SELECT — +inside a `READ ONLY` transaction, the query is planned and never executed — and +renders the plan the way you'd want a colleague to read it to you: + +``` +Query plan · estimates only — the query was planned, not executed + SELECT c.name, count(*) FROM orders o JOIN customers c ON c.id = o.customer_id … + +Sort cost=562.48..567.48 rows≈2.0K + sorts about 2.0K rows by (count(*)) DESC + └─ Aggregate (hashed) cost=432.82..452.82 rows≈2.0K + groups by c.name using a hash table; expects about 2.0K rows + └─ Hash Join (inner) cost=135.96..399.49 rows≈6.7K + inner join: builds a hash table from one side and probes it with the other + Hash Cond: (o.customer_id = c.id) + ├─ Bitmap Heap Scan on orders (as o) cost=79.96..325.96 rows≈6.7K + │ fetches the pages of orders marked in a bitmap by the index scan below + │ Filter: (amount > '0'::numeric) + │ └─ Bitmap Index Scan using orders_status_idx cost=0.00..78.29 rows≈6.7K + │ Index Cond: (status = $1) + └─ Hash cost=31.00..31.00 rows≈2.0K + └─ Seq Scan on customers (as c) cost=0.00..31.00 rows≈2.0K + reads every row of customers, planner expects about 2.0K rows + +What stands out + · The scan via orders_status_idx on orders applies (amount > '0'::numeric) after the + index lookup, so some rows are fetched and then discarded. … + +Costs and row counts are the planner's estimates, not measurements. +Planned as a generic plan: parameter values ($1, …) were unknown … +``` + +Each node gets one plain-English line; **What stands out** lists the shapes +worth a look — a sequential scan the planner expects to return many rows, a +nested loop that repeats a scan per outer row, a sort an index could avoid, a +filter applied after an index lookup — all derived from the planner's estimates, +offline, with no model involved. `pgbot SQL EXPLAIN …` works too (command names +are case-insensitive). + +- On PostgreSQL 16+ the plan is generic (`EXPLAIN (GENERIC_PLAN)`), so `$1`-style + placeholders need no values; older servers plan queries with literal values. +- Only a **single plain SELECT** is accepted — no writes, no `WITH`, no multiple + statements (exit 64 otherwise). pgbot never uses the executing form of EXPLAIN. +- `--json` prints the raw `EXPLAIN (FORMAT JSON)` output. +- `--ai` appends a model's prose reading of the same rendered plan, behind the + disclosure prompt below. Unlike the findings, **your SQL text can carry literal + values**, so the prompt names exactly what is sent: the query and its plan. + ### `explain` — optional AI layer `pgbot explain` runs the exact same read-only inspection, prints the diff --git a/cmd/pgbot/explain.go b/cmd/pgbot/explain.go index b44ce56..3a18ebd 100644 --- a/cmd/pgbot/explain.go +++ b/cmd/pgbot/explain.go @@ -136,18 +136,25 @@ func runExplain(cmd *cobra.Command, args []string, f inspectFlags, yes bool) err // confirmDisclosure identifies the remote destination before data is sent. // Local endpoints do not send findings off the machine and need no confirmation. func confirmDisclosure(cmdName string, llm ai.LanguageModel, yes bool) bool { + return confirmDisclosureOf(cmdName, "the PII-free findings (same as `inspect --json`)", llm, yes) +} + +// confirmDisclosureOf is confirmDisclosure with the payload named: `sql explain +// --ai` sends the user's SQL text and its plan, which — unlike the findings — +// can carry literal values, so the prompt must say exactly that. +func confirmDisclosureOf(cmdName, payload string, llm ai.LanguageModel, yes bool) bool { if ai.Local(llm.Endpoint()) { - fmt.Fprintf(os.Stderr, "%s: using a local model at %s (%s) — the findings do not leave this machine.\n", - cmdName, ai.Host(llm.Endpoint()), llm.Model()) + fmt.Fprintf(os.Stderr, "%s: using a local model at %s (%s) — %s does not leave this machine.\n", + cmdName, ai.Host(llm.Endpoint()), llm.Model(), payload) return true } - fmt.Fprintf(os.Stderr, "%s: this sends the PII-free findings (same as `inspect --json`) to %s at %s (model %s).\n", - cmdName, llm.Provider(), ai.Host(llm.Endpoint()), llm.Model()) + fmt.Fprintf(os.Stderr, "%s: this sends %s to %s at %s (model %s).\n", + cmdName, payload, llm.Provider(), ai.Host(llm.Endpoint()), llm.Model()) if strings.HasPrefix(strings.ToLower(llm.Endpoint()), "http://") { // Not local (that returned above), and not TLS: the API key and the // findings cross the network in the clear. Say so; it is usually a LAN // address copied from a "local model" recipe. - fmt.Fprintf(os.Stderr, "%s: WARNING — %s is plain http, so the key and the findings travel unencrypted.\n", + fmt.Fprintf(os.Stderr, "%s: WARNING — %s is plain http, so the key and the payload travel unencrypted.\n", cmdName, ai.Host(llm.Endpoint())) } if yes || !isInteractive() { diff --git a/cmd/pgbot/main.go b/cmd/pgbot/main.go index 0b8d92e..e9ebaf6 100644 --- a/cmd/pgbot/main.go +++ b/cmd/pgbot/main.go @@ -29,6 +29,11 @@ func main() { // way out rather than per-Target (--all-databases and `mcp` open many). defer conn.CloseSSHTunnel() + // `pgbot SQL EXPLAIN ...` reads like the statement it wraps; cobra matches + // command names case-insensitively when this is set, so no two spellings need + // separate commands. No two pgbot commands differ only by case. + cobra.EnableCaseInsensitive = true + root := &cobra.Command{ Use: "pgbot", Short: "In-database observability for PostgreSQL", @@ -59,6 +64,7 @@ func main() { root.AddCommand(newERDCmd()) root.AddCommand(newActivityCmd()) root.AddCommand(newReportCmd()) + root.AddCommand(newSQLCmd()) // --ssh-tunnel is global: every command that takes a connection can need it, // and it changes only HOW the DSN is reached, never what is inspected. diff --git a/cmd/pgbot/mcp_tools.go b/cmd/pgbot/mcp_tools.go index 8662943..f708fb0 100644 --- a/cmd/pgbot/mcp_tools.go +++ b/cmd/pgbot/mcp_tools.go @@ -8,8 +8,8 @@ import ( "time" "github.com/pgrundev/pgbot/docs" - "github.com/pgrundev/pgbot/internal/advisor" "github.com/pgrundev/pgbot/internal/conn" + "github.com/pgrundev/pgbot/internal/plan" "github.com/pgrundev/pgbot/internal/store" "github.com/pgrundev/pggo" ) @@ -22,8 +22,9 @@ import ( // — a third-party client can still reword or drop it on its side. // explainPlanTool runs a PLAIN EXPLAIN (the plan-only form) of an agent-supplied query -// through the same guard as the advisor: SanitizeSelect (single plain SELECT, no -// semicolons) plus a READ ONLY transaction. The query is planned, never executed. +// through plan.Fetch — the same guarded path as `pgbot sql explain`: SanitizeSelect +// (single plain SELECT, no semicolons) plus a READ ONLY transaction. The query is +// planned, never executed. func explainPlanTool(ctx context.Context, args json.RawMessage) (string, error) { var a struct { ConnectionString string `json:"connection_string"` @@ -33,9 +34,8 @@ func explainPlanTool(ctx context.Context, args json.RawMessage) (string, error) if strings.TrimSpace(a.Query) == "" { return "", fmt.Errorf("query is required") } - clean := advisor.SanitizeSelect(a.Query) - if clean == "" { - return "", fmt.Errorf("only a single plain SELECT can be explained — no writes, no multiple statements, no utility statements") + if _, err := plan.Prepare(a.Query, 160000); err != nil { + return "", err } dsn, err := dsnFromArgs(args) if err != nil { @@ -49,35 +49,26 @@ func explainPlanTool(ctx context.Context, args json.RawMessage) (string, error) } defer target.Close() - // GENERIC_PLAN (PG16+) plans a query with $N placeholders without values; - // below 16, plain EXPLAIN works for a query with literal values and errors - // clearly on one with placeholders. - stmt := "EXPLAIN (FORMAT JSON) " + clean - if target.Caps.VersionNum >= 160000 { - stmt = "EXPLAIN (GENERIC_PLAN, FORMAT JSON) " + clean - } - var planJSON []byte - err = target.ReadOnlyTx(ctx, func(tx *pggo.Tx) error { - res, e := tx.Conn().SimpleQuery(ctx, stmt) - if e != nil { - return e - } - for _, r := range res { - if len(r.Rows) > 0 && len(r.Rows[0]) > 0 { - planJSON = r.Rows[0][0] - return nil - } - } - return fmt.Errorf("no plan returned") - }) + planJSON, generic, err := plan.Fetch(ctx, target, a.Query) if err != nil { - return "", fmt.Errorf("explain: %s", conn.RedactConnString(err.Error())) + return "", err } - return marshal(map[string]any{ + out := map[string]any{ "plan": json.RawMessage(planJSON), "exactness": "estimate", "note": "plain EXPLAIN — the query was planned, NOT executed. Costs and row counts are the planner's ESTIMATES, not measurements; state them as such.", - }) + } + if root, perr := plan.Parse(planJSON); perr == nil { + var obs []map[string]string + for _, o := range plan.Observe(root) { + obs = append(obs, map[string]string{"level": o.Level, "text": o.Text}) + } + out["observations"] = obs + } + if generic { + out["generic_plan"] = true + } + return marshal(out) } // compareToBaselineTool is `pgbot diff` as a tool: it reads the local baseline diff --git a/cmd/pgbot/sql.go b/cmd/pgbot/sql.go new file mode 100644 index 0000000..06bfa98 --- /dev/null +++ b/cmd/pgbot/sql.go @@ -0,0 +1,202 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/pgrundev/pgbot/internal/ai" + "github.com/pgrundev/pgbot/internal/conn" + "github.com/pgrundev/pgbot/internal/plan" + "github.com/pgrundev/pgbot/internal/render" + "github.com/spf13/cobra" +) + +// newSQLCmd groups the statement-level tools. `pgbot sql explain` is the first: +// the planner's plan for one SELECT, read back in plain English. Cobra matches +// command names case-insensitively (see main), so `pgbot SQL EXPLAIN` works too. +func newSQLCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "sql", + Short: "Statement-level tools: explain a query's plan in plain language", + Long: "Tools that take one SQL statement rather than inspecting the whole database.\n" + + "Everything here is read-only: a statement is planned, never executed.", + } + cmd.AddCommand(newSQLExplainCmd()) + return cmd +} + +type sqlExplainFlags struct { + jsonOut bool + useAI bool + yes bool + noColor bool +} + +func newSQLExplainCmd() *cobra.Command { + var f sqlExplainFlags + cmd := &cobra.Command{ + Use: `explain [connection-string] "