Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions pi-web-search/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 4 additions & 6 deletions pi-web-search/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,13 @@
"smoke:classifier": "node --experimental-strip-types scripts/classifier-smoke.ts",
"prepublishOnly": "npm run audit:prod && npm test && npm run typecheck && npm run pack:check"
},
"dependencies": {
"@earendil-works/pi-schema": "1.5.0"
},
"peerDependencies": {
"@earendil-works/pi-ai": "*",
"@earendil-works/pi-coding-agent": "*",
"@earendil-works/pi-tui": "*",
"typebox": "*"
"@earendil-works/pi-tui": "*"
},
"peerDependenciesMeta": {
"@earendil-works/pi-ai": {
Expand All @@ -62,9 +64,6 @@
},
"@earendil-works/pi-tui": {
"optional": true
},
"typebox": {
"optional": true
}
},
"overrides": {
Expand All @@ -75,7 +74,6 @@
"@earendil-works/pi-coding-agent": "1.3.0",
"@earendil-works/pi-tui": "1.3.0",
"@types/node": "26.6.5",
"typebox": "1.3.37",
"typescript": "7.0.2"
},
"pi": {
Expand Down
6 changes: 3 additions & 3 deletions pi-web-search/scripts/package-content.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { SyntaxKind } from "typescript/unstable/ast";
import { createScanner } from "typescript/unstable/ast/scanner";

export const packageAllowlist = ["src", "README.md", "LICENSE", "llms.txt", "docs"];
const hostPeers = ["@earendil-works/pi-ai", "@earendil-works/pi-coding-agent", "@earendil-works/pi-tui", "typebox"];
const hostPeers = ["@earendil-works/pi-ai", "@earendil-works/pi-coding-agent", "@earendil-works/pi-tui"];

export interface PackageManifest {
name: string;
Expand All @@ -29,7 +29,7 @@ export function validatePackageContent(manifest: PackageManifest, files: Map<str
assert.ok(manifest.keywords.includes("pi-package"));
assert.deepEqual(manifest.files, packageAllowlist);
assert.deepEqual(manifest.pi.extensions, ["./src/index.ts"]);
assert.deepEqual(manifest.dependencies ?? {}, {}, "This source-only extension has no runtime dependencies");
assert.deepEqual(manifest.dependencies ?? {}, { "@earendil-works/pi-schema": "1.5.0" }, "The schema library is the only allowed runtime dependency");
for (const hook of ["preinstall", "install", "postinstall", "prepare"]) {
assert.equal(manifest.scripts?.[hook], undefined, `Consumers must not run lifecycle hook ${hook}`);
}
Expand All @@ -52,7 +52,7 @@ export function validatePackageContent(manifest: PackageManifest, files: Map<str
}
if (isBuiltin(specifier)) return;
const name = specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0];
assert.ok(Object.hasOwn(manifest.peerDependencies, name), `${path}: undeclared external import ${specifier}`);
assert.ok(Object.hasOwn(manifest.peerDependencies, name) || Object.hasOwn(manifest.dependencies ?? {}, name), `${path}: undeclared external import ${specifier}`);
};
const scanner = createScanner(true, undefined, contents);
let declaration: "import" | "export" | undefined;
Expand Down
88 changes: 44 additions & 44 deletions pi-web-search/src/format.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Type, type Static } from "@earendil-works/pi-ai";
import { arr, bool, int, lit, nul, num, obj, opt, str, union, type StaticObject } from "@earendil-works/pi-schema";
import type { AgentToolResult } from "@earendil-works/pi-coding-agent";
import {
DEFAULT_MAX_BYTES,
Expand Down Expand Up @@ -31,53 +31,53 @@ export const RESULTS_HEADING = "## Results";
export const SOURCES_HEADING = "## Sources";
export const WARNINGS_HEADING = "## Warnings";

const ProviderSchema = Type.Union([Type.Literal("exa"), Type.Literal("parallel"), Type.Literal("grep"), Type.Literal("sourcegraph"), Type.Literal("github"), Type.Literal("brave"), Type.Literal("jina")]);
const ProviderSchema = union([lit("exa"), lit("parallel"), lit("grep"), lit("sourcegraph"), lit("github"), lit("brave"), lit("jina")]);

/** One declared data contract for renderers, scripts, and failure results. */
const DetailsSchema = Type.Object({
resultId: Type.Optional(Type.String()),
storedCount: Type.Optional(Type.Integer({ minimum: 0 })),
storedClipped: Type.Optional(Type.Boolean()),
ref: Type.Optional(Type.String()),
cursor: Type.Optional(Type.String()),
more: Type.Optional(Type.Boolean()),
truncated: Type.Optional(Type.Boolean()),
matches: Type.Optional(Type.Integer({ minimum: 0 })),
passage: Type.Optional(Type.Object({ index: Type.Integer(), offset: Type.Integer(), nextOffset: Type.Optional(Type.Integer()), totalChars: Type.Integer(), matches: Type.Optional(Type.Integer()) })),
provider: Type.Optional(ProviderSchema),
providers: Type.Optional(Type.Array(ProviderSchema)),
requestId: Type.Optional(Type.String()),
resultCount: Type.Optional(Type.Integer({ minimum: 0 })),
sources: Type.Optional(Type.Array(Type.Object({ title: Type.String(), url: Type.String() }))),
searchResults: Type.Optional(Type.Array(Type.Object({
title: Type.Optional(Type.String()), url: Type.Optional(Type.String()),
query: Type.Optional(Type.String()), source: Type.Optional(Type.String()),
pageAge: Type.Optional(Type.Union([Type.String(), Type.Null()])),
citedText: Type.Optional(Type.String()), status: Type.Optional(Type.String()), type: Type.Optional(Type.String()),
ref: Type.Optional(Type.String()),
}))),
warnings: Type.Optional(Type.Array(Type.String())),
usage: Type.Optional(Type.Array(Type.Object({ name: Type.String(), count: Type.Number() }))),
grounded: Type.Optional(Type.Boolean()),
classifier: Type.Optional(Type.Object({
sufficient: Type.Boolean(), lowConfidence: Type.Boolean(), suppressed: Type.Integer({ minimum: 0 }),
suppressedUrls: Type.Array(Type.String()),
})),
scope: Type.Optional(Type.Union([Type.Literal("web"), Type.Literal("code"), Type.Literal("both")])),
classifierStatus: Type.Optional(Type.Union([Type.Literal("ran"), Type.Literal("disabled"), Type.Literal("unavailable"), Type.Literal("skipped")])),
error: Type.Optional(Type.Object({
code: Type.String(), message: Type.String(), status: Type.Optional(Type.Number()),
rpcCode: Type.Optional(Type.Number()), retryable: Type.Optional(Type.Boolean()), configPath: Type.Optional(Type.String()),
})),
}, { additionalProperties: false });
const DetailsProperties = {
resultId: opt(str()),
storedCount: opt(int({ minimum: 0 })),
storedClipped: opt(bool()),
ref: opt(str()),
cursor: opt(str()),
more: opt(bool()),
truncated: opt(bool()),
matches: opt(int({ minimum: 0 })),
passage: opt(obj({ index: int(), offset: int(), nextOffset: opt(int()), totalChars: int(), matches: opt(int()) }, { open: true })),
provider: opt(ProviderSchema),
providers: opt(arr(ProviderSchema)),
requestId: opt(str()),
resultCount: opt(int({ minimum: 0 })),
sources: opt(arr(obj({ title: str(), url: str() }, { open: true }))),
searchResults: opt(arr(obj({
title: opt(str()), url: opt(str()),
query: opt(str()), source: opt(str()),
pageAge: opt(union([str(), nul()])),
citedText: opt(str()), status: opt(str()), type: opt(str()),
ref: opt(str()),
}, { open: true }))),
warnings: opt(arr(str())),
usage: opt(arr(obj({ name: str(), count: num() }, { open: true }))),
grounded: opt(bool()),
classifier: opt(obj({
sufficient: bool(), lowConfidence: bool(), suppressed: int({ minimum: 0 }),
suppressedUrls: arr(str()),
}, { open: true })),
scope: opt(union([lit("web"), lit("code"), lit("both")])),
classifierStatus: opt(union([lit("ran"), lit("disabled"), lit("unavailable"), lit("skipped")])),
error: opt(obj({
code: str(), message: str(), status: opt(num()),
rpcCode: opt(num()), retryable: opt(bool()), configPath: opt(str()),
}, { open: true })),
};

export type WebSearchDetails = Static<typeof DetailsSchema>;
export type WebSearchDetails = StaticObject<typeof DetailsProperties>;

export const SearchOutputSchema = Type.Object({
status: Type.Union([Type.Literal("success"), Type.Literal("error")]),
text: Type.Optional(Type.String()),
...DetailsSchema.properties,
}, { additionalProperties: false });
export const SearchOutputSchema = obj({
status: union([lit("success"), lit("error")]),
text: opt(str()),
...DetailsProperties,
});

export interface TruncationLimits {
maxLines?: number;
Expand Down
5 changes: 3 additions & 2 deletions pi-web-search/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,9 @@ export default function webSearchExtension(pi: ExtensionAPI) {
namespace: { name: "search", description: "Cited public web and code retrieval." },
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
execute: (toolCallId, params, signal, onUpdate, ctx) =>
search(toolCallId, params, signal, onUpdate, ctx),
renderCall(args: SearchInput, theme) {
search(toolCallId, params as SearchInput, signal, onUpdate, ctx),
renderCall(rawArgs: unknown, theme) {
const args = rawArgs as SearchInput;
const target = args.query ?? args.ref ?? (args.urls?.length ? `${args.urls.length} URL(s)` : "…");
const scope = args.scope ?? "web";
const effort = args.effort ?? (scope === "both" ? "thorough" : "fast");
Expand Down
34 changes: 23 additions & 11 deletions pi-web-search/src/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import type {
AgentToolUpdateCallback,
ExtensionContext,
} from "@earendil-works/pi-coding-agent";
import { StringEnum, Type, type Static } from "@earendil-works/pi-ai";
import { arr, obj, opt, str, type Schema, type Static } from "@earendil-works/pi-schema";
import { MAX_QUERY_CHARS } from "./providers/config.ts";
import { PARALLEL_MCP_SERVER } from "./providers/parallel.ts";
import { executeSearch, trimQuery } from "./execute.ts";
Expand All @@ -18,37 +18,49 @@ export const SEARCH_EFFORTS = ["fast", "thorough"] as const;
export type SearchScope = (typeof SEARCH_SCOPES)[number];
export type SearchEffort = (typeof SEARCH_EFFORTS)[number];

export const SearchSchema = Type.Object({
query: Type.Optional(Type.String({
/** `{ type: "string", enum }`: Google-compatible string enum, identical JSON to pi-ai's StringEnum. */
function stringEnum<const T extends readonly string[]>(
values: T,
options?: { description?: string },
): Schema<T[number]> {
return {
type: "string",
enum: [...values],
...(options?.description ? { description: options.description } : {}),
};
}

export const SearchSchema = obj({
query: opt(str({
minLength: 1,
maxLength: MAX_QUERY_CHARS,
description: "Search terms; code scope expects literal identifiers/snippets",
})),
scope: Type.Optional(StringEnum(SEARCH_SCOPES, {
scope: opt(stringEnum(SEARCH_SCOPES, {
description: "web (default), code, or both",
})),
urls: Type.Optional(Type.Array(Type.String(), {
urls: opt(arr(str(), {
description: "Fetch HTTP(S) pages instead of searching",
maxItems: 20,
})),
ref: Type.Optional(Type.String({
ref: opt(str({
minLength: 1,
maxLength: 200,
description: "Read a returned evidence ref locally; no network",
})),
find: Type.Optional(Type.String({
find: opt(str({
minLength: 1,
maxLength: 200,
description: "With ref: case-insensitive literal passage match",
})),
cursor: Type.Optional(Type.String({
cursor: opt(str({
minLength: 1, maxLength: 200,
description: "With ref: copy the cursor from a previous read",
})),
effort: Type.Optional(StringEnum(SEARCH_EFFORTS, {
effort: opt(stringEnum(SEARCH_EFFORTS, {
description: "fast (default), or thorough (opt-in multi-source)",
})),
});
}, { open: true });

export type SearchInput = Static<typeof SearchSchema>;

Expand All @@ -67,7 +79,7 @@ export interface UnifiedSearchOptions extends RunSearchOptions {
readRef?: SearchReadRef;
}

const SEARCH_PARAMS = Object.keys(SearchSchema.properties);
const SEARCH_PARAMS = Object.keys(SearchSchema.properties as Record<string, unknown>);

/** The registered unified tool. */
const SEARCH_TOOL: SearchToolName = "search";
Expand Down
16 changes: 9 additions & 7 deletions pi-web-search/tests/compact-output.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import test from "node:test";
import { Check } from "typebox/value";
import { compile } from "@earendil-works/pi-schema";

import {
CLIP_MARKER,
Expand All @@ -15,6 +15,8 @@ import { formatSearchError } from "../src/utils.ts";
import { providerError } from "../src/providers/types.ts";
import { mergeStreamResults, previewEvidence, uniqueEvidence } from "../src/providers/results.ts";

const validateOutput = compile(SearchOutputSchema);

function textOf(result: { content: { type: string; text?: string }[] }): string {
const part = result.content[0];
return part?.type === "text" ? (part.text ?? "") : "";
Expand All @@ -32,7 +34,7 @@ test("hit refs are one-based, identical in rendered text and structured data", (
],
}, undefined, { resultId, storedCount: 2, storedClipped: false });

assert.equal(Check(SearchOutputSchema, output.structuredContent), true);
assert.equal(validateOutput.check(output.structuredContent), true);
const data = output.structuredContent as { text?: string; searchResults: Array<{ ref?: string; citedText?: string }> };
assert.equal("text" in data, false);
assert.deepEqual(data.searchResults.map((hit) => hit.ref), [hitRef(resultId, 0), hitRef(resultId, 1)]);
Expand Down Expand Up @@ -100,7 +102,7 @@ test("structured listing includes useful bounded evidence once; errors keep text
searchResults: [{ title: "Doc", url: "https://example.com", citedText: "x".repeat(800) }],
});
const listed = listing.structuredContent as { text?: string; searchResults: Array<{ citedText?: string; url?: string }> };
assert.equal(Check(SearchOutputSchema, listing.structuredContent), true);
assert.equal(validateOutput.check(listing.structuredContent), true);
assert.equal("text" in listed, false);
assert.equal(listed.searchResults[0].citedText, "x".repeat(EXCERPT_MAX_CHARS) + CLIP_MARKER);
assert.equal(listed.searchResults[0].url, "https://example.com");
Expand All @@ -110,7 +112,7 @@ test("structured listing includes useful bounded evidence once; errors keep text
resultCount: 0, grounded: false, sources: [], searchResults: [],
error: { code: "timeout", message: "web_search failed (timeout): boom" },
});
assert.equal(Check(SearchOutputSchema, failure.structuredContent), true);
assert.equal(validateOutput.check(failure.structuredContent), true);
assert.equal((failure.structuredContent as { text: string }).text, textOf(failure));
});

Expand Down Expand Up @@ -196,7 +198,7 @@ test("source-only records and fixed fields include JSON escaping in their budget
{ text: "\u0001".repeat(1000), providerKind: "exa" as const, warnings: Array(6).fill("\u0001".repeat(200)) },
]) {
const output = formatWebSearchResult(input);
assert.equal(Check(SearchOutputSchema, output.structuredContent), true);
assert.equal(validateOutput.check(output.structuredContent), true);
assert.ok(textOf(output).length <= OUTPUT_BUDGET_CHARS);
assert.ok(JSON.stringify(output.structuredContent).length <= OUTPUT_BUDGET_CHARS);
}
Expand All @@ -207,7 +209,7 @@ test("escaped failure envelopes fit without changing error identity or shortenin
configPath: "\u0001".repeat(20_000),
});
const output = formatSearchError("search", error);
assert.equal(Check(SearchOutputSchema, output.structuredContent), true);
assert.equal(validateOutput.check(output.structuredContent), true);
assert.ok(JSON.stringify(output.structuredContent).length <= OUTPUT_BUDGET_CHARS);
assert.equal(output.details.error?.code, "http_error");
assert.equal(output.details.error?.status, 503);
Expand Down Expand Up @@ -239,7 +241,7 @@ test("standalone citations survive structured output alongside search hits", ()
});
assert.deepEqual((output.structuredContent as { sources?: unknown }).sources, [source]);
assert.ok(textOf(output).includes(source.url));
assert.equal(Check(SearchOutputSchema, output.structuredContent), true);
assert.equal(validateOutput.check(output.structuredContent), true);
});

test("clipping warnings describe model previews and point to local evidence", () => {
Expand Down
Loading
Loading