From d737d2ddebdd0b63df09bdd50a687749f4552be7 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:46:35 -0500 Subject: [PATCH 01/72] Harden public SDLC truthfulness gates --- docs/provider-support.md | 10 +- scripts/audit-search-mvp-package.sh | 6 +- scripts/bazel-test.sh | 3 + scripts/build-public-cli-artifact.sh | 2 + scripts/check-docs.sh | 1 + scripts/check-provider-support-matrix.py | 261 +++++++++++++++++++++++ scripts/check-public-cli-artifact.sh | 129 +++++++++++ 7 files changed, 404 insertions(+), 8 deletions(-) create mode 100755 scripts/check-provider-support-matrix.py create mode 100755 scripts/check-public-cli-artifact.sh diff --git a/docs/provider-support.md b/docs/provider-support.md index a7bc050d0..ecce897a7 100644 --- a/docs/provider-support.md +++ b/docs/provider-support.md @@ -41,11 +41,11 @@ modes import them explicitly. ## Provider Smoke -Provider smoke coverage uses static local-history fixtures checked into the -repository. The public smoke target exercises supported imports, blocked -unsupported providers, provider filtering, citations, and deterministic search -without executing provider CLIs, reading real user history, requiring API keys, -or making network calls: +Provider smoke coverage uses public fixture data and generated local-history +trees. The public smoke target exercises supported imports, blocked unsupported +providers, provider filtering, citations, and deterministic search without +executing provider CLIs, reading real user history, requiring API keys, or +making network calls: ```bash bazel test //:provider_fixture_e2e --config=ci diff --git a/scripts/audit-search-mvp-package.sh b/scripts/audit-search-mvp-package.sh index 3675347df..f5f90832f 100755 --- a/scripts/audit-search-mvp-package.sh +++ b/scripts/audit-search-mvp-package.sh @@ -75,11 +75,11 @@ if tracked_files | grep -E '^(\.ctx/exec-plans|docs/exec-plans|.*exec[_-]plan.*\ fail 'execution plans are present in package-visible source' fi -if tracked_files | grep -E '^(examples|assets)/' | grep -E -i 'dashboard|work-[r]ecord|ctx-records|capture-spool|evidence|link-pr|publish|shim' >/dev/null; then +if tracked_files | grep -E '^(examples|assets)/' | grep -E -i 'dashboard|work-[r]ecord|ctx-records|capture-spool|evidence|link-pr|publish|shim|provider-live|completion-certificate|freebsd-native-release-proof|r2-' >/dev/null; then fail 'tracked examples or assets contain removed product-surface material' fi -if grep_files 'dashboard|shim|shims|pull request|pull-request|pr evidence|pr-evidence|ctx publish|ctx evidence|ctx pr|ctx link-pr|ctx context|ctx update|ctx uninstall|\bADE\b|\b[Aa]mp\b|[Aa]mpcode|normalized-only|normalized only|normalized_import_only|normalized provider JSONL|CTX_PROVIDER_NORMALIZED_IMPORT_DEV|[W]ork Recorder|[w]ork recorder|\bwork-[r]ecord\b' \ +if grep_files 'dashboard|shim|shims|pull request|pull-request|pr evidence|pr-evidence|ctx publish|ctx evidence|ctx pr|ctx link-pr|ctx context|ctx update|ctx uninstall|\bADE\b|\b[Aa]mp\b|[Aa]mpcode|normalized-only|normalized only|normalized_import_only|normalized provider JSONL|CTX_PROVIDER_NORMALIZED_IMPORT_DEV|provider-live|completion-certificate|freebsd-native-release-proof|r2-|[W]ork Recorder|[w]ork recorder|\bwork-[r]ecord\b' \ "${public_user_docs[@]}" >/dev/null 2>&1; then fail 'public docs contain removed product-surface wording' fi @@ -93,7 +93,7 @@ if ! diff -u skills/ctx-agent-history-search/SKILL.md plugins/ctx-agent-history- fail 'plugin skill copy differs from public skill source' fi -if grep_files '[W]ork Recorder|[w]ork recorder|ctx publish|ctx evidence|ctx pr|ctx link-pr|ctx context|ctx update|ctx uninstall|update checks|auto-update|update-state|auto_update|CTX_UPDATE|release manifest|dashboard export|gh CLI|GhCli|upsert_github|write-shim-command|write_shim_command|capture_shim_command|shim_command_envelope|\bADE\b|\b[Aa]mp\b|[Aa]mpcode' \ +if grep_files '[W]ork Recorder|[w]ork recorder|ctx publish|ctx evidence|ctx pr|ctx link-pr|ctx context|ctx update|ctx uninstall|update checks|auto-update|update-state|auto_update|CTX_UPDATE|release manifest|provider-live|completion-certificate|freebsd-native-release-proof|r2-|dashboard export|gh CLI|GhCli|upsert_github|write-shim-command|write_shim_command|capture_shim_command|shim_command_envelope|\bADE\b|\b[Aa]mp\b|[Aa]mpcode' \ .bazelignore .bazelrc .bazelversion .buildkite .gitignore README.md SECURITY.md docs skills scripts crates/ctx-cli/src >/dev/null 2>&1; then fail 'public docs/help/release path contains removed product-surface text' fi diff --git a/scripts/bazel-test.sh b/scripts/bazel-test.sh index 7699040e7..d6f6acaa9 100755 --- a/scripts/bazel-test.sh +++ b/scripts/bazel-test.sh @@ -108,6 +108,9 @@ case "${mode}" in provider_fixture_e2e) run_cargo_test -p ctx --test cli codex_cli_provider_oracle_covers_retrieval_and_claimed_fidelity run_cargo_test -p ctx --test cli pi_cli_import_search_flow + run_cargo_test -p ctx --test cli native_provider_cli_flow_imports_new_supported_provider_paths + run_cargo_test -p ctx --test cli native_provider_cli_requires_existing_history_or_explicit_path + run_cargo_test -p ctx --test cli antigravity_cli_imports_native_transcript_tree ;; local_transcript_oracle) run_cargo_test -p ctx --test cli local_transcript_oracle_preserves_cli_json_and_sqlite diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index 4a7cd0c64..baae36177 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -110,4 +110,6 @@ case "${platform}" in ;; esac +scripts/check-public-cli-artifact.sh "${platform}" "${out_dir}" + printf 'built %s for %s sha256=%s\n' "${staged}" "${platform}" "$(cat "${sha_file}")" diff --git a/scripts/check-docs.sh b/scripts/check-docs.sh index fc93ea6a7..a491edf5f 100755 --- a/scripts/check-docs.sh +++ b/scripts/check-docs.sh @@ -40,6 +40,7 @@ done if command -v jq >/dev/null 2>&1; then jq empty docs/provider-support-matrix.json fi +python3 scripts/check-provider-support-matrix.py public_docs=( README.md diff --git a/scripts/check-provider-support-matrix.py b/scripts/check-provider-support-matrix.py new file mode 100755 index 000000000..87f2cbe55 --- /dev/null +++ b/scripts/check-provider-support-matrix.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Validate the public provider support matrix. + +This is a public truthfulness gate. It checks that documented provider support +has public docs, public tests, and any claimed fixture paths in the repository. +It intentionally does not require live provider runs, private fixture +provenance, release evidence, or network access. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[1] +MATRIX_PATH = REPO_ROOT / "docs/provider-support-matrix.json" +ALLOWED_STATUSES = { + "local_import", + "local_import_when_supported", + "fixture_only", + "detected_unsupported", + "blocked", +} +ALLOWED_PATH_KINDS = { + "native_import", + "fixture_import", + "detected_unsupported", + "blocked", +} +ALLOWED_FIDELITY = { + "imported", + "partial", + "fixture_only", + "detected_unsupported", + "blocked", +} +REQUIRED_FIDELITY_FIELDS = { + "user_prompts", + "assistant_messages", + "tool_calls", + "tool_output", + "command_output", + "files_touched", + "artifacts", + "model_identity", + "costs", + "token_usage", + "parent_child_session_edges", +} +PROVIDER_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_]*$") +PRIVATE_TEXT_MARKERS = ("/home/", "ctx-" + "private", "ctx-multi" + "-repo-workspace") +SUPPORT_DOC_PATH = REPO_ROOT / "docs/provider-support.md" + + +class MatrixError(Exception): + pass + + +def fail(message: str) -> None: + raise MatrixError(message) + + +def expect_type(value: Any, expected_type: type, field: str) -> Any: + if not isinstance(value, expected_type): + fail(f"{field} must be {expected_type.__name__}") + return value + + +def require_non_empty_string(value: Any, field: str) -> str: + text = expect_type(value, str, field) + if not text.strip(): + fail(f"{field} must be non-empty") + return text + + +def require_string_list(value: Any, field: str, *, allow_empty: bool = False) -> list[str]: + items = expect_type(value, list, field) + if not allow_empty and not items: + fail(f"{field} must not be empty") + for index, item in enumerate(items): + require_non_empty_string(item, f"{field}[{index}]") + return items + + +def require_repo_path(value: str, field: str) -> Path: + if value.startswith("/") or ".." in Path(value).parts: + fail(f"{field} must be a relative repository path") + path = REPO_ROOT / value + if not path.exists(): + fail(f"{field} does not exist: {value}") + return path + + +def scan_private_text(value: Any, field: str) -> None: + if isinstance(value, str): + if any(token in value for token in PRIVATE_TEXT_MARKERS): + fail(f"{field} contains private path wording") + return + if isinstance(value, list): + for index, item in enumerate(value): + scan_private_text(item, f"{field}[{index}]") + return + if isinstance(value, dict): + for key, item in value.items(): + scan_private_text(item, f"{field}.{key}") + + +def text_mentions_provider(text: str, provider: dict[str, Any]) -> bool: + needles = { + str(provider["id"]), + str(provider["capture_provider"]), + str(provider["capture_provider"]).replace("_", "-"), + str(provider["display_name"]), + str(provider["display_name"]).lower(), + } + lowered = text.lower() + return any(needle and needle.lower() in lowered for needle in needles) + + +def validate_implemented_path(path: Any, provider_id: str, index: int) -> None: + label = f"providers[{provider_id}].implemented_paths[{index}]" + expect_type(path, dict, label) + + kind = require_non_empty_string(path.get("kind"), f"{label}.kind") + if kind not in ALLOWED_PATH_KINDS: + fail(f"{label}.kind has unsupported value: {kind}") + + source_format = require_non_empty_string(path.get("source_format"), f"{label}.source_format") + if any(token in source_format for token in PRIVATE_TEXT_MARKERS): + fail(f"{label}.source_format contains private path wording") + + fidelity = require_non_empty_string(path.get("fidelity"), f"{label}.fidelity") + if fidelity not in ALLOWED_FIDELITY: + fail(f"{label}.fidelity has unsupported value: {fidelity}") + + proof = require_string_list(path.get("proof"), f"{label}.proof") + if not any("ctx " in item or item.startswith("cargo test") or item == "ctx sources" for item in proof): + fail(f"{label}.proof must name a public ctx command or cargo test") + + notes = require_string_list(path.get("notes", []), f"{label}.notes", allow_empty=True) + for note_index, note in enumerate(notes): + if any(token in note for token in PRIVATE_TEXT_MARKERS): + fail(f"{label}.notes[{note_index}] contains private path wording") + + +def validate_provider(provider: Any, index: int, seen_ids: set[str]) -> None: + label = f"providers[{index}]" + expect_type(provider, dict, label) + + provider_id = require_non_empty_string(provider.get("id"), f"{label}.id") + if not PROVIDER_ID_RE.fullmatch(provider_id): + fail(f"{label}.id must use lowercase snake_case") + if provider_id in seen_ids: + fail(f"duplicate provider id: {provider_id}") + seen_ids.add(provider_id) + scan_private_text(provider, f"providers[{provider_id}]") + + require_non_empty_string(provider.get("display_name"), f"providers[{provider_id}].display_name") + require_non_empty_string(provider.get("priority"), f"providers[{provider_id}].priority") + require_non_empty_string(provider.get("capture_provider"), f"providers[{provider_id}].capture_provider") + + status = require_non_empty_string(provider.get("status"), f"providers[{provider_id}].status") + if status not in ALLOWED_STATUSES: + fail(f"providers[{provider_id}].status has unsupported value: {status}") + + public_docs = require_non_empty_string(provider.get("public_docs"), f"providers[{provider_id}].public_docs") + public_doc_path = require_repo_path(public_docs, f"providers[{provider_id}].public_docs") + public_doc_text = public_doc_path.read_text(encoding="utf-8") + if provider["display_name"] not in public_doc_text and provider_id not in public_doc_text: + fail(f"providers[{provider_id}].public_docs does not mention the provider") + + support_doc_text = SUPPORT_DOC_PATH.read_text(encoding="utf-8") + support_row = f"| {provider['display_name']} | `{status}` |" + if support_row not in support_doc_text: + fail(f"docs/provider-support.md is missing matrix row for {provider_id} with status {status}") + + tests = require_string_list(provider.get("tests"), f"providers[{provider_id}].tests") + provider_specific_test = False + for test_index, test_path in enumerate(tests): + resolved_test_path = require_repo_path(test_path, f"providers[{provider_id}].tests[{test_index}]") + if resolved_test_path.is_file() and text_mentions_provider( + resolved_test_path.read_text(encoding="utf-8", errors="ignore"), + provider, + ): + provider_specific_test = True + + fixture_paths = require_string_list( + provider.get("fixture_paths", []), + f"providers[{provider_id}].fixture_paths", + allow_empty=True, + ) + for fixture_index, fixture_path in enumerate(fixture_paths): + require_repo_path(fixture_path, f"providers[{provider_id}].fixture_paths[{fixture_index}]") + + implemented_paths = expect_type( + provider.get("implemented_paths", []), + list, + f"providers[{provider_id}].implemented_paths", + ) + if not implemented_paths and status not in {"detected_unsupported", "blocked"}: + fail(f"providers[{provider_id}].implemented_paths must not be empty") + for path_index, implemented_path in enumerate(implemented_paths): + validate_implemented_path(implemented_path, provider_id, path_index) + + imports_existing_history = provider.get("imports_existing_history") + if not isinstance(imports_existing_history, bool): + fail(f"providers[{provider_id}].imports_existing_history must be boolean") + if status.startswith("local_import") and not imports_existing_history: + fail(f"providers[{provider_id}] is {status} but imports_existing_history is false") + if imports_existing_history and not implemented_paths: + fail(f"providers[{provider_id}] imports history but has no implemented_paths") + + for bool_field in ("captures_new_runs_passively", "child_sessions_supported"): + if not isinstance(provider.get(bool_field), bool): + fail(f"providers[{provider_id}].{bool_field} must be boolean") + + fidelity = expect_type(provider.get("fidelity"), dict, f"providers[{provider_id}].fidelity") + missing_fidelity = REQUIRED_FIDELITY_FIELDS.difference(fidelity) + if missing_fidelity: + fail(f"providers[{provider_id}].fidelity missing fields: {', '.join(sorted(missing_fidelity))}") + for field in REQUIRED_FIDELITY_FIELDS: + if not isinstance(fidelity[field], bool): + fail(f"providers[{provider_id}].fidelity.{field} must be boolean") + + if status == "local_import" and not fixture_paths: + fail(f"providers[{provider_id}] is local_import but has no fixture_paths") + if status.startswith("local_import") and "crates/ctx-cli/tests/cli.rs" not in tests: + fail(f"providers[{provider_id}] needs public CLI coverage in crates/ctx-cli/tests/cli.rs") + if status.startswith("local_import") and not provider_specific_test: + fail(f"providers[{provider_id}] has no provider-specific public test references") + + +def main() -> int: + try: + matrix = json.loads(MATRIX_PATH.read_text(encoding="utf-8")) + expect_type(matrix, dict, "provider support matrix") + scan_private_text(matrix, "provider support matrix") + if matrix.get("schema_version") != 1: + fail("schema_version must be 1") + require_non_empty_string(matrix.get("scope"), "scope") + providers = expect_type(matrix.get("providers"), list, "providers") + if not providers: + fail("providers must not be empty") + + seen_ids: set[str] = set() + for index, provider in enumerate(providers): + validate_provider(provider, index, seen_ids) + except (OSError, json.JSONDecodeError, MatrixError) as exc: + print(f"provider support matrix check failed: {exc}", file=sys.stderr) + return 1 + + print("provider support matrix ok") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-public-cli-artifact.sh b/scripts/check-public-cli-artifact.sh new file mode 100755 index 000000000..b223aecb7 --- /dev/null +++ b/scripts/check-public-cli-artifact.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: scripts/check-public-cli-artifact.sh PLATFORM [ARTIFACT_DIR] + +Checks one locally staged public ctx CLI artifact. This validates only local +public release outputs: artifact presence, SHA-256 sidecar consistency, and +version sidecar contents. +USAGE +} + +platform="${1:-}" +artifact_dir="${2:-target/public-cli-artifacts}" +if [[ -z "${platform}" || "${platform}" == "-h" || "${platform}" == "--help" ]]; then + usage + exit 2 +fi + +case "${platform}" in + linux-x64) + binary_name="ctx" + ;; + macos-arm64) + binary_name="ctx-macos-arm64" + ;; + macos-x64) + binary_name="ctx-macos-x64" + ;; + windows-x64) + binary_name="ctx.exe" + ;; + freebsd-x64) + binary_name="ctx-freebsd-x64" + ;; + *) + usage + exit 2 + ;; +esac + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; data=json.load(sys.stdin); print(next(pkg["version"] for pkg in data["packages"] if pkg["name"] == "ctx"))')" +artifact="${artifact_dir%/}/${binary_name}" +sha_file="${artifact}.sha256" +version_file="${artifact}.version" + +if [[ ! -f "${artifact}" ]]; then + printf 'public CLI artifact missing: %s\n' "${artifact}" >&2 + exit 1 +fi + +if [[ ! -s "${sha_file}" ]]; then + printf 'public CLI artifact SHA-256 sidecar missing or empty: %s\n' "${sha_file}" >&2 + exit 1 +fi + +expected_sha="$(tr -d '[:space:]' < "${sha_file}")" +if [[ ! "${expected_sha}" =~ ^[0-9a-fA-F]{64}$ ]]; then + printf 'public CLI artifact SHA-256 sidecar is not a digest: %s\n' "${sha_file}" >&2 + exit 1 +fi + +if command -v sha256sum >/dev/null 2>&1; then + actual_sha="$(sha256sum "${artifact}" | awk '{ print $1 }')" +else + actual_sha="$(shasum -a 256 "${artifact}" | awk '{ print $1 }')" +fi + +actual_sha_lower="$(printf '%s' "${actual_sha}" | tr 'A-F' 'a-f')" +expected_sha_lower="$(printf '%s' "${expected_sha}" | tr 'A-F' 'a-f')" +if [[ "${actual_sha_lower}" != "${expected_sha_lower}" ]]; then + printf 'public CLI artifact checksum mismatch for %s: expected %s got %s\n' \ + "${artifact}" "${expected_sha}" "${actual_sha}" >&2 + exit 1 +fi + +if [[ ! -s "${version_file}" ]]; then + printf 'public CLI artifact version sidecar missing or empty: %s\n' "${version_file}" >&2 + exit 1 +fi + +actual_version="$(tr -d '\r' < "${version_file}" | sed 's/[[:space:]]*$//' | tail -n 1)" +can_run_on_host=0 +case "${platform}" in + linux-x64) + if [[ "$(uname -s 2>/dev/null || true)" == "Linux" ]]; then + case "$(uname -m 2>/dev/null || true)" in + x86_64|amd64) can_run_on_host=1 ;; + esac + fi + ;; + macos-arm64) + if [[ "$(uname -s 2>/dev/null || true)" == "Darwin" && "$(uname -m 2>/dev/null || true)" == "arm64" ]]; then + can_run_on_host=1 + fi + ;; + macos-x64) + if [[ "$(uname -s 2>/dev/null || true)" == "Darwin" ]] && /usr/bin/arch -x86_64 /usr/bin/true >/dev/null 2>&1; then + can_run_on_host=1 + fi + ;; + freebsd-x64) + if [[ "$(uname -s 2>/dev/null || true)" == "FreeBSD" ]]; then + case "$(uname -m 2>/dev/null || true)" in + x86_64|amd64) can_run_on_host=1 ;; + esac + fi + ;; +esac + +case "${actual_version}" in + "ctx ${version}") ;; + "not run on this host: ${platform}") + if [[ "${can_run_on_host}" == "1" ]]; then + printf 'public CLI artifact version sidecar skipped a runnable host platform: %s\n' "${platform}" >&2 + exit 1 + fi + ;; + *) + printf 'public CLI artifact version sidecar has unexpected content: %s\n' "${actual_version}" >&2 + exit 1 + ;; +esac + +printf 'public CLI artifact ok: %s sha256=%s\n' "${platform}" "${actual_sha}" From 010666b8ab873f60f478467322756725430c540f Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:55:30 -0500 Subject: [PATCH 02/72] search: make subagent history opt-in Default search now favors primary sessions so human intent and decisions stay prominent. --include-subagents remains the explicit path for implementation details, review notes, and failure traces while the old primary-only input is kept as hidden compatibility. --- crates/ctx-cli/src/main.rs | 10 +++++++--- crates/ctx-cli/src/mcp.rs | 5 ++--- crates/ctx-cli/tests/cli.rs | 8 +++++--- crates/ctx-history-search/src/lib.rs | 2 +- docs/cli-reference.md | 6 +++++- docs/getting-started.md | 2 +- docs/search.md | 8 ++++---- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 0cad64cb5..08d6172d0 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -231,11 +231,15 @@ struct SearchArgs { help = "Filter to recent history, as RFC3339 or a day window like 30d" )] since: Option, - #[arg(long, help = "Return only primary-agent sessions")] + #[arg( + long, + hide = true, + help = "Deprecated alias for the default primary-agent search scope" + )] primary_only: bool, #[arg( long, - help = "Include subagent sessions; this is the default unless --primary-only is set" + help = "Include subagent sessions in addition to primary-agent sessions" )] include_subagents: bool, #[arg(long, help = "Filter by event type, such as message or tool_call")] @@ -4844,7 +4848,7 @@ fn search_filters( repo: input.workspace, since: input.since.as_deref().map(parse_since_filter).transpose()?, primary_only: input.primary_only, - include_subagents: input.include_subagents || !input.primary_only, + include_subagents: input.include_subagents && !input.primary_only, event_type: input .event_type .as_deref() diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index 1cf3eb1bb..b77c71ed0 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -324,7 +324,7 @@ fn tool_search(arguments: &Value, data_root: &Path) -> Result { let workspace = optional_string(arguments, "workspace")?; let since = optional_string(arguments, "since")?; let primary_only = optional_bool(arguments, "primary_only")?.unwrap_or(false); - let include_subagents = optional_bool(arguments, "include_subagents")?.unwrap_or(!primary_only); + let include_subagents = optional_bool(arguments, "include_subagents")?.unwrap_or(false); let event_type = optional_string(arguments, "event_type")?; let file = optional_string(arguments, "file")?.map(PathBuf::from); let events = optional_bool(arguments, "events")?.unwrap_or(false) || session.is_some(); @@ -495,8 +495,7 @@ fn tool_definitions() -> Vec { "provider": { "type": "string", "enum": provider_names() }, "workspace": { "type": "string", "description": "Workspace path or name text." }, "since": { "type": "string", "description": "RFC3339 timestamp or day window such as 30d." }, - "primary_only": { "type": "boolean", "default": false }, - "include_subagents": { "type": "boolean", "default": true }, + "include_subagents": { "type": "boolean", "default": false, "description": "Include subagent sessions in addition to primary-agent sessions." }, "event_type": { "type": "string", "enum": event_type_names() }, "file": { "type": "string" }, "session": { "type": "string", "description": "ctx session id." }, diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 2e0158f8f..b614f4257 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1057,8 +1057,6 @@ fn public_subcommand_help_is_golden_enough_for_session_retrieval() { "Filter by workspace path or name text", "--since ", "Filter to recent history, as RFC3339 or a day window like 30d", - "--primary-only", - "Return only primary-agent sessions", "--include-subagents", "Include subagent sessions", "--event-type ", @@ -2396,7 +2394,11 @@ fn codex_cli_resume_is_idempotent_rescan_and_filters_subagents() { assert_eq!(first["totals"]["imported_events"], 4); assert_eq!(first["totals"]["imported_edges"], 1); - let with_subagents = json_output(ctx(&temp).args(["search", "subagent", "--json"])); + let primary_default = json_output(ctx(&temp).args(["search", "subagent", "--json"])); + assert_eq!(primary_default["filters"]["include_subagents"], false); + + let with_subagents = + json_output(ctx(&temp).args(["search", "subagent", "--include-subagents", "--json"])); assert!(!with_subagents["results"].as_array().unwrap().is_empty()); assert_eq!(with_subagents["filters"]["include_subagents"], true); diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index e027c22f4..39e72ba2a 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -94,7 +94,7 @@ impl Default for SearchFilters { repo: None, since: None, primary_only: false, - include_subagents: true, + include_subagents: false, event_type: None, file: None, exclude_provider_session: None, diff --git a/docs/cli-reference.md b/docs/cli-reference.md index a9f656be8..8c4ed8dfc 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -154,6 +154,7 @@ ctx search "token budget" --refresh off ctx search "signed metadata" --term checksum --term release ctx search "token budget" --limit 5 ctx search "token budget" --session +ctx search "review findings" --include-subagents ctx search "this current task" --include-current-session ``` @@ -176,6 +177,10 @@ search has identified a session to inspect; scoped session search returns dense event hits. Use `--events` without `--session` for dense event-level results across sessions. Repeat `--term ` when you want to broaden a search across several related words or phrases and merge the ranked results. +Default search excludes subagent sessions so primary human-agent intent and +decisions stay prominent. Use `--include-subagents` when implementation details, +code review notes, test output, or failure analysis from subagent sessions +should be searched too. When ctx is run from Codex and `CODEX_THREAD_ID` is available, search excludes the active Codex session tree by default so the current task and its subagents @@ -201,7 +206,6 @@ Filters: - `--session `, for dense event results within one session; - `--term `, repeatable broadening terms merged with the main query; - `--events`, for dense event-level results instead of the default session-diverse results; -- `--primary-only`; - `--include-subagents`; - `--limit `, capped at `200`; - `--refresh auto|off|strict`; diff --git a/docs/getting-started.md b/docs/getting-started.md index 10b227c94..ed83a5749 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -92,7 +92,7 @@ ctx show session Use `ctx_event_id` with `ctx show event` when you need a hit plus surrounding events. Use `ctx_session_id` with `ctx show session` when you need the transcript. Search also accepts filters such as `--provider`, `--workspace`, -`--since`, `--event-type`, `--file`, `--primary-only`, `--include-subagents`, +`--since`, `--event-type`, `--file`, `--include-subagents`, `--include-current-session`, `--term`, `--limit`, and `--refresh auto|off|strict`. `--limit` is capped at `200`. diff --git a/docs/search.md b/docs/search.md index ed165b002..dd116a72c 100644 --- a/docs/search.md +++ b/docs/search.md @@ -20,6 +20,7 @@ ctx search "token budget" --refresh off ctx search "signed metadata" --term checksum --term release ctx search "token budget" --limit 5 ctx search "token budget" --session +ctx search "review findings" --include-subagents ctx search "this current task" --include-current-session ``` @@ -60,7 +61,6 @@ Search filters narrow both human output and JSON: - `--session `; - `--term `, repeatable broadening terms merged with the main query; - `--events`; -- `--primary-only`; - `--include-subagents`; - `--limit `; - `--refresh auto|off|strict`; @@ -75,9 +75,9 @@ for a file, or combine it with query terms to find sessions that both mention a topic and touched that path. It searches paths recorded during import; it does not inspect the current filesystem. -The default includes subagent material. `--primary-only` restricts results to -primary sessions and excludes subagent material. `--include-subagents` keeps the -default explicit; it does not override `--primary-only`. +The default searches primary-agent sessions so human intent and decisions stay +prominent. Use `--include-subagents` when you want implementation details, code +review notes, test output, or failure analysis from subagent sessions too. `--limit` defaults to `20` and is capped at `200`. From 3effe4037503631d7e5afb571d30048a25402350 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:58:43 -0500 Subject: [PATCH 03/72] sql: link touched files through capture sources ctx_files_touched now resolves provider/session/history-record metadata when a touch row is associated only by source_id, matching the existing --file search fallback. The schema migration recreates stable views without reimporting provider transcripts. --- crates/ctx-history-store/src/lib.rs | 169 +++++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 5 deletions(-) diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 3989ec54f..3605e9110 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -1039,16 +1039,27 @@ SELECT ft.line_count_delta AS line_count_delta, ft.confidence AS confidence, ft.event_id AS ctx_event_id, - COALESCE(e.session_id, r.session_id) AS ctx_session_id, - COALESCE(e.history_record_id, r.history_record_id, ft.history_record_id) AS history_record_id, - s.provider AS provider, - s.external_session_id AS provider_session_id, + COALESCE(e.session_id, r.session_id, source_session.id) AS ctx_session_id, + COALESCE( + e.history_record_id, + r.history_record_id, + ft.history_record_id, + event_session.history_record_id, + run_session.history_record_id, + source_session.history_record_id + ) AS history_record_id, + COALESCE(s.provider, cs.provider) AS provider, + COALESCE(s.external_session_id, cs.external_session_id) AS provider_session_id, ft.created_at_ms AS created_at_ms, ft.updated_at_ms AS updated_at_ms FROM files_touched ft LEFT JOIN events e ON e.id = ft.event_id LEFT JOIN runs r ON r.id = ft.run_id -LEFT JOIN sessions s ON s.id = COALESCE(e.session_id, r.session_id) +LEFT JOIN capture_sources cs ON cs.id = ft.source_id +LEFT JOIN sessions event_session ON event_session.id = e.session_id +LEFT JOIN sessions run_session ON run_session.id = r.session_id +LEFT JOIN sessions source_session ON source_session.capture_source_id = ft.source_id +LEFT JOIN sessions s ON s.id = COALESCE(e.session_id, r.session_id, source_session.id) WHERE ft.deleted_at_ms IS NULL; DROP VIEW IF EXISTS ctx_sources; @@ -7893,6 +7904,154 @@ mod catalog_tests { assert_eq!(result.rows[0][0], RawSqlValue::Integer(0)); } + #[test] + fn ctx_files_touched_resolves_session_from_source_id() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let record_id = "018f45d0-0000-7000-8000-000000080001"; + let source_id = "018f45d0-0000-7000-8000-000000080002"; + let session_id = "018f45d0-0000-7000-8000-000000080003"; + let touch_id = "018f45d0-0000-7000-8000-000000080004"; + let detached_source_id = "018f45d0-0000-7000-8000-000000080005"; + let detached_touch_id = "018f45d0-0000-7000-8000-000000080006"; + + store + .conn + .execute( + r#" + INSERT INTO history_records + (id, title, last_activity_at_ms, created_at_ms, updated_at_ms, body, created_at, updated_at) + VALUES (?1, 'Touched file view record', 1, 1, 1, '', '', '') + "#, + [record_id], + ) + .unwrap(); + store + .conn + .execute( + r#" + INSERT INTO capture_sources + (id, kind, provider, machine_id, raw_source_path, external_session_id, started_at_ms, fidelity) + VALUES (?1, 'provider_import', 'codex', 'test-machine', '/tmp/session.jsonl', 'codex-session-1', 1, 'imported') + "#, + [source_id], + ) + .unwrap(); + store + .conn + .execute( + r#" + INSERT INTO capture_sources + (id, kind, provider, machine_id, raw_source_path, external_session_id, started_at_ms, fidelity) + VALUES (?1, 'provider_import', 'opencode', 'test-machine', '/tmp/opencode.db', 'opencode-session-1', 1, 'imported') + "#, + [detached_source_id], + ) + .unwrap(); + store + .conn + .execute( + r#" + INSERT INTO sessions + ( + id, history_record_id, capture_source_id, provider, external_session_id, + agent_type, is_primary, status, fidelity, started_at_ms, created_at_ms, updated_at_ms + ) + VALUES (?1, ?2, ?3, 'codex', 'codex-session-1', 'primary', 1, 'imported', 'imported', 1, 1, 1) + "#, + params![session_id, record_id, source_id], + ) + .unwrap(); + store + .conn + .execute( + r#" + INSERT INTO files_touched + (id, source_id, path, change_kind, confidence, created_at_ms, updated_at_ms, fidelity) + VALUES (?1, ?2, 'src/main.rs', 'modified', 'explicit', 1, 1, 'imported') + "#, + params![touch_id, source_id], + ) + .unwrap(); + store + .conn + .execute( + r#" + INSERT INTO files_touched + (id, source_id, path, change_kind, confidence, created_at_ms, updated_at_ms, fidelity) + VALUES (?1, ?2, 'detached.rs', 'modified', 'explicit', 1, 1, 'imported') + "#, + params![detached_touch_id, detached_source_id], + ) + .unwrap(); + + let result = store + .raw_sql_query( + "SELECT provider, provider_session_id, ctx_session_id, history_record_id FROM ctx_files_touched WHERE path = 'src/main.rs'", + RawSqlOptions::default(), + ) + .unwrap(); + assert_eq!(result.returned_rows, 1); + assert_eq!( + result.rows[0][0], + RawSqlValue::Text { + value: "codex".to_owned(), + bytes: 5, + truncated: false, + } + ); + assert_eq!( + result.rows[0][1], + RawSqlValue::Text { + value: "codex-session-1".to_owned(), + bytes: 15, + truncated: false, + } + ); + assert_eq!( + result.rows[0][2], + RawSqlValue::Text { + value: session_id.to_owned(), + bytes: session_id.len(), + truncated: false, + } + ); + assert_eq!( + result.rows[0][3], + RawSqlValue::Text { + value: record_id.to_owned(), + bytes: record_id.len(), + truncated: false, + } + ); + + let detached = store + .raw_sql_query( + "SELECT provider, provider_session_id, ctx_session_id, history_record_id FROM ctx_files_touched WHERE path = 'detached.rs'", + RawSqlOptions::default(), + ) + .unwrap(); + assert_eq!(detached.returned_rows, 1); + assert_eq!( + detached.rows[0][0], + RawSqlValue::Text { + value: "opencode".to_owned(), + bytes: 8, + truncated: false, + } + ); + assert_eq!( + detached.rows[0][1], + RawSqlValue::Text { + value: "opencode-session-1".to_owned(), + bytes: 18, + truncated: false, + } + ); + assert_eq!(detached.rows[0][2], RawSqlValue::Null); + assert_eq!(detached.rows[0][3], RawSqlValue::Null); + } + #[test] fn raw_sql_query_rejects_writes_parameters_and_multiple_statements() { let temp = tempdir(); From bf0c0319b5bb709f16dcbe4f4e6b47cc6e37b04b Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:01:54 -0500 Subject: [PATCH 04/72] docs: add sql reference topic Adds a dedicated embedded SQL guide with stable view schemas, examples, limits, and output formats. Exact docs topic matches now rank first so 1. storage - Storage And Privacy score 33 | Local storage layout, command read/write behavior, privacy, and upgrades. inspect: ctx docs show storage 2. cli-reference - CLI Reference score 24 | Command and option reference for the installed ctx CLI. inspect: ctx docs show cli-reference 3. json-contracts - JSON Contracts score 15 | Machine-readable JSON output contracts for scripts and integrations. inspect: ctx docs show json-contracts 4. search - Search score 3 | Search behavior, filters, result metadata, and agent-readable output. inspect: ctx docs show search 5. agent-usage - Agent Usage score 2 | How agents should search, inspect, cite, and report local history. inspect: ctx docs show agent-usage 6. getting-started - Getting Started score 2 | Install ctx, set up local storage, import history, and run first searches. inspect: ctx docs show getting-started 7. limitations - Limitations score 1 | Provider, import, search, retrieval, and operations limits. inspect: ctx docs show limitations 8. provider-support - Provider Support score 1 | Current provider support matrix and promotion evidence requirements. inspect: ctx docs show provider-support 9. providers - Providers score 1 | Supported local provider imports and fidelity rules. inspect: ctx docs show providers points agents directly at the advanced-query reference. --- crates/ctx-cli/src/docs.rs | 37 ++++++--- crates/ctx-cli/tests/cli.rs | 3 + docs/agent-usage.md | 3 +- docs/sql.md | 155 ++++++++++++++++++++++++++++++++++++ docs/storage.md | 1 + 5 files changed, 189 insertions(+), 10 deletions(-) create mode 100644 docs/sql.md diff --git a/crates/ctx-cli/src/docs.rs b/crates/ctx-cli/src/docs.rs index f213a70a2..34c7cf06a 100644 --- a/crates/ctx-cli/src/docs.rs +++ b/crates/ctx-cli/src/docs.rs @@ -124,6 +124,15 @@ const TOPICS: &[DocTopic] = &[ source_path: "docs/search.md", body: include_str!("../../../docs/search.md"), }, + DocTopic { + id: "sql", + title: "SQL", + audience: "agent", + summary: "Read-only SQL usage, stable view schemas, limits, and examples.", + tags: &["sql", "sqlite", "views", "advanced"], + source_path: "docs/sql.md", + body: include_str!("../../../docs/sql.md"), + }, DocTopic { id: "agent-usage", title: "Agent Usage", @@ -239,15 +248,7 @@ fn search_docs(query: &str, limit: usize, json_output: bool) -> Result<()> { let mut results: Vec<(usize, &DocTopic)> = TOPICS .iter() .filter_map(|topic| { - let haystack = format!( - "{} {} {} {}", - topic.id, topic.title, topic.summary, topic.body - ) - .to_ascii_lowercase(); - let score = terms - .iter() - .map(|term| haystack.matches(term).count()) - .sum::(); + let score = score_doc_topic(topic, &terms); (score > 0).then_some((score, topic)) }) .collect(); @@ -282,6 +283,24 @@ fn search_docs(query: &str, limit: usize, json_output: bool) -> Result<()> { Ok(()) } +fn score_doc_topic(topic: &DocTopic, terms: &[String]) -> usize { + let haystack = format!( + "{} {} {} {}", + topic.id, topic.title, topic.summary, topic.body + ) + .to_ascii_lowercase(); + let title = topic.title.to_ascii_lowercase(); + terms + .iter() + .map(|term| { + let exact_topic_match = topic.id == term + || title == *term + || topic.tags.iter().any(|tag| tag.eq_ignore_ascii_case(term)); + haystack.matches(term).count() + usize::from(exact_topic_match) * 1_000 + }) + .sum() +} + fn show_doc(args: DocsShowArgs) -> Result<()> { let topic = TOPICS .iter() diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index b614f4257..d7cb32a3b 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1189,6 +1189,9 @@ fn docs_commands_expose_embedded_docs_and_man_pages() { assert_eq!(search["query"], "upgrade"); assert!(!search["results"].as_array().unwrap().is_empty()); + let sql_search = json_output(ctx(&temp).args(["docs", "search", "sql", "--json"])); + assert_eq!(sql_search["results"][0]["id"], "sql"); + let show = json_output(ctx(&temp).args(["docs", "show", "cli-reference", "--format", "json"])); assert_eq!(show["schema_version"], 1); assert_eq!(show["id"], "cli-reference"); diff --git a/docs/agent-usage.md b/docs/agent-usage.md index 459c3d70d..e149b20c0 100644 --- a/docs/agent-usage.md +++ b/docs/agent-usage.md @@ -24,7 +24,8 @@ query over the existing index. Use `ctx sql` only when normal search does not express the question, such as exact counts, joins, audits, or scripting over stable `ctx_*` views. It is -read-only and does not refresh or import provider history. +read-only and does not refresh or import provider history. See +`ctx docs show sql` for stable view schemas and examples. When ctx runs inside Codex and `CODEX_THREAD_ID` is available, search excludes the active Codex session tree by default to avoid returning the current prompt diff --git a/docs/sql.md b/docs/sql.md new file mode 100644 index 000000000..9485ec45b --- /dev/null +++ b/docs/sql.md @@ -0,0 +1,155 @@ +# SQL + +`ctx sql` runs read-only SQL against the existing local ctx SQLite index. Use it +when normal `ctx search` does not express the question: counts, audits, joins, +file/session metadata lookups, or scripts that need structured output. + +`ctx sql` does not refresh provider history, import files, initialize storage, or +migrate schemas. Run `ctx status`, `ctx setup`, or `ctx import` first if the +local store needs to be created or migrated. + +## Examples + +```bash +ctx sql "SELECT provider, COUNT(*) AS sessions FROM ctx_sessions GROUP BY provider" +ctx sql "SELECT event_type, COUNT(*) AS events FROM ctx_events GROUP BY event_type ORDER BY events DESC" +ctx sql "SELECT path, provider, provider_session_id FROM ctx_files_touched WHERE path LIKE '%AGENTS.md%' LIMIT 20" +ctx sql --format json "SELECT ctx_session_id, cwd FROM ctx_sessions ORDER BY started_at_ms DESC LIMIT 5" +ctx sql --format csv --file query.sql +ctx sql - --format raw < query.sql +``` + +Use normal `ctx search` for transcript text search. Avoid broad scans over +`payload_json`; event payloads can be large and normal search is optimized for +finding text. + +## Stable Views + +Prefer stable `ctx_*` views. Internal tables remain queryable locally, but they +are implementation details and can change between versions. + +`ctx_sessions`: + +| Column | Meaning | +| --- | --- | +| `ctx_session_id` | ctx-owned session ID for `ctx show session`. | +| `history_record_id` | ctx history record backing the session, when known. | +| `parent_ctx_session_id` | Parent ctx session ID for subagent/session trees. | +| `root_ctx_session_id` | Root ctx session ID for session trees. | +| `provider` | Provider name such as `codex`, `claude`, or `opencode`. | +| `provider_session_id` | Provider-owned session ID. | +| `external_agent_id` | Provider-owned agent identifier, when present. | +| `agent_type` | `primary`, `subagent`, `reviewer`, `implementer`, or related type. | +| `role_hint` | Provider/importer role hint. | +| `is_primary` | `1` for primary-agent sessions, `0` otherwise. | +| `status` | Imported/session status. | +| `fidelity` | Import fidelity. | +| `started_at_ms`, `ended_at_ms` | Unix epoch milliseconds. | +| `cwd` | Captured working directory, when known. | +| `source_path` | Raw provider source path, when known. | + +`ctx_events`: + +| Column | Meaning | +| --- | --- | +| `ctx_event_id` | ctx-owned event ID for `ctx show event`. | +| `ctx_session_id` | ctx session ID, when known. | +| `history_record_id` | ctx history record backing the event, when known. | +| `provider`, `provider_session_id` | Provider context from the session. | +| `event_seq` | Provider/session event sequence. | +| `event_type` | `message`, `tool_call`, `tool_output`, `command_started`, `command_output`, `command_finished`, `file_touched`, `vcs_change`, `artifact`, `summary`, or `notice`. | +| `role` | Event role such as `user`, `assistant`, or `tool`, when known. | +| `occurred_at_ms` | Unix epoch milliseconds. | +| `payload_json` | Local private event payload. | +| `redaction_state` | Payload redaction/preview state. | +| `fidelity` | Import fidelity. | +| `cwd`, `source_path` | Captured source context, when known. | + +`ctx_files_touched`: + +| Column | Meaning | +| --- | --- | +| `ctx_file_touch_id` | ctx-owned touched-file row ID. | +| `path`, `old_path` | Touched path and prior path for renames. | +| `change_kind` | `read`, `created`, `modified`, `deleted`, `renamed`, or `unknown`. | +| `line_count_delta` | Imported line delta, when known. | +| `confidence` | `explicit`, `high`, `medium`, `low`, or `unknown`. | +| `ctx_event_id` | Associated event ID, when importer knows it. | +| `ctx_session_id` | Associated session ID, resolved from event, run, or capture source. | +| `history_record_id` | Associated history record, resolved from event, run, row, or capture source. | +| `provider`, `provider_session_id` | Provider context, when resolvable. | +| `created_at_ms`, `updated_at_ms` | Unix epoch milliseconds. | + +`ctx_sources`: + +| Column | Meaning | +| --- | --- | +| `provider`, `source_format` | Provider and importer/source format. | +| `source_root`, `source_path` | Discovered provider source location. | +| `provider_session_id`, `parent_provider_session_id` | Provider session identifiers. | +| `agent_type`, `role_hint` | Imported session role metadata. | +| `cwd` | Captured working directory, when known. | +| `session_started_at_ms` | Provider session start time in Unix epoch milliseconds. | +| `file_size_bytes`, `file_modified_at_ms`, `cataloged_at_ms` | Catalog metadata. | +| `indexed_status`, `indexed_at_ms`, `indexed_error`, `indexed_event_count` | Import/index status. | + +## File Path Queries + +Touched-file rows are metadata about files mentioned by imported provider +events. They are not a live filesystem index. A row may be associated directly +with an event, with a command/run, with a history record, or only with a capture +source. The stable view resolves provider and session context when possible. + +```sql +SELECT path, provider, provider_session_id, ctx_session_id +FROM ctx_files_touched +WHERE path = 'crates/ctx-cli/src/main.rs' +ORDER BY updated_at_ms DESC +LIMIT 20; +``` + +Combine file metadata with normal search when you need transcript relevance: + +```bash +ctx search "release blocker" --file crates/ctx-cli/src/main.rs +``` + +## Input And Output + +Pass SQL as an argument, from stdin with `-`, or with `--file`: + +```bash +ctx sql "SELECT COUNT(*) FROM ctx_events" +ctx sql - < query.sql +ctx sql --file query.sql +``` + +Formats: + +- `--format table`, the default human-readable table; +- `--format json`, structured output with columns, rows, limits, timing, and truncation; +- `--json`, alias for `--format json`; +- `--format csv`, script-friendly CSV; +- `--format raw`, one-column raw lines for piping. + +`--format raw` requires exactly one selected column. + +## Limits + +`ctx sql` is intentionally bounded: + +- read-only statements only; +- one statement per invocation; +- no query parameters; +- default row, column, SQL byte, and value byte caps; +- timeout for long-running queries; +- JSON output marked `share_safe: false`. + +Increase limits only when scripting needs them: + +```bash +ctx sql "SELECT * FROM ctx_events LIMIT 500" --max-rows 500 --timeout 30s +``` + +Keep SQL output local unless you have reviewed it. Payloads, paths, prompts, +tool output, and repository names can contain private data. diff --git a/docs/storage.md b/docs/storage.md index cd8272dc6..00b99052a 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -173,6 +173,7 @@ Stable read-only views are the preferred compatibility surface: - `ctx_files_touched`; - `ctx_sources`. +Run `ctx docs show sql` for view schemas, examples, limits, and output formats. Internal tables remain local and queryable, but they are implementation details and can change across versions. SQL output is private local history by default. From 9b7a68b5d43a53e4a0957f4303f7402b93b8fb3e Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:02:50 -0500 Subject: [PATCH 05/72] skills: teach agents advanced ctx workflows Adds status/source checks, read-only search examples, verbose output guidance, the new primary-vs-subagent search default, and a SQL escape hatch so packaged agents can self-serve advanced history queries. --- .../skills/ctx-agent-history-search/SKILL.md | 56 +++++++++++++++++-- skills/ctx-agent-history-search/SKILL.md | 56 +++++++++++++++++-- 2 files changed, 102 insertions(+), 10 deletions(-) diff --git a/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md b/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md index 1bec9819d..f1e99b091 100644 --- a/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md +++ b/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md @@ -32,16 +32,28 @@ Use this skill in two modes: ## Workflow -1. Search with normal language first. Add terms or filters when useful: +1. Confirm ctx is ready when starting from a cold context: + + ```bash + ctx status + ctx sources + ``` + + Use `ctx status --json` or `ctx sources --json` only when a script needs + exact fields. + +2. Search with normal language first. Add terms or filters when useful: ```bash ctx search "" + ctx search "" --refresh off ctx search "" --provider codex ctx search "" --workspace ctx search "" --file ctx search "" --since 30d ctx search "" --term "" --term "" ctx search "" --session + ctx search "" --verbose ``` Use default text output for agent reading. Do not add `--json` for @@ -55,6 +67,14 @@ Use this skill in two modes: when a session looks relevant and you need dense event-level matches from that session. + Default search returns primary-agent sessions so human intent and decisions + stay prominent. Use `--include-subagents` when implementation details, code + review notes, test output, or failure traces from subagent sessions are + likely to matter. + + Use `--verbose` when you need full ctx IDs, provider IDs, citations, and + copyable follow-up commands without switching to JSON. + You can write a session transcript to a temporary file, check the file size, and then read the relevant parts: @@ -68,27 +88,51 @@ Use this skill in two modes: dominate historical retrieval. Use `--include-current-session` only when the active session tree is the target. -2. Inspect relevant results before relying on them: +3. Inspect relevant results before relying on them: ```bash ctx show event --window 5 ctx show session ``` -3. Locate original provider material when source identity or resume hints matter: +4. Locate original provider material when source identity or resume hints matter: ```bash ctx locate event ctx locate session ``` -4. Write a transcript of relevant sessions when you, the human, or another +5. Write a transcript of relevant sessions when you, the human, or another agent needs a file: ```bash ctx show session --format markdown --out ``` +## When Search Is Not Enough + +Use `ctx sql` only when normal search cannot express the question, such as +counts, joins, audits, or scripts over stable local views. Do not use SQL for +broad transcript text search; `ctx search` is built for that. + +Start with the bundled SQL docs: + +```bash +ctx docs show sql +ctx docs search "stable views" +``` + +Common SQL examples: + +```bash +ctx sql "SELECT provider, COUNT(*) AS sessions FROM ctx_sessions GROUP BY provider" +ctx sql "SELECT event_type, COUNT(*) AS events FROM ctx_events GROUP BY event_type ORDER BY events DESC" +ctx sql "SELECT path, provider, provider_session_id FROM ctx_files_touched WHERE path LIKE '%AGENTS.md%' LIMIT 20" +``` + +`ctx sql` is read-only and queries the existing index. It does not refresh, +import, initialize, or migrate ctx storage. + ## History Research Reports When asked to research a historical topic, stay read-only unless the user also @@ -103,7 +147,9 @@ material. with default `ctx search`, then broaden with `--term` or narrow with `--workspace`, `--provider`, `--file`, `--since`, or `--session `. - Add `--refresh off` when the report must not update the local ctx index. + Use `--include-subagents` when reviews, implementation attempts, test output, + or failure traces are likely to live in delegated sessions. Add + `--refresh off` when the report must not update the local ctx index. 3. Inspect focused sources before drawing conclusions. Prefer `ctx show event` for a hit plus nearby turns, and `ctx show session` when the whole session arc matters: diff --git a/skills/ctx-agent-history-search/SKILL.md b/skills/ctx-agent-history-search/SKILL.md index 1bec9819d..f1e99b091 100644 --- a/skills/ctx-agent-history-search/SKILL.md +++ b/skills/ctx-agent-history-search/SKILL.md @@ -32,16 +32,28 @@ Use this skill in two modes: ## Workflow -1. Search with normal language first. Add terms or filters when useful: +1. Confirm ctx is ready when starting from a cold context: + + ```bash + ctx status + ctx sources + ``` + + Use `ctx status --json` or `ctx sources --json` only when a script needs + exact fields. + +2. Search with normal language first. Add terms or filters when useful: ```bash ctx search "" + ctx search "" --refresh off ctx search "" --provider codex ctx search "" --workspace ctx search "" --file ctx search "" --since 30d ctx search "" --term "" --term "" ctx search "" --session + ctx search "" --verbose ``` Use default text output for agent reading. Do not add `--json` for @@ -55,6 +67,14 @@ Use this skill in two modes: when a session looks relevant and you need dense event-level matches from that session. + Default search returns primary-agent sessions so human intent and decisions + stay prominent. Use `--include-subagents` when implementation details, code + review notes, test output, or failure traces from subagent sessions are + likely to matter. + + Use `--verbose` when you need full ctx IDs, provider IDs, citations, and + copyable follow-up commands without switching to JSON. + You can write a session transcript to a temporary file, check the file size, and then read the relevant parts: @@ -68,27 +88,51 @@ Use this skill in two modes: dominate historical retrieval. Use `--include-current-session` only when the active session tree is the target. -2. Inspect relevant results before relying on them: +3. Inspect relevant results before relying on them: ```bash ctx show event --window 5 ctx show session ``` -3. Locate original provider material when source identity or resume hints matter: +4. Locate original provider material when source identity or resume hints matter: ```bash ctx locate event ctx locate session ``` -4. Write a transcript of relevant sessions when you, the human, or another +5. Write a transcript of relevant sessions when you, the human, or another agent needs a file: ```bash ctx show session --format markdown --out ``` +## When Search Is Not Enough + +Use `ctx sql` only when normal search cannot express the question, such as +counts, joins, audits, or scripts over stable local views. Do not use SQL for +broad transcript text search; `ctx search` is built for that. + +Start with the bundled SQL docs: + +```bash +ctx docs show sql +ctx docs search "stable views" +``` + +Common SQL examples: + +```bash +ctx sql "SELECT provider, COUNT(*) AS sessions FROM ctx_sessions GROUP BY provider" +ctx sql "SELECT event_type, COUNT(*) AS events FROM ctx_events GROUP BY event_type ORDER BY events DESC" +ctx sql "SELECT path, provider, provider_session_id FROM ctx_files_touched WHERE path LIKE '%AGENTS.md%' LIMIT 20" +``` + +`ctx sql` is read-only and queries the existing index. It does not refresh, +import, initialize, or migrate ctx storage. + ## History Research Reports When asked to research a historical topic, stay read-only unless the user also @@ -103,7 +147,9 @@ material. with default `ctx search`, then broaden with `--term` or narrow with `--workspace`, `--provider`, `--file`, `--since`, or `--session `. - Add `--refresh off` when the report must not update the local ctx index. + Use `--include-subagents` when reviews, implementation attempts, test output, + or failure traces are likely to live in delegated sessions. Add + `--refresh off` when the report must not update the local ctx index. 3. Inspect focused sources before drawing conclusions. Prefer `ctx show event` for a hit plus nearby turns, and `ctx show session` when the whole session arc matters: From 480385960c6312bc0e6552f26daf3cdca450d801 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:07:26 -0500 Subject: [PATCH 06/72] cli: clarify search filters and accept id prefixes Search/show/locate now accept unambiguous 8+ character ctx ID prefixes, matching the short IDs displayed in compact output. Help and docs now explain OR-style --term broadening, touched-file --file semantics, workspace matching, and valid event types. --- crates/ctx-cli/src/main.rs | 123 +++++++++++++++++++++++----- crates/ctx-cli/src/mcp.rs | 2 +- crates/ctx-cli/tests/cli.rs | 41 +++++++++- crates/ctx-history-store/src/lib.rs | 16 ++++ docs/cli-reference.md | 23 ++++-- docs/getting-started.md | 5 +- docs/search.md | 26 +++--- 7 files changed, 195 insertions(+), 41 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 08d6172d0..36ef152e6 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -140,7 +140,8 @@ enum ShowTarget { #[derive(Debug, Args)] struct ShowSessionArgs { - id: Option, + #[arg(help = "ctx session id or unambiguous id prefix")] + id: Option, #[arg(long, value_enum)] provider: Option, #[arg(long = "provider-session")] @@ -157,7 +158,8 @@ struct ShowSessionArgs { #[derive(Debug, Args)] struct ShowEventArgs { - id: Uuid, + #[arg(help = "ctx event id or unambiguous id prefix")] + id: String, #[arg(long, default_value_t = 0)] before: usize, #[arg(long, default_value_t = 0)] @@ -186,7 +188,8 @@ enum LocateTarget { #[derive(Debug, Args)] struct LocateSessionArgs { - id: Option, + #[arg(help = "ctx session id or unambiguous id prefix")] + id: Option, #[arg(long, value_enum)] provider: Option, #[arg(long = "provider-session")] @@ -199,7 +202,8 @@ struct LocateSessionArgs { #[derive(Debug, Args)] struct LocateEventArgs { - id: Uuid, + #[arg(help = "ctx event id or unambiguous id prefix")] + id: String, #[arg(long, value_enum, default_value_t = LocateFormat::Text)] format: LocateFormat, #[arg(long)] @@ -212,7 +216,7 @@ struct SearchArgs { query: Option, #[arg( long, - help = "Add another search query or keyword; repeat to broaden and merge results" + help = "Add another search query or keyword; repeat to broaden with OR-style merged results" )] term: Vec, #[arg( @@ -224,7 +228,10 @@ struct SearchArgs { limit: usize, #[arg(long, help = "Search only one provider")] provider: Option, - #[arg(long, help = "Filter by workspace path or name text")] + #[arg( + long, + help = "Filter by stored workspace, cwd, source path, or repo-name text" + )] workspace: Option, #[arg( long, @@ -242,12 +249,21 @@ struct SearchArgs { help = "Include subagent sessions in addition to primary-agent sessions" )] include_subagents: bool, - #[arg(long, help = "Filter by event type, such as message or tool_call")] + #[arg( + long, + help = "Filter by event type: message, tool_call, tool_output, command_started, command_output, command_finished, file_touched, vcs_change, artifact, summary, or notice" + )] event_type: Option, - #[arg(long, help = "Filter by file path text")] + #[arg( + long, + help = "Filter by indexed touched-file path metadata, not the current filesystem" + )] file: Option, - #[arg(long, help = "Search event hits within one ctx session id")] - session: Option, + #[arg( + long, + help = "Search event hits within one ctx session id or unambiguous id prefix" + )] + session: Option, #[arg( long, help = "Return dense event-level results instead of diverse session results" @@ -314,7 +330,7 @@ impl SqlArgs { } pub(crate) struct SearchFilterInput { - session: Option, + session: Option, provider: Option, workspace: Option, since: Option, @@ -2311,7 +2327,7 @@ fn run_show( write_rendered_session(&store, &session, &events, args.mode, format, args.out)?; } ShowTarget::Event(args) => { - let event = store.get_event(args.id)?; + let event = resolve_event(&store, &args.id)?; let events = event_window(&store, &event, args.before, args.after, args.window)?; analytics::insert_count_bucket( analytics_properties, @@ -2356,7 +2372,7 @@ fn run_locate( } } LocateTarget::Event(args) => { - let event = store.get_event(args.id)?; + let event = resolve_event(&store, &args.id)?; let value = locate_event_json(&store, &event); if locate_json_output(args.format, args.json) { print_json(value)?; @@ -2382,14 +2398,12 @@ fn locate_json_output(format: LocateFormat, json: bool) -> bool { fn resolve_session( store: &Store, - id: Option, + id: Option, provider: Option, provider_session: Option<&str>, ) -> Result { if let Some(id) = id { - return store.get_session(id).with_context(|| { - format!("session {id} was not found; use `ctx search` or `ctx search --verbose` to get ctx_session_id") - }); + return resolve_session_by_id_text(store, &id); } let provider = provider.ok_or_else(|| { anyhow!( @@ -2687,6 +2701,67 @@ fn push_session_metadata_markdown( } } +fn resolve_session_by_id_text(store: &Store, value: &str) -> Result { + if let Ok(id) = Uuid::parse_str(value.trim()) { + return store.get_session(id).with_context(|| { + format!("session {id} was not found; use `ctx search --verbose` to get ctx_session_id") + }); + } + let prefix = normalize_uuid_prefix(value, "session")?; + match store.sessions_by_id_prefix(&prefix)?.as_slice() { + [session] => Ok(session.clone()), + [] => Err(anyhow!( + "session id prefix {prefix:?} was not found; use `ctx search --verbose` to get ctx_session_id" + )), + matches => Err(anyhow!( + "session id prefix {prefix:?} is ambiguous; first matches are {} and {}; use a longer ctx_session_id", + matches[0].id, + matches[1].id + )), + } +} + +fn resolve_session_id(store: &Store, value: &str) -> Result { + Ok(resolve_session_by_id_text(store, value)?.id) +} + +fn resolve_event(store: &Store, value: &str) -> Result { + if let Ok(id) = Uuid::parse_str(value.trim()) { + return store.get_event(id).with_context(|| { + format!( + "event {id} was not found; use `ctx search --events --verbose` to get ctx_event_id" + ) + }); + } + let prefix = normalize_uuid_prefix(value, "event")?; + match store.events_by_id_prefix(&prefix)?.as_slice() { + [event] => Ok(event.clone()), + [] => Err(anyhow!( + "event id prefix {prefix:?} was not found; use `ctx search --events --verbose` to get ctx_event_id" + )), + matches => Err(anyhow!( + "event id prefix {prefix:?} is ambiguous; first matches are {} and {}; use a longer ctx_event_id", + matches[0].id, + matches[1].id + )), + } +} + +fn normalize_uuid_prefix(value: &str, kind: &str) -> Result { + let prefix = value.trim(); + if prefix.len() < 8 { + return Err(anyhow!( + "{kind} id prefix must be at least 8 hex characters, or pass a full ctx UUID" + )); + } + if prefix.contains('-') || !prefix.chars().all(|ch| ch.is_ascii_hexdigit()) { + return Err(anyhow!( + "{kind} id must be a full ctx UUID or an unambiguous hex prefix from `ctx search --verbose`" + )); + } + Ok(prefix.to_ascii_lowercase()) +} + fn push_event_text_block(out: &mut String, event: &Event) { let role = event.role.map(|role| role.as_str()).unwrap_or("-"); out.push_str(&format!( @@ -4837,13 +4912,23 @@ fn search_filters( input: SearchFilterInput, store: Option<&Store>, ) -> Result { - let exclude_provider_session = if input.include_current_session || input.session.is_some() { + let session = input + .session + .as_deref() + .map(|value| { + let store = store.ok_or_else(|| { + anyhow!("session id prefix resolution requires an open ctx store") + })?; + resolve_session_id(store, value) + }) + .transpose()?; + let exclude_provider_session = if input.include_current_session || session.is_some() { None } else { current_codex_provider_session_filter(store) }; Ok(ctx_history_search::SearchFilters { - session: input.session, + session, provider: input.provider.map(ProviderArg::capture_provider), repo: input.workspace, since: input.since.as_deref().map(parse_since_filter).transpose()?, diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index b77c71ed0..2bd15a76c 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -320,7 +320,7 @@ fn tool_search(arguments: &Value, data_root: &Path) -> Result { return Err(anyhow!("limit must be between 1 and {MAX_SEARCH_LIMIT}")); } let provider = optional_provider(arguments, "provider")?; - let session = optional_uuid(arguments, "session")?; + let session = optional_string(arguments, "session")?; let workspace = optional_string(arguments, "workspace")?; let since = optional_string(arguments, "since")?; let primary_only = optional_bool(arguments, "primary_only")?.unwrap_or(false); diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index d7cb32a3b..dc4668436 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1054,13 +1054,15 @@ fn public_subcommand_help_is_golden_enough_for_session_retrieval() { "Add another search query or keyword", "--provider ", "--workspace ", - "Filter by workspace path or name text", + "Filter by stored workspace", "--since ", "Filter to recent history, as RFC3339 or a day window like 30d", "--include-subagents", "Include subagent sessions", "--event-type ", + "Filter by event type:", "--file ", + "indexed touched-file path metadata", "--session ", "--events", "--limit ", @@ -1838,6 +1840,28 @@ fn fresh_home_search_mvp_flow() { .iter() .all(|result| result["ctx_session_id"] == ctx_session_id)); + let session_prefix = &ctx_session_id[..8]; + let prefixed_session_events = json_output(ctx(&temp).args([ + "search", + "onboarding", + "--provider", + "codex", + "--session", + session_prefix, + "--json", + ])); + assert_event_search_provider_oracle( + &prefixed_session_events, + "codex", + "onboarding", + 1, + "message", + ); + assert_eq!( + prefixed_session_events["filters"]["session"], + ctx_session_id + ); + let human_search = ctx(&temp) .args(["search", "onboarding"]) .assert() @@ -1913,6 +1937,17 @@ fn fresh_home_search_mvp_flow() { && event["ctx_session_id"].is_string() && event["preview"].is_string())); + let show_event_prefix = json_output(ctx(&temp).args([ + "show", + "event", + &ctx_event_id[..8], + "--window", + "1", + "--format", + "json", + ])); + assert_eq!(show_event_prefix["event"]["ctx_event_id"], ctx_event_id); + let show_session = json_output(ctx(&temp).args(["show", "session", &ctx_session_id, "--format", "json"])); assert_eq!(show_session["schema_version"], 1); @@ -1921,6 +1956,10 @@ fn fresh_home_search_mvp_flow() { assert_eq!(show_session["session"]["item_id"], ctx_session_id); assert_eq!(show_session["mode"], "lite"); + let show_session_prefix = + json_output(ctx(&temp).args(["show", "session", &ctx_session_id[..8], "--format", "json"])); + assert_eq!(show_session_prefix["session"]["item_id"], ctx_session_id); + let show_session_full = json_output(ctx(&temp).args([ "show", "session", diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 3605e9110..098061f22 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -2228,6 +2228,14 @@ impl Store { .ok_or(StoreError::NotFound(id)) } + pub fn sessions_by_id_prefix(&self, prefix: &str) -> Result> { + let mut stmt = self + .conn + .prepare(session_select_sql("WHERE id LIKE ?1 ORDER BY id LIMIT 2").as_str())?; + let rows = stmt.query_map(params![format!("{prefix}%")], session_from_row)?; + collect_rows(rows) + } + pub fn session_by_external_session( &self, provider: CaptureProvider, @@ -2623,6 +2631,14 @@ impl Store { .ok_or(StoreError::NotFound(id)) } + pub fn events_by_id_prefix(&self, prefix: &str) -> Result> { + let mut stmt = self + .conn + .prepare(event_select_sql("WHERE id LIKE ?1 ORDER BY id LIMIT 2").as_str())?; + let rows = stmt.query_map(params![format!("{prefix}%")], event_from_row)?; + collect_rows(rows) + } + pub fn events_for_session(&self, session_id: Uuid) -> Result> { let mut stmt = self.conn.prepare( event_select_sql("WHERE session_id = ?1 ORDER BY seq, occurred_at_ms").as_str(), diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 8c4ed8dfc..ca6342d49 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -174,9 +174,12 @@ returns the strongest matching span from each session, plus `more_matches_in_session` and `session_importance` when more indexed events from that session also matched. Use `--session ` after a default search has identified a session to inspect; scoped session search returns dense -event hits. Use `--events` without `--session` for dense event-level results -across sessions. Repeat `--term ` when you want to broaden a -search across several related words or phrases and merge the ranked results. +event hits. Session/event commands accept full ctx IDs or unambiguous ctx ID +prefixes of at least eight hex characters. Use `--events` without `--session` +for dense event-level results across sessions. Repeat +`--term ` when you want to broaden a search across several +related words or phrases and merge the ranked results; `--term` is OR-style +broadening, not a must-include filter. Default search excludes subagent sessions so primary human-agent intent and decisions stay prominent. Use `--include-subagents` when implementation details, code review notes, test output, or failure analysis from subagent sessions @@ -199,12 +202,16 @@ optimized for agent reading; use `--verbose` for expanded text diagnostics. Filters: - `--provider codex|pi|claude|opencode|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; -- `--workspace `; +- `--workspace `, substring match over stored workspace, cwd, + source path, or repository-name text; - `--since d`, for example `2026-06-01T00:00:00Z` or `30d`; -- `--event-type `; -- `--file `; -- `--session `, for dense event results within one session; -- `--term `, repeatable broadening terms merged with the main query; +- `--event-type `, one of `message`, `tool_call`, `tool_output`, + `command_started`, `command_output`, `command_finished`, `file_touched`, + `vcs_change`, `artifact`, `summary`, or `notice`; +- `--file `, indexed touched-file path metadata, not the current + filesystem; +- `--session `, for dense event results within one session; +- `--term `, repeatable broadening terms merged with OR-style semantics; - `--events`, for dense event-level results instead of the default session-diverse results; - `--include-subagents`; - `--limit `, capped at `200`; diff --git a/docs/getting-started.md b/docs/getting-started.md index ed83a5749..5bfd73724 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -91,8 +91,9 @@ ctx show session Use `ctx_event_id` with `ctx show event` when you need a hit plus surrounding events. Use `ctx_session_id` with `ctx show session` when you need the -transcript. Search also accepts filters such as `--provider`, `--workspace`, -`--since`, `--event-type`, `--file`, `--include-subagents`, +transcript. Commands accept full ctx IDs or unambiguous ID prefixes of at least +eight hex characters. Search also accepts filters such as `--provider`, +`--workspace`, `--since`, `--event-type`, `--file`, `--include-subagents`, `--include-current-session`, `--term`, `--limit`, and `--refresh auto|off|strict`. `--limit` is capped at `200`. diff --git a/docs/search.md b/docs/search.md index dd116a72c..ee30b496b 100644 --- a/docs/search.md +++ b/docs/search.md @@ -43,23 +43,29 @@ A result can include: - `suggested_next_commands`, copyable commands for `ctx show`, `ctx locate`, and scoped follow-up searches. -Search result IDs are ctx-owned. Provider-owned IDs are exposed as metadata so -humans can recognize the original provider session, but they are not positional -lookup IDs. Provider-owned lookup must be explicit, for example -`--provider codex --provider-session ` on commands that -support it. +Search result IDs are ctx-owned. Commands accept full ctx IDs or unambiguous +ctx ID prefixes of at least eight hex characters. Provider-owned IDs are +exposed as metadata so humans can recognize the original provider session, but +they are not positional lookup IDs. Provider-owned lookup must be explicit, for +example `--provider codex --provider-session ` on commands +that support it. ## Filters Search filters narrow both human output and JSON: - `--provider codex|pi|claude|opencode|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; -- `--workspace `; +- `--workspace `, substring match over stored workspace, cwd, + source path, or repository-name text; - `--since d`; -- `--event-type `; -- `--file `; -- `--session `; -- `--term `, repeatable broadening terms merged with the main query; +- `--event-type `, one of `message`, `tool_call`, `tool_output`, + `command_started`, `command_output`, `command_finished`, `file_touched`, + `vcs_change`, `artifact`, `summary`, or `notice`; +- `--file `, indexed touched-file path metadata, not the current + filesystem; +- `--session `; +- `--term `, repeatable broadening terms merged with OR-style + semantics, not required terms; - `--events`; - `--include-subagents`; - `--limit `; From 0d1838dbc257636824f18a842e478d42276871d1 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:08:49 -0500 Subject: [PATCH 07/72] docs: suppress weak docs search matches Docs search now favors exact topic/tag matches, ignores short substring noise, and returns next-step suggestions when no strong topic matches. This prevents agents from treating low-confidence docs hits as authoritative. --- crates/ctx-cli/src/docs.rs | 68 ++++++++++++++++++++++++++++++++----- crates/ctx-cli/tests/cli.rs | 8 +++++ 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/crates/ctx-cli/src/docs.rs b/crates/ctx-cli/src/docs.rs index 34c7cf06a..09fc02270 100644 --- a/crates/ctx-cli/src/docs.rs +++ b/crates/ctx-cli/src/docs.rs @@ -240,16 +240,12 @@ fn list_docs(json_output: bool) -> Result<()> { } fn search_docs(query: &str, limit: usize, json_output: bool) -> Result<()> { - let terms: Vec = query - .split_whitespace() - .map(|term| term.to_ascii_lowercase()) - .filter(|term| !term.is_empty()) - .collect(); + let terms = docs_query_terms(query); let mut results: Vec<(usize, &DocTopic)> = TOPICS .iter() .filter_map(|topic| { let score = score_doc_topic(topic, &terms); - (score > 0).then_some((score, topic)) + (score >= docs_min_score(&terms)).then_some((score, topic)) }) .collect(); results.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.id.cmp(right.1.id))); @@ -268,11 +264,15 @@ fn search_docs(query: &str, limit: usize, json_output: bool) -> Result<()> { serde_json::to_string_pretty(&json!({ "schema_version": 1, "query": query, - "results": rows + "results": rows, + "suggested_next_commands": docs_search_suggestions(query, rows.is_empty()) }))? ); } else if results.is_empty() { println!("no docs matched"); + for command in docs_search_suggestions(query, true) { + println!("next: {command}"); + } } else { for (index, (score, topic)) in results.iter().enumerate() { println!("{}. {} - {}", index + 1, topic.id, topic.title); @@ -283,6 +283,22 @@ fn search_docs(query: &str, limit: usize, json_output: bool) -> Result<()> { Ok(()) } +fn docs_query_terms(query: &str) -> Vec { + query + .split_whitespace() + .map(|term| term.trim().to_ascii_lowercase()) + .filter(|term| !term.is_empty()) + .collect() +} + +fn docs_min_score(terms: &[String]) -> usize { + if terms.is_empty() { + usize::MAX + } else { + terms.len().max(2) + } +} + fn score_doc_topic(topic: &DocTopic, terms: &[String]) -> usize { let haystack = format!( "{} {} {} {}", @@ -296,11 +312,47 @@ fn score_doc_topic(topic: &DocTopic, terms: &[String]) -> usize { let exact_topic_match = topic.id == term || title == *term || topic.tags.iter().any(|tag| tag.eq_ignore_ascii_case(term)); - haystack.matches(term).count() + usize::from(exact_topic_match) * 1_000 + let text_matches = if term.len() >= 3 { + haystack.matches(term).count() + } else { + 0 + }; + text_matches + usize::from(exact_topic_match) * 1_000 }) .sum() } +fn docs_search_suggestions(query: &str, no_results: bool) -> Vec { + if no_results { + let mut suggestions = vec!["ctx docs list".to_owned()]; + let trimmed = query.trim(); + if !trimmed.is_empty() { + suggestions.push(format!( + "ctx docs search {}", + docs_shell_quote_arg(first_docs_search_term(trimmed)) + )); + } + suggestions + } else { + Vec::new() + } +} + +fn first_docs_search_term(query: &str) -> &str { + query.split_whitespace().next().unwrap_or(query) +} + +fn docs_shell_quote_arg(value: &str) -> String { + if value + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.' | '/' | ':')) + { + value.to_owned() + } else { + format!("'{}'", value.replace('\'', "'\"'\"'")) + } +} + fn show_doc(args: DocsShowArgs) -> Result<()> { let topic = TOPICS .iter() diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index dc4668436..fd620bcad 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1194,6 +1194,14 @@ fn docs_commands_expose_embedded_docs_and_man_pages() { let sql_search = json_output(ctx(&temp).args(["docs", "search", "sql", "--json"])); assert_eq!(sql_search["results"][0]["id"], "sql"); + let weak_search = json_output(ctx(&temp).args(["docs", "search", "a", "--json"])); + assert!(weak_search["results"].as_array().unwrap().is_empty()); + assert!(weak_search["suggested_next_commands"] + .as_array() + .unwrap() + .iter() + .any(|command| command == "ctx docs list")); + let show = json_output(ctx(&temp).args(["docs", "show", "cli-reference", "--format", "json"])); assert_eq!(show["schema_version"], 1); assert_eq!(show["id"], "cli-reference"); From b7c778003c65ba0e03de934a3f025ee6342275f8 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:11:58 -0500 Subject: [PATCH 08/72] doctor: report progress during health checks ctx doctor can now emit opening/checking/done progress to stderr, which makes slow SQLite integrity checks visible without polluting JSON stdout. This keeps health checks scriptable while making interactive runs less opaque. --- crates/ctx-cli/src/main.rs | 28 ++++++++++++++++++++++++++-- crates/ctx-cli/tests/cli.rs | 14 +++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 36ef152e6..ba4abe745 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -89,7 +89,7 @@ enum CommandRoot { #[command(about = "Check or apply signed ctx CLI upgrades")] Upgrade(upgrade::UpgradeArgs), #[command(about = "Check local ctx health")] - Doctor(JsonArgs), + Doctor(DoctorArgs), } #[derive(Debug, Args)] @@ -108,6 +108,14 @@ struct JsonArgs { json: bool, } +#[derive(Debug, Args, Clone)] +struct DoctorArgs { + #[arg(long)] + json: bool, + #[arg(long, value_enum, default_value_t = ProgressArg::Auto)] + progress: ProgressArg, +} + #[derive(Debug, Args)] struct ImportArgs { #[arg(long, value_enum)] @@ -4010,11 +4018,17 @@ fn refresh_sources_for_search( } fn run_doctor( - args: JsonArgs, + args: DoctorArgs, data_root: PathBuf, analytics_properties: &mut AnalyticsProperties, ) -> Result<()> { + let progress = ProgressReporter::new(args.progress, args.json, "doctor", 0); + progress.message("opening", "opening ctx store"); let store = Store::open(database_path(data_root.clone()))?; + progress.message( + "checking", + "running sqlite integrity and foreign key checks", + ); let mut findings = store.validate()?; if !data_root.exists() { findings.push(format!("data root does not exist: {}", data_root.display())); @@ -4024,10 +4038,20 @@ fn run_doctor( "finding_count_bucket", findings.len() as u64, ); + progress.done( + "done", + if findings.is_empty() { + "ctx doctor passed" + } else { + "ctx doctor found issues" + }, + 0, + ); if args.json { print_json(json!({ "schema_version": 1, "ok": findings.is_empty(), + "progress": progress_mode_name(args.progress), "findings": findings, }))?; } else if findings.is_empty() { diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index fd620bcad..3a706c5e8 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1077,7 +1077,7 @@ fn public_subcommand_help_is_golden_enough_for_session_retrieval() { "Print expanded text details", ], ), - ("doctor", vec!["Usage: ctx doctor", "--json"]), + ("doctor", vec!["Usage: ctx doctor", "--json", "--progress"]), ] { let output = ctx(&temp) .args([command, "--help"]) @@ -2043,6 +2043,18 @@ fn fresh_home_search_mvp_flow() { let doctor = json_output(ctx(&temp).args(["doctor", "--json"])); assert_eq!(doctor["schema_version"], 1); assert_eq!(doctor["ok"], true); + assert_eq!(doctor["progress"], "auto"); + + let doctor_progress = ctx(&temp) + .args(["doctor", "--json", "--progress", "json"]) + .assert() + .success() + .get_output() + .stderr + .clone(); + let doctor_progress = String::from_utf8(doctor_progress).unwrap(); + assert!(doctor_progress.contains(r#""operation":"doctor""#)); + assert!(doctor_progress.contains(r#""phase":"checking""#)); } #[test] From ebafca4633eaa2b79bf84bb8d6e728230e02f3cf Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:13:26 -0500 Subject: [PATCH 09/72] docs: suggest recovery commands for missing topics ctx docs show now points agents toward nearby topics, ctx docs list, and ctx docs search when a topic id is wrong. This keeps the built-in docs usable when the agent has no external documentation. --- crates/ctx-cli/src/docs.rs | 51 +++++++++++++++++++++++++++++++++++-- crates/ctx-cli/tests/cli.rs | 6 +++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/crates/ctx-cli/src/docs.rs b/crates/ctx-cli/src/docs.rs index 09fc02270..2f87a1f33 100644 --- a/crates/ctx-cli/src/docs.rs +++ b/crates/ctx-cli/src/docs.rs @@ -357,7 +357,7 @@ fn show_doc(args: DocsShowArgs) -> Result<()> { let topic = TOPICS .iter() .find(|topic| topic.id == args.id) - .ok_or_else(|| anyhow!("unknown ctx docs topic: {}", args.id))?; + .ok_or_else(|| unknown_doc_topic_error(&args.id))?; let body = if args.json || args.format == DocsFormat::Json { serde_json::to_string_pretty(&topic_json_with_body(topic))? } else { @@ -401,7 +401,54 @@ fn man_page(name: &str) -> Result<(String, Command)> { man_pages() .into_iter() .find(|(candidate, _)| candidate == name) - .ok_or_else(|| anyhow!("unknown ctx man page: {name}")) + .ok_or_else(|| unknown_man_page_error(name)) +} + +fn unknown_doc_topic_error(id: &str) -> anyhow::Error { + let mut message = format!("unknown ctx docs topic: {id}"); + let suggestions = suggested_doc_topics(id); + if !suggestions.is_empty() { + message.push_str("\nnearest topics:"); + for topic in suggestions { + message.push_str(&format!(" {topic}")); + } + } + message.push_str("\ntry: ctx docs list"); + message.push_str(&format!( + "\ntry: ctx docs search {}", + docs_shell_quote_arg(first_docs_search_term(id)) + )); + anyhow!(message) +} + +fn suggested_doc_topics(id: &str) -> Vec<&'static str> { + let query = id.to_ascii_lowercase(); + let terms = docs_query_terms(id); + let mut scored: Vec<(usize, &'static str)> = TOPICS + .iter() + .filter_map(|topic| { + let score = score_doc_topic(topic, &terms) + + common_prefix_len(&query, topic.id) + + usize::from(topic.id.contains(&query)) * 20; + (score > 0).then_some((score, topic.id)) + }) + .collect(); + scored.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(right.1))); + scored.truncate(3); + scored.into_iter().map(|(_, id)| id).collect() +} + +fn common_prefix_len(left: &str, right: &str) -> usize { + left.chars() + .zip(right.chars()) + .take_while(|(left, right)| left == right) + .count() +} + +fn unknown_man_page_error(name: &str) -> anyhow::Error { + anyhow!( + "unknown ctx man page: {name}\ntry: ctx docs man --print ctx\ntry: ctx docs man --out ./man" + ) } fn man_pages() -> Vec<(String, Command)> { diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 3a706c5e8..6877374cc 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1207,6 +1207,12 @@ fn docs_commands_expose_embedded_docs_and_man_pages() { assert_eq!(show["id"], "cli-reference"); assert!(show["body"].as_str().unwrap().contains("ctx search")); + let missing_topic = failure_stderr(ctx(&temp).args(["docs", "show", "cli"])); + assert!(missing_topic.contains("unknown ctx docs topic: cli")); + assert!(missing_topic.contains("nearest topics:")); + assert!(missing_topic.contains("ctx docs list")); + assert!(missing_topic.contains("ctx docs search cli")); + let man = ctx(&temp) .args(["docs", "man", "--print", "ctx"]) .assert() From 87c543c68511f82acfe83352c59ea952a4b31822 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:15:05 -0500 Subject: [PATCH 10/72] release: bump ctx to 0.13.0 Update crate versions, Cargo.lock, Bazel module metadata, and public artifact version assertions so the 0.13 release branch builds and verifies binaries as ctx 0.13.0. --- Cargo.lock | 10 +++++----- MODULE.bazel | 2 +- crates/ctx-cli/Cargo.toml | 2 +- crates/ctx-history-capture/Cargo.toml | 2 +- crates/ctx-history-core/Cargo.toml | 2 +- crates/ctx-history-search/Cargo.toml | 2 +- crates/ctx-history-store/Cargo.toml | 2 +- scripts/build-public-cli-artifact.sh | 10 +++++----- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47bfc9bb2..97b76076a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,7 +275,7 @@ dependencies = [ [[package]] name = "ctx" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "assert_cmd", @@ -300,7 +300,7 @@ dependencies = [ [[package]] name = "ctx-history-capture" -version = "0.12.0" +version = "0.13.0" dependencies = [ "chrono", "ctx-history-core", @@ -315,7 +315,7 @@ dependencies = [ [[package]] name = "ctx-history-core" -version = "0.12.0" +version = "0.13.0" dependencies = [ "chrono", "directories", @@ -328,7 +328,7 @@ dependencies = [ [[package]] name = "ctx-history-search" -version = "0.12.0" +version = "0.13.0" dependencies = [ "chrono", "ctx-history-core", @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "ctx-history-store" -version = "0.12.0" +version = "0.13.0" dependencies = [ "chrono", "ctx-history-core", diff --git a/MODULE.bazel b/MODULE.bazel index 3801ee356..19a28dfc3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1 +1 @@ -module(name = "ctx_search", version = "0.12.0") +module(name = "ctx_search", version = "0.13.0") diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index ca2fdf9e9..d91a6463b 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx" -version = "0.12.0" +version = "0.13.0" description = "Local CLI for indexing and searching agent session history" edition.workspace = true autobins = false diff --git a/crates/ctx-history-capture/Cargo.toml b/crates/ctx-history-capture/Cargo.toml index 0d9f767f5..4b5b658b2 100644 --- a/crates/ctx-history-capture/Cargo.toml +++ b/crates/ctx-history-capture/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-capture" -version = "0.12.0" +version = "0.13.0" description = "Internal provider import adapters for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-core/Cargo.toml b/crates/ctx-history-core/Cargo.toml index 5b396fc30..b241edd45 100644 --- a/crates/ctx-history-core/Cargo.toml +++ b/crates/ctx-history-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-core" -version = "0.12.0" +version = "0.13.0" description = "Internal core types for ctx local agent history indexing" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-search/Cargo.toml b/crates/ctx-history-search/Cargo.toml index 59ffdf3ea..782390d87 100644 --- a/crates/ctx-history-search/Cargo.toml +++ b/crates/ctx-history-search/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-search" -version = "0.12.0" +version = "0.13.0" description = "Internal search projection and ranking helpers for ctx" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-store/Cargo.toml b/crates/ctx-history-store/Cargo.toml index 1b884d52e..4833a31c6 100644 --- a/crates/ctx-history-store/Cargo.toml +++ b/crates/ctx-history-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-store" -version = "0.12.0" +version = "0.13.0" description = "Internal SQLite storage layer for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index baae36177..e8bfdd9c3 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -47,8 +47,8 @@ root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${root_dir}" version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; data=json.load(sys.stdin); print(next(pkg["version"] for pkg in data["packages"] if pkg["name"] == "ctx"))')" -if [[ "${version}" != "0.12.0" ]]; then - echo "error: ctx package version must be 0.12.0 for this release, got ${version}" >&2 +if [[ "${version}" != "0.13.0" ]]; then + echo "error: ctx package version must be 0.13.0 for this release, got ${version}" >&2 exit 1 fi @@ -87,12 +87,12 @@ fi case "${platform}" in linux-x64) "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.12.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.13.0" "${staged}.version" >/dev/null ;; macos-arm64) if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.12.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.13.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi @@ -100,7 +100,7 @@ case "${platform}" in macos-x64) if [[ "$(uname -s)" == "Darwin" ]] && /usr/bin/arch -x86_64 /usr/bin/true >/dev/null 2>&1; then /usr/bin/arch -x86_64 "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.12.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.13.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi From e6886030e8a2a5bd72c59257957304e12eac9b7a Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:17:43 -0500 Subject: [PATCH 11/72] docs: expose mcp and upgrade topics Add first-class embedded docs topics for ctx docs, MCP, and managed upgrades so agents can discover these newer command surfaces with ctx docs show/search instead of digging through the full CLI reference. --- crates/ctx-cli/src/docs.rs | 27 ++++++++++++++++++++++ crates/ctx-cli/tests/cli.rs | 22 ++++++++++++++++++ docs/docs.md | 38 ++++++++++++++++++++++++++++++ docs/mcp.md | 38 ++++++++++++++++++++++++++++++ docs/upgrade.md | 46 +++++++++++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+) create mode 100644 docs/docs.md create mode 100644 docs/mcp.md create mode 100644 docs/upgrade.md diff --git a/crates/ctx-cli/src/docs.rs b/crates/ctx-cli/src/docs.rs index 2f87a1f33..b2cb08977 100644 --- a/crates/ctx-cli/src/docs.rs +++ b/crates/ctx-cli/src/docs.rs @@ -115,6 +115,15 @@ const TOPICS: &[DocTopic] = &[ source_path: "docs/cli-reference.md", body: include_str!("../../../docs/cli-reference.md"), }, + DocTopic { + id: "docs", + title: "Docs", + audience: "human-agent", + summary: "Use embedded ctx docs, local documentation search, and generated man pages.", + tags: &["docs", "help", "man"], + source_path: "docs/docs.md", + body: include_str!("../../../docs/docs.md"), + }, DocTopic { id: "search", title: "Search", @@ -133,6 +142,24 @@ const TOPICS: &[DocTopic] = &[ source_path: "docs/sql.md", body: include_str!("../../../docs/sql.md"), }, + DocTopic { + id: "mcp", + title: "MCP", + audience: "agent", + summary: "Read-only MCP server tools, behavior, and privacy expectations.", + tags: &["mcp", "tools", "agents"], + source_path: "docs/mcp.md", + body: include_str!("../../../docs/mcp.md"), + }, + DocTopic { + id: "upgrade", + title: "Upgrade", + audience: "human-agent", + summary: "Managed upgrades, background auto-upgrade behavior, and local state.", + tags: &["upgrade", "auto-upgrade", "install"], + source_path: "docs/upgrade.md", + body: include_str!("../../../docs/upgrade.md"), + }, DocTopic { id: "agent-usage", title: "Agent Usage", diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 6877374cc..137671968 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1185,6 +1185,13 @@ fn docs_commands_expose_embedded_docs_and_man_pages() { .unwrap() .iter() .any(|topic| topic["id"] == "cli-reference")); + for topic_id in ["docs", "mcp", "sql", "upgrade"] { + assert!(list["topics"] + .as_array() + .unwrap() + .iter() + .any(|topic| topic["id"] == topic_id)); + } let search = json_output(ctx(&temp).args(["docs", "search", "upgrade", "--json"])); assert_eq!(search["schema_version"], 1); @@ -1194,6 +1201,12 @@ fn docs_commands_expose_embedded_docs_and_man_pages() { let sql_search = json_output(ctx(&temp).args(["docs", "search", "sql", "--json"])); assert_eq!(sql_search["results"][0]["id"], "sql"); + let mcp_search = json_output(ctx(&temp).args(["docs", "search", "mcp", "--json"])); + assert_eq!(mcp_search["results"][0]["id"], "mcp"); + + let upgrade_search = json_output(ctx(&temp).args(["docs", "search", "upgrade", "--json"])); + assert_eq!(upgrade_search["results"][0]["id"], "upgrade"); + let weak_search = json_output(ctx(&temp).args(["docs", "search", "a", "--json"])); assert!(weak_search["results"].as_array().unwrap().is_empty()); assert!(weak_search["suggested_next_commands"] @@ -1207,6 +1220,15 @@ fn docs_commands_expose_embedded_docs_and_man_pages() { assert_eq!(show["id"], "cli-reference"); assert!(show["body"].as_str().unwrap().contains("ctx search")); + let mcp = json_output(ctx(&temp).args(["docs", "show", "mcp", "--format", "json"])); + assert!(mcp["body"].as_str().unwrap().contains("ctx mcp serve")); + + let upgrade = json_output(ctx(&temp).args(["docs", "show", "upgrade", "--format", "json"])); + assert!(upgrade["body"] + .as_str() + .unwrap() + .contains("ctx upgrade status")); + let missing_topic = failure_stderr(ctx(&temp).args(["docs", "show", "cli"])); assert!(missing_topic.contains("unknown ctx docs topic: cli")); assert!(missing_topic.contains("nearest topics:")); diff --git a/docs/docs.md b/docs/docs.md new file mode 100644 index 000000000..c77d33bbf --- /dev/null +++ b/docs/docs.md @@ -0,0 +1,38 @@ +# Docs + +`ctx docs` exposes curated public ctx documentation embedded in the installed +binary. It is for humans and agents that need local command help without +opening a website or reading repository files. + +```bash +ctx docs +ctx docs list +ctx docs list --json +ctx docs search "file path" +ctx docs search "upgrade" --limit 5 --json +ctx docs show cli-reference +ctx docs show search --format text +ctx docs show json-contracts --format json +ctx docs man --print ctx +ctx docs man --out ~/.local/share/man/man1 +``` + +`ctx docs list`, `ctx docs search`, and `ctx docs show` read embedded text and +do not touch provider history or the local SQLite index. `ctx docs show --out +PATH` writes one embedded topic to that explicit path. + +`ctx docs man --print PAGE` prints one generated man page to stdout. `ctx docs +man --out DIR` writes generated section-1 man pages for `ctx` and its public +subcommands. + +Agents should usually use `ctx docs search` or `ctx docs show` rather than +shelling through `man`, because the docs commands return concise markdown, +text, or JSON that is easier for agents to inspect and cite. + +Useful starting points: + +- `ctx docs show search` for search filters and output behavior; +- `ctx docs show sql` for stable read-only SQL views; +- `ctx docs show mcp` for read-only MCP tools; +- `ctx docs show upgrade` for managed upgrade and auto-upgrade behavior; +- `ctx docs show json-contracts` for structured output contracts. diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 000000000..fb37399cb --- /dev/null +++ b/docs/mcp.md @@ -0,0 +1,38 @@ +# MCP + +`ctx mcp serve` starts a read-only MCP server over newline-delimited stdio +JSON-RPC. It is for agents or MCP hosts that prefer tool discovery over shell +commands. The CLI remains the primary interface. + +```bash +ctx mcp serve +``` + +The server exposes these tools: + +- `status`, local ctx index status; +- `sources`, discovered local agent history sources; +- `search`, search the existing index; +- `sql`, run one read-only SQL statement against the existing index; +- `show_session`, return an indexed session transcript by ctx session ID; +- `show_event`, return an indexed event and optional surrounding window by ctx + event ID. + +MCP search and SQL query the existing index only. They do not refresh provider +history, import files, initialize storage, or write provider data. + +MCP search defaults to primary-agent sessions only, matching `ctx search`. +Pass `include_subagents: true` when implementation details, code review notes, +test output, or failure traces from subagent sessions are relevant. When +`CODEX_THREAD_ID` is set, MCP search also excludes the active Codex session tree +by default; pass `include_current_session: true` when the active session tree is +the target. + +The MCP `sql` tool uses the same read-only stable views and result limits as +`ctx sql --json`. Prefer stable `ctx_*` views for scripts and agent workflows. +Run `ctx docs show sql` for the view schemas and examples. + +Tool results include MCP text content plus `structuredContent` JSON. Treat all +MCP output as private local history: it may include absolute paths, source +metadata, snippets, transcript text, and raw SQL result fields, and the MCP host +may log or forward tool output. diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 000000000..dd23d009b --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,46 @@ +# Upgrade + +`ctx upgrade` checks and applies signed ctx CLI releases for binaries installed +by the official hosted installer. + +```bash +ctx upgrade status +ctx upgrade status --json +ctx upgrade check +ctx upgrade check --json +ctx upgrade --dry-run +ctx upgrade +ctx upgrade disable +ctx upgrade enable +``` + +The installer writes a sidecar marker next to the binary, such as +`~/.local/bin/ctx.install.json`, recording the managed install path, platform, +version, channel, binary SHA-256, metadata URL, and artifact URL. Source builds, +`cargo install`, package-manager installs, copied binaries, and mismatched +sidecars are treated as unmanaged and will not self-upgrade. + +Official installer-managed installs default to background auto-upgrade after +successful normal commands when signed release metadata explicitly allows +auto-upgrade. Background checks never run for `--json` commands, MCP, +`ctx docs`, `ctx sql`, `ctx upgrade`, CI, unmanaged installs, or process-level +opt-outs. They write state and logs under the ctx data root and do not write to +stdout or stderr. + +Use `CTX_UPGRADE_OFF=1` or `CTX_DISABLE_AUTO_UPGRADE=1` for process-level +opt-out, or `ctx upgrade disable` to write `upgrade.auto = "off"` in +`config.toml`. Use `ctx upgrade enable` to restore managed background +auto-upgrade for installer-managed binaries. + +Manual `ctx upgrade` verifies signed release metadata, explicit self-upgrade +policy, artifact SHA-256, the current managed install marker, and the staged +binary's `ctx --version` output before replacing the installed binary. + +On Windows, replacement may be scheduled by a helper that finishes after the +running `ctx.exe` exits; JSON reports `status: "scheduled"` and +`applied: false` until replacement completes. + +Background checks write `upgrade-state.json` and `logs/upgrade.log` under the +ctx data root. `ctx upgrade status` reads that local state. Upgrade metadata +checks do not send provider transcript text, search queries, result snippets, +source paths, repository names, or command output. From 3d1282d6411b949493a0d00d55fd3c43a4b065c7 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:19:29 -0500 Subject: [PATCH 12/72] search: keep legacy primary flag out of JSON The hidden --primary-only compatibility flag still narrows search, but search packets no longer serialize primary_only. Structured output now presents one current model: default primary-agent results plus include_subagents when explicitly broadened. --- crates/ctx-cli/tests/cli.rs | 2 +- crates/ctx-history-search/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 137671968..4e6d5d94c 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -2495,7 +2495,7 @@ fn codex_cli_resume_is_idempotent_rescan_and_filters_subagents() { let primary_only = json_output(ctx(&temp).args(["search", "subagent", "--primary-only", "--json"])); assert_eq!(primary_only["filters"]["include_subagents"], false); - assert_eq!(primary_only["filters"]["primary_only"], true); + assert!(primary_only["filters"]["primary_only"].is_null()); assert!( primary_only["results"].as_array().unwrap().len() <= with_subagents["results"].as_array().unwrap().len() diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index 39e72ba2a..5dee7920f 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -66,7 +66,7 @@ pub struct SearchFilters { pub repo: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub since: Option>, - #[serde(default)] + #[serde(skip_serializing)] pub primary_only: bool, #[serde(default)] pub include_subagents: bool, From f71d114c180c4262d0d77592103ddd4ee05ac1a6 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:20:28 -0500 Subject: [PATCH 13/72] docs: clarify provider names in outputs Document the difference between kebab-case provider CLI values and provider IDs shown in JSON or SQL views. This avoids changing stable stored identifiers while reducing confusion around multiword providers. --- docs/cli-reference.md | 4 ++++ docs/providers.md | 4 ++++ docs/search.md | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index ca6342d49..b29b9a572 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -218,6 +218,10 @@ Filters: - `--refresh auto|off|strict`; - `--include-current-session`. +CLI provider filters use kebab-case names. JSON output and stable SQL views use +provider IDs in ctx output; multiword IDs may be snake_case, such as `copilot_cli` or +`factory_ai_droid`. + `search` reads discovered native provider files for pre-search refresh plus SQLite, and may write newly discovered native provider history into the local index before querying. diff --git a/docs/providers.md b/docs/providers.md index 52b347b45..e5a8ecd5a 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -31,6 +31,10 @@ ctx sources ctx sources --json ``` +CLI provider flags use names such as `copilot-cli` and `factory-ai-droid`. +Structured JSON and stable SQL views use provider IDs in ctx output; multiword IDs may be +snake_case, such as `copilot_cli` or `factory_ai_droid`. + `ctx sources --json` reports each known provider source with `import_support` and `importable` fields. A native source is marked available/importable only when provider-specific transcript files exist. Sources with diff --git a/docs/search.md b/docs/search.md index ee30b496b..956668343 100644 --- a/docs/search.md +++ b/docs/search.md @@ -72,6 +72,10 @@ Search filters narrow both human output and JSON: - `--refresh auto|off|strict`; - `--include-current-session`. +CLI provider filters use the kebab-case names above. JSON output and stable SQL +views use provider IDs in ctx output; multiword provider IDs may be snake_case, such as +`copilot_cli` or `factory_ai_droid`. + `--since` accepts RFC 3339 timestamps such as `2026-06-01T00:00:00Z` or a day window such as `30d`. From 34d641d0b4b72a442b8f9f8393a7f75d9ac2fbdd Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:28:01 -0500 Subject: [PATCH 14/72] search: derive default filter state Use the derived Default implementation for SearchFilters after the serialization cleanup. This removes a hand-written default path and keeps clippy clean for release checks. --- crates/ctx-history-search/src/lib.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index 5dee7920f..d2b4422c3 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -56,7 +56,7 @@ pub enum SearchResultMode { Events, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] pub struct SearchFilters { #[serde(default, skip_serializing_if = "Option::is_none")] pub session: Option, @@ -86,22 +86,6 @@ pub struct ProviderSessionFilter { pub session_id: Option, } -impl Default for SearchFilters { - fn default() -> Self { - Self { - session: None, - provider: None, - repo: None, - since: None, - primary_only: false, - include_subagents: false, - event_type: None, - file: None, - exclude_provider_session: None, - } - } -} - #[derive(Debug, Clone, PartialEq, Serialize)] pub struct SearchPacket { pub schema_version: u32, From f7775be9793058cf8a852490569e5e54f0df8fd0 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:33:08 -0500 Subject: [PATCH 15/72] search: filter subagent sections by default Record-level fallback search now applies the same session-scope rules to individual sections before matching, snippets, and citations are produced. This prevents mixed primary/subagent records from returning child-session hits unless --include-subagents is set, while explicit --session still works for known subagent IDs. --- crates/ctx-cli/tests/cli.rs | 38 ++++++ crates/ctx-history-search/src/lib.rs | 192 +++++++++++++++++++++------ 2 files changed, 189 insertions(+), 41 deletions(-) diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 4e6d5d94c..b56bd971f 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -2486,11 +2486,49 @@ fn codex_cli_resume_is_idempotent_rescan_and_filters_subagents() { let primary_default = json_output(ctx(&temp).args(["search", "subagent", "--json"])); assert_eq!(primary_default["filters"]["include_subagents"], false); + let primary_default_text = serde_json::to_string(&primary_default).unwrap(); + assert!( + !primary_default_text.contains("codex-session-child"), + "{primary_default_text}" + ); + + let default_events = json_output(ctx(&temp).args(["search", "subagent", "--events", "--json"])); + assert_eq!(default_events["filters"]["include_subagents"], false); + let default_events_text = serde_json::to_string(&default_events).unwrap(); + assert!( + !default_events_text.contains("codex-session-child"), + "{default_events_text}" + ); let with_subagents = json_output(ctx(&temp).args(["search", "subagent", "--include-subagents", "--json"])); assert!(!with_subagents["results"].as_array().unwrap().is_empty()); assert_eq!(with_subagents["filters"]["include_subagents"], true); + assert!(serde_json::to_string(&with_subagents) + .unwrap() + .contains("codex-session-child")); + + let child_session_lookup = json_output(ctx(&temp).args([ + "sql", + "SELECT ctx_session_id FROM ctx_sessions WHERE provider_session_id = 'codex-session-child'", + "--format", + "json", + ])); + let child_session_id = child_session_lookup["rows"][0][0].as_str().unwrap(); + let explicit_child_session = json_output(ctx(&temp).args([ + "search", + "subagent", + "--session", + child_session_id, + "--json", + ])); + assert_eq!( + explicit_child_session["filters"]["session"], + child_session_id + ); + assert!(serde_json::to_string(&explicit_child_session) + .unwrap() + .contains("codex-session-child")); let primary_only = json_output(ctx(&temp).args(["search", "subagent", "--primary-only", "--json"])); diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index d2b4422c3..618f4025b 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -682,15 +682,7 @@ fn event_hit_matches_filters( return false; } } - if filters.primary_only { - let is_primary = hit.session_is_primary.unwrap_or(false) - || hit.agent_type == Some(ctx_history_core::AgentType::Primary); - if !is_primary { - return false; - } - } else if !filters.include_subagents - && hit.agent_type == Some(ctx_history_core::AgentType::Subagent) - { + if !event_hit_matches_agent_scope(hit, filters) { return false; } if let Some(event_type) = filters.event_type { @@ -1292,7 +1284,9 @@ fn search_sections( ), hit: record_hit.clone(), }); - if !context_has_excluded_provider_session(context, filters) { + let include_record_text = record_text_matches_agent_scope(context, filters) + && !context_has_excluded_provider_session(context, filters); + if include_record_text { sections.push(SearchSection { reason: "primary_user_message", weight: 5.0, @@ -1306,21 +1300,26 @@ fn search_sections( hit: record_hit.clone(), }); } - for tag in &record.tags { - sections.push(SearchSection { - reason: "tag", - weight: 3.0, - text: tag.clone(), - citation: citation( - ContextCitationType::HistoryRecord, - record.id, - "session tag", - record.updated_at, - ), - hit: record_hit.clone(), - }); + if include_record_text { + for tag in &record.tags { + sections.push(SearchSection { + reason: "tag", + weight: 3.0, + text: tag.clone(), + citation: citation( + ContextCitationType::HistoryRecord, + record.id, + "session tag", + record.updated_at, + ), + hit: record_hit.clone(), + }); + } } for session in &context.sessions { + if !session_matches_agent_scope(session, filters) { + continue; + } let hit = session_hit(session, context); sections.push(SearchSection { reason: "session_metadata", @@ -1344,6 +1343,9 @@ fn search_sections( } for run in &context.runs { + if !item_matches_agent_scope(run.session_id, run.source_id, context, filters) { + continue; + } let hit = run_hit(run, context); sections.push(SearchSection { reason: "run_command", @@ -1369,6 +1371,9 @@ fn search_sections( } for event in &context.events { + if !item_matches_agent_scope(event.session_id, event.capture_source_id, context, filters) { + continue; + } let event_text = event_text(event); let hit = event_hit(event, context); sections.push(SearchSection { @@ -1394,6 +1399,9 @@ fn search_sections( } for artifact in &context.artifacts { + if !item_matches_agent_scope(None, artifact.source_id, context, filters) { + continue; + } let hit = artifact_hit(artifact, context); sections.push(SearchSection { reason: "artifact", @@ -1415,6 +1423,16 @@ fn search_sections( } for file in &context.files_touched { + let session_id = file.event_id.and_then(|id| { + context + .events + .iter() + .find(|event| event.id == id) + .and_then(|event| event.session_id) + }); + if !item_matches_agent_scope(session_id, file.source_id, context, filters) { + continue; + } let hit = file_hit(file, context); sections.push(SearchSection { reason: "file_touched", @@ -1431,6 +1449,9 @@ fn search_sections( } for change in &context.vcs_changes { + if !item_matches_agent_scope(None, change.source_id, context, filters) { + continue; + } let parent_change_ids = change.parent_change_ids.join(" "); let hit = source_hit( change.source_id, @@ -1458,6 +1479,9 @@ fn search_sections( } for summary in &context.summaries { + if !item_matches_agent_scope(None, summary.source_id, context, filters) { + continue; + } let hit = source_hit(summary.source_id, summary.timestamps.updated_at, context); sections.push(SearchSection { reason: "summary", @@ -1476,6 +1500,83 @@ fn search_sections( sections } +fn session_matches_agent_scope(session: &Session, filters: &SearchFilters) -> bool { + if filters.session == Some(session.id) { + return true; + } + if filters.include_subagents && !filters.primary_only { + return true; + } + session_is_primary(session) + || (!filters.primary_only + && session.agent_type == ctx_history_core::AgentType::Unknown + && session.parent_session_id.is_none()) +} + +fn session_is_primary(session: &Session) -> bool { + session.is_primary || session.agent_type == ctx_history_core::AgentType::Primary +} + +fn event_hit_matches_agent_scope(hit: &EventSearchHit, filters: &SearchFilters) -> bool { + if filters.session.is_some() && filters.session == hit.session_id { + return true; + } + if filters.include_subagents && !filters.primary_only { + return true; + } + if hit.session_is_primary == Some(true) + || hit.agent_type == Some(ctx_history_core::AgentType::Primary) + { + return true; + } + if filters.primary_only { + return false; + } + hit.session_is_primary.is_none() && hit.agent_type.is_none() +} + +fn record_text_matches_agent_scope(context: &RecordContext, filters: &SearchFilters) -> bool { + context + .sessions + .iter() + .all(|session| session_matches_agent_scope(session, filters)) +} + +fn item_matches_agent_scope( + session_id: Option, + source_id: Option, + context: &RecordContext, + filters: &SearchFilters, +) -> bool { + associated_session(session_id, source_id, context) + .map(|session| session_matches_agent_scope(session, filters)) + .unwrap_or(true) +} + +fn associated_session( + session_id: Option, + source_id: Option, + context: &RecordContext, +) -> Option<&Session> { + session_id + .and_then(|id| context.sessions.iter().find(|session| session.id == id)) + .or_else(|| source_id.and_then(|id| associated_session_for_source(id, context))) +} + +fn associated_session_for_source(source_id: Uuid, context: &RecordContext) -> Option<&Session> { + context + .sessions + .iter() + .find(|session| session.capture_source_id == Some(source_id)) + .or_else(|| { + let source = context.sources.get(&source_id)?; + context.sessions.iter().find(|session| { + session.provider == source.descriptor.provider + && session.external_session_id == source.descriptor.external_session_id + }) + }) +} + fn record_context_display_hit( context: &RecordContext, filters: &SearchFilters, @@ -1485,12 +1586,18 @@ fn record_context_display_hit( .sessions .iter() .find(|session| { - filters - .provider - .map_or(true, |provider| session.provider == provider) + session_matches_agent_scope(session, filters) + && filters + .provider + .map_or(true, |provider| session.provider == provider) && filters.session.map_or(true, |id| session.id == id) }) - .or_else(|| context.sessions.first()) + .or_else(|| { + context + .sessions + .iter() + .find(|session| session_matches_agent_scope(session, filters)) + }) .map(|session| session_hit(session, context)) .unwrap_or_else(|| empty_hit(time)) } @@ -1637,7 +1744,7 @@ fn source_hit( return empty_hit(time); }; let raw_source_path = source.descriptor.raw_source_path.clone(); - HitMetadata { + let mut hit = HitMetadata { time, provider: Some(source.descriptor.provider), provider_session_id: source.descriptor.external_session_id.clone(), @@ -1652,7 +1759,15 @@ fn source_hit( .map(|path| Path::new(path).exists()), raw_source_path, cursor: source_cursor(source), + }; + if let Some(session) = associated_session_for_source(source.id, context) { + hit.provider = Some(session.provider); + hit.provider_session_id = session.external_session_id.clone(); + hit.session_id = Some(session.id); + hit.parent_session_id = session.parent_session_id; + hit.root_session_id = session.root_session_id; } + hit } fn source_for_id( @@ -1912,20 +2027,12 @@ fn record_matches_filters( } } - if filters.primary_only { - if !context.sessions.iter().any(|session| { - session.is_primary || session.agent_type == ctx_history_core::AgentType::Primary - }) { - return false; - } - } else if !filters.include_subagents - && context + if (filters.primary_only || !filters.include_subagents) + && !context.sessions.is_empty() + && !context .sessions .iter() - .any(|session| session.agent_type == ctx_history_core::AgentType::Subagent) - && !context.sessions.iter().any(|session| { - session.is_primary || session.agent_type == ctx_history_core::AgentType::Primary - }) + .any(|session| session_matches_agent_scope(session, filters)) { return false; } @@ -2060,7 +2167,10 @@ fn search_snippet( return matched_snippet(§ion.text, &terms, max_chars); } } - if !record.body.trim().is_empty() && !context_has_excluded_provider_session(context, filters) { + if !record.body.trim().is_empty() + && record_text_matches_agent_scope(context, filters) + && !context_has_excluded_provider_session(context, filters) + { return local_snippet(&record.body, max_chars); } String::new() From 572ba1438d3200c008231c67248d9b478b0a3525 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:36:20 -0500 Subject: [PATCH 16/72] test: keep checkpoint helpers clippy-clean --- crates/ctx-cli/src/main.rs | 2 +- crates/ctx-history-store/src/lib.rs | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index ba4abe745..05b6d60a2 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -5046,7 +5046,7 @@ mod tests { let path = temp.path().join("session.jsonl"); { let mut file = fs::File::create(&path).unwrap(); - write!(file, "prefix\n").unwrap(); + writeln!(file, "prefix").unwrap(); } let prefix_hash = sha256_file_prefix_hex(&path, 7).unwrap(); assert!(catalog_import_checkpoint_matches(&path, 7, Some(&prefix_hash)).unwrap()); diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 098061f22..d55961e51 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -7317,6 +7317,18 @@ mod search_order_tests { mod catalog_tests { use super::*; + type CatalogSessionCheckpointRow = ( + String, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + ); + fn tempdir() -> tempfile::TempDir { let root = std::env::current_dir().unwrap().join("target/test-data"); fs::create_dir_all(&root).unwrap(); @@ -7667,17 +7679,7 @@ mod catalog_tests { checkpoint_size, checkpoint_mtime, checkpoint_event_count, - ): ( - String, - Option, - Option, - Option, - Option, - Option, - Option, - Option, - Option, - ) = store + ): CatalogSessionCheckpointRow = store .conn .query_row( "SELECT indexed_status, indexed_at_ms, indexed_file_size_bytes, indexed_file_modified_at_ms, indexed_event_count, last_imported_at_ms, last_imported_file_size_bytes, last_imported_file_modified_at_ms, last_imported_event_count FROM catalog_sessions WHERE source_path = ?1", From bad3cace3ed578199d90bf014cfcf3ea12208260 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:40:36 -0500 Subject: [PATCH 17/72] build: support Darwin CLI release artifacts from Linux --- Cargo.lock | 96 --------------------------- Cargo.toml | 2 +- crates/ctx-cli/src/analytics.rs | 4 +- crates/ctx-cli/src/identity.rs | 4 +- crates/ctx-cli/src/main.rs | 21 +++--- crates/ctx-cli/src/upgrade.rs | 10 +-- crates/ctx-history-capture/src/lib.rs | 36 +++++----- crates/ctx-history-core/src/lib.rs | 8 ++- crates/ctx-history-search/src/lib.rs | 10 +-- crates/ctx-history-store/src/lib.rs | 6 +- scripts/build-public-cli-artifact.sh | 11 ++- 11 files changed, 63 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 97b76076a..ad09f72b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,15 +29,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "1.0.0" @@ -175,12 +166,8 @@ version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ - "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", - "windows-link", ] [[package]] @@ -239,12 +226,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -545,30 +526,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "icu_collections" version = "2.2.0" @@ -1324,65 +1281,12 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 730b352ef..fdb648039 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rust-version = "1.81" anyhow = "1.0" assert_cmd = "2.0" base64 = "0.22" -chrono = { version = "0.4", features = ["serde"] } +chrono = { version = "0.4", default-features = false, features = ["std", "serde"] } clap = { version = "4.5", features = ["derive", "env"] } clap_mangen = "0.2" directories = "5.0" diff --git a/crates/ctx-cli/src/analytics.rs b/crates/ctx-cli/src/analytics.rs index 642dcbbcc..c0e7769a0 100644 --- a/crates/ctx-cli/src/analytics.rs +++ b/crates/ctx-cli/src/analytics.rs @@ -1,7 +1,7 @@ use std::{env, path::Path, time::Duration}; use anyhow::Result; -use chrono::Utc; +use ctx_history_core::utc_now; use serde_json::{json, Map, Value}; use uuid::Uuid; @@ -60,7 +60,7 @@ fn send_cli_event_inner( "event_id": Uuid::now_v7().to_string(), "event_name": "cli_invocation", "event_version": 1, - "occurred_at": Utc::now(), + "occurred_at": utc_now(), "plane": "product", "delivery": "remote", "origin_runtime": "cli", diff --git a/crates/ctx-cli/src/identity.rs b/crates/ctx-cli/src/identity.rs index 7c2a2ce03..8d5a74619 100644 --- a/crates/ctx-cli/src/identity.rs +++ b/crates/ctx-cli/src/identity.rs @@ -4,7 +4,7 @@ use std::{ }; use anyhow::{Context, Result}; -use chrono::Utc; +use ctx_history_core::utc_now; use serde_json::json; use uuid::Uuid; @@ -29,7 +29,7 @@ pub fn install_id(data_root: &Path) -> Result { let body = serde_json::to_vec_pretty(&json!({ "schema_version": 1, "install_id": id, - "created_at": Utc::now(), + "created_at": utc_now(), }))?; fs::write(&path, body).with_context(|| format!("write {}", path.display()))?; Ok(id) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 05b6d60a2..b0e6ec42b 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -40,8 +40,9 @@ use ctx_history_capture::{ ProviderImportSummary, ProviderImportSupport, ProviderSource, ProviderSourceStatus, }; use ctx_history_core::{ - database_path, default_data_root, CaptureProvider, ContextCitation, ContextCitationType, Event, - EventRole, EventType, HistoryRecord, ProviderRawRetention, RedactionState, Session, + database_path, default_data_root, utc_now, CaptureProvider, ContextCitation, + ContextCitationType, Event, EventRole, EventType, HistoryRecord, ProviderRawRetention, + RedactionState, Session, }; use ctx_history_store::{ CatalogSession, CatalogSourceIndexUpdate, RawSqlOptions, RawSqlResult, RawSqlValue, @@ -4365,7 +4366,7 @@ fn import_manifested_source( .iter() .map(|file| file.source_path.clone()) .collect::>(); - let observed_at_ms = Utc::now().timestamp_millis(); + let observed_at_ms = utc_now().timestamp_millis(); store.begin_immediate_batch()?; let persist = (|| -> Result<()> { store.upsert_source_import_files(&files)?; @@ -4406,7 +4407,7 @@ fn import_manifested_source( source_path: &pending_file.source_path, file_size_bytes: pending_file.file_size_bytes, file_modified_at_ms: pending_file.file_modified_at_ms, - indexed_at_ms: Utc::now().timestamp_millis(), + indexed_at_ms: utc_now().timestamp_millis(), }, )?; merge_provider_import_summary(&mut summary, file_summary); @@ -4417,7 +4418,7 @@ fn import_manifested_source( &source_root, &pending_file.source_path, &err.to_string(), - Utc::now().timestamp_millis(), + utc_now().timestamp_millis(), )?; return Err(err); } @@ -4455,7 +4456,7 @@ fn merge_provider_import_summary( fn collect_source_import_files(source: &SourceInfo) -> Result> { let paths = collect_source_import_paths(source)?; let source_root = source.path.display().to_string(); - let observed_at_ms = Utc::now().timestamp_millis(); + let observed_at_ms = utc_now().timestamp_millis(); let mut files = Vec::with_capacity(paths.len()); for path in paths { let metadata = fs::metadata(&path) @@ -4640,7 +4641,7 @@ fn import_incremental_codex_session_tree( store, session, event_count, - Utc::now().timestamp_millis(), + utc_now().timestamp_millis(), )?; merge_provider_import_summary(&mut summary, tail_summary); } else { @@ -4686,7 +4687,7 @@ fn mark_catalog_sessions_indexed( sessions: &[CatalogSession], summary: &ProviderImportSummary, ) -> Result<()> { - let indexed_at_ms = Utc::now().timestamp_millis(); + let indexed_at_ms = utc_now().timestamp_millis(); let event_count = if sessions.len() == 1 { Some( summary @@ -4763,7 +4764,7 @@ fn mark_catalog_sessions_failed( sessions: &[CatalogSession], error: &str, ) -> Result<()> { - let indexed_at_ms = Utc::now().timestamp_millis(); + let indexed_at_ms = utc_now().timestamp_millis(); for session in sessions { store.mark_catalog_source_failed( session.provider, @@ -4998,7 +4999,7 @@ fn parse_since_filter(value: &str) -> Result> { let days: i64 = days .parse() .with_context(|| format!("invalid --since day window: {value}"))?; - return Ok(Utc::now() - Duration::days(days)); + return Ok(utc_now() - Duration::days(days)); } Ok(chrono::DateTime::parse_from_rfc3339(trimmed) .with_context(|| format!("invalid --since value: {value}"))? diff --git a/crates/ctx-cli/src/upgrade.rs b/crates/ctx-cli/src/upgrade.rs index f36204222..33b1e1590 100644 --- a/crates/ctx-cli/src/upgrade.rs +++ b/crates/ctx-cli/src/upgrade.rs @@ -8,8 +8,8 @@ use std::{ use anyhow::{anyhow, Context, Result}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; -use chrono::Utc; use clap::{Args, Subcommand}; +use ctx_history_core::utc_now; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; @@ -949,7 +949,7 @@ fn write_install_marker_to(marker_path: &Path, plan: &UpgradePlan) -> Result<()> "source_commit": plan.metadata.source_commit, "published_at": plan.metadata.published_at, "store_schema_version": plan.metadata.store_schema_version, - "installed_at": Utc::now(), + "installed_at": utc_now(), }); atomic_write_json(marker_path, &body) } @@ -988,7 +988,7 @@ fn write_state_checked(data_root: &Path, plan: &UpgradePlan, status: &str) -> Re let body = json!({ "schema_version": 1, "status": status, - "checked_at": Utc::now(), + "checked_at": utc_now(), "last_checked_unix_s": now_unix_s(), "current_version": plan.current_version, "latest_version": plan.latest_version, @@ -1007,7 +1007,7 @@ fn write_state_error(data_root: &Path, error: &str) -> Result<()> { let body = json!({ "schema_version": 1, "status": "error", - "checked_at": Utc::now(), + "checked_at": utc_now(), "last_checked_unix_s": now_unix_s(), "error": error, }); @@ -1081,7 +1081,7 @@ fn append_upgrade_log(data_root: &Path, message: &str) { let _ = fs::create_dir_all(parent); } if let Ok(mut file) = fs::OpenOptions::new().create(true).append(true).open(&path) { - let _ = writeln!(file, "{} {}", Utc::now().to_rfc3339(), message); + let _ = writeln!(file, "{} {}", utc_now().to_rfc3339(), message); } } diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index a748c18f6..7e435bfd4 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -12,7 +12,7 @@ use std::{ use chrono::{DateTime, Utc}; use ctx_history_core::{ - inbox_dir as core_inbox_dir, new_id, AgentType, CaptureEnvelope, CaptureProvider, + inbox_dir as core_inbox_dir, new_id, utc_now, AgentType, CaptureEnvelope, CaptureProvider, CaptureSource, CaptureSourceDescriptor, CaptureSourceKind, Confidence, EntityTimestamps, Event, EventRole, EventType, Fidelity, FileChangeKind, FileTouched, HistoryRecord, ProviderCaptureEnvelope, ProviderCursorCheckpoint, ProviderCursorRange, ProviderEventEnvelope, @@ -86,7 +86,7 @@ impl SpoolWriter { let machine_id = sanitize_filename_component(machine_id); let pid = std::process::id(); - let unix_ms = Utc::now().timestamp_millis(); + let unix_ms = utc_now().timestamp_millis(); let random = new_id().simple().to_string(); let name = format!("capture-{machine_id}-{pid}-{unix_ms}-{random}.jsonl"); let final_path = inbox.join(name); @@ -148,7 +148,7 @@ impl Default for FixtureOptions { dedupe_key: None, machine_id: None, cwd: None, - occurred_at: Utc::now(), + occurred_at: utc_now(), } } } @@ -194,7 +194,7 @@ impl Default for ProviderFixtureImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, expected_provider: None, allow_partial_failures: false, @@ -239,7 +239,7 @@ impl Default for CodexHistoryImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -273,7 +273,7 @@ impl Default for CodexSessionImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, max_session_files: None, @@ -346,7 +346,7 @@ impl Default for CodexSessionCatalogOptions { fn default() -> Self { Self { source_root: None, - cataloged_at: Utc::now(), + cataloged_at: utc_now(), allow_partial_failures: true, max_session_files: None, max_total_bytes: None, @@ -380,7 +380,7 @@ impl Default for PiSessionImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -401,7 +401,7 @@ impl Default for ClaudeProjectsImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -422,7 +422,7 @@ impl Default for OpenCodeSqliteImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -443,7 +443,7 @@ impl Default for AntigravityCliImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -464,7 +464,7 @@ impl Default for GeminiCliImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -485,7 +485,7 @@ impl Default for FactoryAiDroidImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -506,7 +506,7 @@ impl Default for CopilotCliImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -527,7 +527,7 @@ impl Default for CursorNativeImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -646,7 +646,7 @@ impl Default for ProviderAdapterContext { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), tool_output_mode: CodexToolOutputMode::Full, event_mode: CodexEventImportMode::Rich, include_notices: true, @@ -6419,7 +6419,7 @@ fn pi_session_event(entry: &Value, line_number: usize) -> ProviderEventEnvelope .and_then(Value::as_str) .and_then(|timestamp| DateTime::parse_from_rfc3339(timestamp).ok()) .map(|time| time.with_timezone(&Utc)) - .unwrap_or_else(Utc::now); + .unwrap_or_else(utc_now); let event_type = pi_event_type(entry_type, message); let role = message_role.map(pi_event_role); let text = message.and_then(pi_message_text); @@ -7454,7 +7454,7 @@ fn ensure_regular_spool_file(path: &Path) -> Result<()> { fn write_failure_metadata(failed_path: &Path, err: &CaptureError) -> Result<()> { let sidecar = append_suffix(failed_path, ".error.json")?; let metadata = json!({ - "failed_at": Utc::now(), + "failed_at": utc_now(), "spool_file": failed_path, "error": err.to_string(), }); diff --git a/crates/ctx-history-core/src/lib.rs b/crates/ctx-history-core/src/lib.rs index 4e1898427..8e58d2716 100644 --- a/crates/ctx-history-core/src/lib.rs +++ b/crates/ctx-history-core/src/lib.rs @@ -1,4 +1,4 @@ -use std::{env, fmt, path::PathBuf, str::FromStr, sync::OnceLock}; +use std::{env, fmt, path::PathBuf, str::FromStr, sync::OnceLock, time::SystemTime}; use chrono::{DateTime, Utc}; use directories::BaseDirs; @@ -20,6 +20,10 @@ pub enum CoreError { pub type Result = std::result::Result; +pub fn utc_now() -> DateTime { + DateTime::::from(SystemTime::now()) +} + macro_rules! text_enum { ( $(#[$meta:meta])* @@ -588,7 +592,7 @@ impl HistoryRecord { kind: impl Into, workspace: Option, ) -> Self { - let now = Utc::now(); + let now = utc_now(); Self { id: new_id(), title: title.into(), diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index 618f4025b..9afd0563c 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -6,7 +6,7 @@ use std::{ use chrono::Utc; use ctx_history_core::{ - Artifact, ContextCitation, ContextCitationType, ContextLinks, ContextPagination, + utc_now, Artifact, ContextCitation, ContextCitationType, ContextLinks, ContextPagination, ContextTruncation, Event, EventType, FileTouched, HistoryRecord, RedactionState, Run, Session, Summary, VcsChange, Visibility, }; @@ -250,7 +250,7 @@ pub fn search_packet(store: &Store, query: &str, options: &PacketOptions) -> Res schema_version: SEARCH_PACKET_SCHEMA_VERSION, query: query.to_owned(), filters: options.filters, - generated_at: Utc::now(), + generated_at: utc_now(), results, pagination: pagination(Some(cursor_offset), has_more), truncation, @@ -324,7 +324,7 @@ pub fn search_packet_terms( schema_version: SEARCH_PACKET_SCHEMA_VERSION, query: search_terms.join(" OR "), filters: options.filters, - generated_at: Utc::now(), + generated_at: utc_now(), results: merged_results, pagination: pagination(Some(cursor_offset), has_more), truncation, @@ -640,7 +640,7 @@ fn fast_event_search_packet( schema_version: SEARCH_PACKET_SCHEMA_VERSION, query: query.to_owned(), filters: options.filters.clone(), - generated_at: Utc::now(), + generated_at: utc_now(), results, pagination: pagination(Some(cursor_offset), has_more), truncation, @@ -652,7 +652,7 @@ fn empty_search_packet(query: &str, options: &PacketOptions) -> SearchPacket { schema_version: SEARCH_PACKET_SCHEMA_VERSION, query: query.to_owned(), filters: options.filters.clone(), - generated_at: Utc::now(), + generated_at: utc_now(), results: Vec::new(), pagination: pagination(Some(0), false), truncation: ContextTruncation::default(), diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index d55961e51..b9eb28abb 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -14,7 +14,7 @@ use std::os::unix::fs::PermissionsExt; use chrono::{DateTime, Utc}; use ctx_history_core::{ - new_id, AgentType, Artifact, ArtifactKind, CaptureProvider, CaptureSource, + new_id, utc_now, AgentType, Artifact, ArtifactKind, CaptureProvider, CaptureSource, CaptureSourceDescriptor, EntityTimestamps, Event, EventRole, EventType, Fidelity, FileTouched, HistoryRecord, HistoryRecordLink, RedactionState, Run, RunStatus, RunType, Session, SessionEdge, SessionHistoryArchive, SessionStatus, Summary, SyncCursor, SyncMetadata, @@ -2789,7 +2789,7 @@ impl Store { if let Some(device) = self.local_device()? { return Ok(device); } - let now = Utc::now(); + let now = utc_now(); let device = LocalDeviceIdentity { id: new_id(), stable_device_id: format!("ctx-device-{}", new_id().simple()), @@ -2821,7 +2821,7 @@ impl Store { let root = root_path.as_ref(); let root_path_hash = sha256_hex(root.display().to_string().as_bytes()); let display_root = root.display().to_string(); - let now = Utc::now(); + let now = utc_now(); let id = new_id(); self.conn.execute( r#" diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index e8bfdd9c3..a62bceb88 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -56,7 +56,16 @@ rustup target add "${target}" >/dev/null out_dir="${CTX_PUBLIC_CLI_ARTIFACT_DIR:-target/public-cli-artifacts}" mkdir -p "${out_dir}" -if [[ "${platform}" == "freebsd-x64" ]]; then +if [[ "${platform}" == macos-* && "$(uname -s)" != "Darwin" ]]; then + if ! command -v cargo-zigbuild >/dev/null 2>&1; then + cargo install cargo-zigbuild --locked + fi + if ! command -v zig >/dev/null 2>&1; then + echo "error: zig is required to cross-build ${platform} from $(uname -s)" >&2 + exit 127 + fi + cargo zigbuild -p ctx --release --target "${target}" --locked +elif [[ "${platform}" == "freebsd-x64" ]]; then if ! command -v cross >/dev/null 2>&1; then cargo install cross --locked fi From be42c1a6301c909d7422fa1dbcfcb5e7b1ed17cf Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:54:30 -0500 Subject: [PATCH 18/72] ci: cross-build macOS CLI artifacts on Linux --- .buildkite/pipeline.yml | 20 +++----- scripts/build-public-cli-artifact.sh | 74 +++++++++++++++++++++++++--- scripts/check-buildkite-pipeline.sh | 43 ++++++++++++++-- 3 files changed, 115 insertions(+), 22 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 302fc84c5..a33229f19 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -115,12 +115,10 @@ steps: command: | scripts/build-public-cli-artifact.sh macos-arm64 agents: - queue: "ctx-mac-gui-shared-arm64" - os: "darwin" - arch: "arm64" - concurrency: 1 - concurrency_group: "ctx-mac-gui-shared-arm64" - concurrency_method: "eager" + queue: "release-linux-managed" + ctx-runner-class: "release-linux-control" + os: "linux" + arch: "x86_64" timeout_in_minutes: 60 artifact_paths: - "target/public-cli-artifacts/ctx-macos-arm64" @@ -133,12 +131,10 @@ steps: command: | scripts/build-public-cli-artifact.sh macos-x64 agents: - queue: "ctx-mac-gui-shared-arm64" - os: "darwin" - arch: "arm64" - concurrency: 1 - concurrency_group: "ctx-mac-gui-shared-arm64" - concurrency_method: "eager" + queue: "release-linux-managed" + ctx-runner-class: "release-linux-control" + os: "linux" + arch: "x86_64" timeout_in_minutes: 60 artifact_paths: - "target/public-cli-artifacts/ctx-macos-x64" diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index a62bceb88..83430f465 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash set -euo pipefail +ZIG_VERSION="0.14.1" +ZIG_LINUX_X64_URL="https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" +ZIG_LINUX_X64_SHA256="24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c" +CARGO_ZIGBUILD_VERSION="0.23.0" + usage() { cat >&2 <<'USAGE' Usage: scripts/build-public-cli-artifact.sh PLATFORM @@ -46,6 +51,67 @@ esac root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${root_dir}" +ensure_zig_for_linux_x64() { + if command -v zig >/dev/null 2>&1; then + return + fi + + if [[ "$(uname -s)" != "Linux" ]]; then + echo "error: zig is required to cross-build ${platform} from $(uname -s)" >&2 + exit 127 + fi + case "$(uname -m)" in + x86_64|amd64) ;; + *) + echo "error: automatic Zig bootstrap only supports Linux x86_64, got $(uname -m)" >&2 + exit 127 + ;; + esac + + for required_tool in curl tar; do + if ! command -v "${required_tool}" >/dev/null 2>&1; then + echo "error: ${required_tool} is required to bootstrap Zig ${ZIG_VERSION}" >&2 + exit 127 + fi + done + + toolchain_dir="${CTX_PUBLIC_CLI_TOOLCHAIN_DIR:-target/public-cli-toolchain}" + install_dir="${toolchain_dir}/zig-x86_64-linux-${ZIG_VERSION}" + if [[ ! -x "${install_dir}/zig" ]]; then + mkdir -p "${toolchain_dir}" + archive="${toolchain_dir}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" + tmp_archive="${archive}.tmp" + curl -fsSL "${ZIG_LINUX_X64_URL}" -o "${tmp_archive}" + if command -v sha256sum >/dev/null 2>&1; then + actual_sha="$(sha256sum "${tmp_archive}" | awk '{ print $1 }')" + elif command -v shasum >/dev/null 2>&1; then + actual_sha="$(shasum -a 256 "${tmp_archive}" | awk '{ print $1 }')" + else + echo "error: sha256sum or shasum is required to verify Zig ${ZIG_VERSION}" >&2 + exit 127 + fi + if [[ "${actual_sha}" != "${ZIG_LINUX_X64_SHA256}" ]]; then + echo "error: Zig ${ZIG_VERSION} checksum mismatch: expected ${ZIG_LINUX_X64_SHA256}, got ${actual_sha}" >&2 + exit 1 + fi + mv "${tmp_archive}" "${archive}" + rm -rf "${install_dir}" + tar -C "${toolchain_dir}" -xf "${archive}" + fi + export PATH="${install_dir}:${PATH}" +} + +ensure_darwin_cross_tools() { + if ! command -v cargo-zigbuild >/dev/null 2>&1; then + cargo install cargo-zigbuild --version "${CARGO_ZIGBUILD_VERSION}" --locked + fi + ensure_zig_for_linux_x64 + command -v zig >/dev/null 2>&1 || { + echo "error: zig is required to cross-build ${platform} from $(uname -s)" >&2 + exit 127 + } +} + version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; data=json.load(sys.stdin); print(next(pkg["version"] for pkg in data["packages"] if pkg["name"] == "ctx"))')" if [[ "${version}" != "0.13.0" ]]; then echo "error: ctx package version must be 0.13.0 for this release, got ${version}" >&2 @@ -57,13 +123,7 @@ out_dir="${CTX_PUBLIC_CLI_ARTIFACT_DIR:-target/public-cli-artifacts}" mkdir -p "${out_dir}" if [[ "${platform}" == macos-* && "$(uname -s)" != "Darwin" ]]; then - if ! command -v cargo-zigbuild >/dev/null 2>&1; then - cargo install cargo-zigbuild --locked - fi - if ! command -v zig >/dev/null 2>&1; then - echo "error: zig is required to cross-build ${platform} from $(uname -s)" >&2 - exit 127 - fi + ensure_darwin_cross_tools cargo zigbuild -p ctx --release --target "${target}" --locked elif [[ "${platform}" == "freebsd-x64" ]]; then if ! command -v cross >/dev/null 2>&1; then diff --git a/scripts/check-buildkite-pipeline.sh b/scripts/check-buildkite-pipeline.sh index 70705d20d..18f1d6bb0 100755 --- a/scripts/check-buildkite-pipeline.sh +++ b/scripts/check-buildkite-pipeline.sh @@ -34,6 +34,14 @@ if command -v ruby >/dev/null 2>&1; then next unless step.is_a?(Hash) abort "artifact step #{step["key"]} must be gated" unless step["if"].to_s.include?("CTX_PUBLIC_CLI_ARTIFACT_MATRIX") end + %w[public-cli-macos-arm64 public-cli-macos-x64].each do |key| + step = steps.find { |candidate| candidate.is_a?(Hash) && candidate["key"] == key } + abort "missing macOS artifact step #{key}" unless step + abort "#{key} must cross-build on release-linux-managed" unless step.dig("agents", "queue") == "release-linux-managed" + abort "#{key} must run on linux" unless step.dig("agents", "os") == "linux" + abort "#{key} must run on x86_64" unless step.dig("agents", "arch") == "x86_64" + abort "#{key} must not serialize on the Mac GUI queue" if step.key?("concurrency_group") + end ' "${pipeline}" else top_level_steps="$( @@ -62,13 +70,42 @@ for required in \ 'scripts/build-public-cli-artifact.sh windows-x64' \ 'scripts/build-public-cli-artifact.sh freebsd-x64' \ 'scripts/build-public-cli-artifact.sh macos-arm64' \ - 'scripts/build-public-cli-artifact.sh macos-x64'; do + 'scripts/build-public-cli-artifact.sh macos-x64' \ + 'cargo zigbuild -p ctx --release --target "${target}" --locked' \ + 'CARGO_ZIGBUILD_VERSION' \ + 'ZIG_LINUX_X64_SHA256'; do if ! grep -F -q "${required}" "${pipeline}"; then - printf 'pipeline missing required snippet: %s\n' "${required}" >&2 - exit 1 + if ! grep -F -q "${required}" scripts/build-public-cli-artifact.sh; then + printf 'pipeline or artifact script missing required snippet: %s\n' "${required}" >&2 + exit 1 + fi + continue fi done +if grep -F -q 'ctx-mac-gui-shared-arm64' "${pipeline}"; then + printf 'public CLI artifact matrix must not use the scarce Mac GUI queue\n' >&2 + exit 1 +fi + +for mac_step in public-cli-macos-arm64 public-cli-macos-x64; do + for required in \ + 'queue: "release-linux-managed"' \ + 'ctx-runner-class: "release-linux-control"' \ + 'os: "linux"' \ + 'arch: "x86_64"'; do + if ! awk ' + index($0, "key: \"" step "\"") { in_step = 1 } + in_step && /^ - label:/ && index($0, step) == 0 { in_step = 0 } + in_step && index($0, needle) { found = 1 } + END { exit found ? 0 : 1 } + ' step="${mac_step}" needle="${required}" "${pipeline}"; then + printf '%s artifact step missing required Linux runner snippet: %s\n' "${mac_step}" "${required}" >&2 + exit 1 + fi + done +done + if grep -E -q 'release-artifact|r2-|provider-live|OpenRouter|completion-certificate|freebsd-native-release-proof|CTX_PUBLIC_CLI_PERF_GATES|--mode=perf|public-perf' "${pipeline}"; then printf 'pipeline contains non-smoke release or provider-live wiring\n' >&2 exit 1 From 93a0c10b9af0687c30cdf713eea136b5320d6955 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:24:03 -0500 Subject: [PATCH 19/72] Add privacy-safe telemetry device identity --- crates/ctx-cli/src/analytics.rs | 25 ++ crates/ctx-cli/src/config.rs | 6 +- crates/ctx-cli/src/identity.rs | 133 ++++++++- crates/ctx-cli/src/main.rs | 107 +++++++- crates/ctx-cli/tests/cli.rs | 411 +++++++++++++++++++++++++++- crates/ctx-history-store/src/lib.rs | 21 +- docs/storage.md | 19 +- 7 files changed, 705 insertions(+), 17 deletions(-) diff --git a/crates/ctx-cli/src/analytics.rs b/crates/ctx-cli/src/analytics.rs index c0e7769a0..e87eb612b 100644 --- a/crates/ctx-cli/src/analytics.rs +++ b/crates/ctx-cli/src/analytics.rs @@ -34,6 +34,7 @@ fn send_cli_event_inner( config: &AppConfig, event: AnalyticsEvent<'_>, ) -> Result<()> { + let device_id = identity::device_id(data_root)?; let install_id = identity::install_id(data_root)?; let status = if event.success { "ok" } else { "error" }; let duration_ms = event.duration.as_millis().min(i64::MAX as u128) as i64; @@ -52,6 +53,7 @@ fn send_cli_event_inner( } let payload = json!({ "broker_install_id": install_id, + "broker_device_id": device_id, "broker_runtime": "cli", "broker_app_version": env!("CARGO_PKG_VERSION"), "broker_os": std::env::consts::OS, @@ -65,6 +67,7 @@ fn send_cli_event_inner( "delivery": "remote", "origin_runtime": "cli", "origin_install_id": install_id, + "origin_device_id": device_id, "app_version": env!("CARGO_PKG_VERSION"), "os": std::env::consts::OS, "arch": std::env::consts::ARCH, @@ -101,6 +104,18 @@ pub fn insert_bytes_bucket(properties: &mut AnalyticsProperties, key: &str, byte insert_str(properties, key, bytes_bucket(bytes)); } +pub fn insert_duration(properties: &mut AnalyticsProperties, prefix: &str, duration: Duration) { + insert_str( + properties, + &format!("{prefix}_bucket"), + duration_bucket(duration), + ); +} + +pub fn insert_text_length_bucket(properties: &mut AnalyticsProperties, key: &str, chars: usize) { + insert_str(properties, key, text_length_bucket(chars)); +} + pub fn count_bucket(count: u64) -> &'static str { match count { 0 => "0", @@ -125,6 +140,16 @@ pub fn bytes_bucket(bytes: u64) -> &'static str { } } +pub fn text_length_bucket(chars: usize) -> &'static str { + match chars { + 0 => "0", + 1..=20 => "1-20", + 21..=100 => "21-100", + 101..=500 => "101-500", + _ => "500+", + } +} + fn duration_bucket(duration: Duration) -> &'static str { let ms = duration.as_millis(); match ms { diff --git a/crates/ctx-cli/src/config.rs b/crates/ctx-cli/src/config.rs index 0fbe6b14a..aade01a13 100644 --- a/crates/ctx-cli/src/config.rs +++ b/crates/ctx-cli/src/config.rs @@ -86,14 +86,14 @@ impl AppConfig { } fn apply_env(&mut self) { - if env_flag("CTX_ANALYTICS_OFF") || env_flag("CTX_DISABLE_ANALYTICS") { - self.analytics.enabled = false; - } if let Ok(value) = env::var("CTX_ANALYTICS_ENABLED") { if let Some(enabled) = parse_bool_value(&value) { self.analytics.enabled = enabled; } } + if env_flag("CTX_ANALYTICS_OFF") || env_flag("CTX_DISABLE_ANALYTICS") { + self.analytics.enabled = false; + } if let Ok(endpoint) = env::var("CTX_ANALYTICS_ENDPOINT") { if !endpoint.trim().is_empty() { self.analytics.endpoint = endpoint; diff --git a/crates/ctx-cli/src/identity.rs b/crates/ctx-cli/src/identity.rs index 8d5a74619..d9d5a0902 100644 --- a/crates/ctx-cli/src/identity.rs +++ b/crates/ctx-cli/src/identity.rs @@ -1,14 +1,15 @@ use std::{ - fs, + env, fs, path::{Path, PathBuf}, }; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use ctx_history_core::utc_now; use serde_json::json; use uuid::Uuid; const INSTALL_FILE: &str = "install.json"; +const DEVICE_FILE: &str = "device.json"; pub fn install_id(data_root: &Path) -> Result { fs::create_dir_all(data_root)?; @@ -38,3 +39,131 @@ pub fn install_id(data_root: &Path) -> Result { pub fn install_path(data_root: &Path) -> PathBuf { data_root.join(INSTALL_FILE) } + +pub fn device_id(data_root: &Path) -> Result { + let path = device_path(data_root)?; + if path.exists() { + let value: serde_json::Value = serde_json::from_slice( + &fs::read(&path).with_context(|| format!("read {}", path.display()))?, + ) + .with_context(|| format!("parse {}", path.display()))?; + if let Some(id) = value.get("device_id").and_then(|value| value.as_str()) { + if Uuid::parse_str(id.trim()).is_ok() { + return Ok(id.trim().to_owned()); + } + } + } + + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let id = Uuid::new_v4().to_string(); + let body = serde_json::to_vec_pretty(&json!({ + "schema_version": 1, + "device_id": id, + "created_at": utc_now(), + }))?; + write_private_file(&path, &body).with_context(|| format!("write {}", path.display()))?; + Ok(id) +} + +pub fn device_path(data_root: &Path) -> Result { + let path = device_state_dir()?.join(DEVICE_FILE); + ensure_device_path_outside_data_root(&path, data_root)?; + Ok(path) +} + +fn ensure_device_path_outside_data_root(path: &Path, data_root: &Path) -> Result<()> { + let normalized_path = normalize_for_prefix_check(path); + let normalized_data_root = normalize_for_prefix_check(data_root); + if normalized_path.starts_with(&normalized_data_root) { + bail!( + "refusing to store telemetry device identity under ctx data root: {}", + path.display() + ); + } + Ok(()) +} + +fn normalize_for_prefix_check(path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + env::current_dir() + .map(|cwd| cwd.join(path)) + .unwrap_or_else(|_| path.to_path_buf()) + } +} + +#[cfg(target_os = "windows")] +fn device_state_dir() -> Result { + if let Some(local_app_data) = non_empty_env_path("LOCALAPPDATA") { + return Ok(local_app_data.join("ctx")); + } + Ok(home_dir() + .context("resolve home directory")? + .join("AppData") + .join("Local") + .join("ctx")) +} + +#[cfg(target_os = "macos")] +fn device_state_dir() -> Result { + Ok(home_dir() + .context("resolve home directory")? + .join("Library") + .join("Application Support") + .join("ctx")) +} + +#[cfg(all(not(target_os = "windows"), not(target_os = "macos")))] +fn device_state_dir() -> Result { + if let Some(xdg_state_home) = non_empty_env_path("XDG_STATE_HOME") { + return Ok(xdg_state_home.join("ctx")); + } + Ok(home_dir() + .context("resolve home directory")? + .join(".local") + .join("state") + .join("ctx")) +} + +fn non_empty_env_path(key: &str) -> Option { + env::var_os(key) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) +} + +fn home_dir() -> Option { + non_empty_env_path("HOME") + .or_else(|| non_empty_env_path("USERPROFILE")) + .or_else(|| { + let drive = env::var_os("HOMEDRIVE")?; + let path = env::var_os("HOMEPATH")?; + Some(PathBuf::from(format!( + "{}{}", + drive.to_string_lossy(), + path.to_string_lossy() + ))) + }) +} + +#[cfg(unix)] +fn write_private_file(path: &Path, body: &[u8]) -> Result<()> { + use std::{fs::OpenOptions, io::Write, os::unix::fs::OpenOptionsExt}; + + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(path)?; + file.write_all(body)?; + Ok(()) +} + +#[cfg(not(unix))] +fn write_private_file(path: &Path, body: &[u8]) -> Result<()> { + fs::write(path, body)?; + Ok(()) +} diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index b0e6ec42b..10091d1b0 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -1522,6 +1522,36 @@ fn indexed_history_item_count(store: &Store) -> Result { Ok(store.indexed_history_item_count()?) } +fn insert_store_analytics_counts( + analytics_properties: &mut AnalyticsProperties, + store: &Store, +) -> Result<()> { + let counts = store.indexed_history_counts()?; + analytics::insert_count_bucket( + analytics_properties, + "indexed_sessions_bucket", + counts.sessions as u64, + ); + analytics::insert_count_bucket( + analytics_properties, + "indexed_events_bucket", + counts.events as u64, + ); + analytics::insert_count_bucket( + analytics_properties, + "indexed_items_bucket", + counts.items() as u64, + ); + Ok(()) +} + +fn insert_db_size_bucket(analytics_properties: &mut AnalyticsProperties, db_path: &Path) { + let bytes = fs::metadata(db_path) + .map(|metadata| metadata.len()) + .unwrap_or(0); + analytics::insert_bytes_bucket(analytics_properties, "db_size_bucket", bytes); +} + fn setup_has_failed_sources(report: Option<&ImportReport>) -> bool { report.is_some_and(|report| report.totals.failed_sources > 0) } @@ -1534,18 +1564,27 @@ fn run_status( let db_path = database_path(data_root.clone()); let initialized = db_path.exists(); let config_path = data_root.join(CONFIG_FILE); - let (records, sources, catalog_counts) = if initialized { + let (records, sessions, events, sources, catalog_counts) = if initialized { let store = Store::open(&db_path)?; + let counts = store.indexed_history_counts()?; ( - indexed_history_item_count(&store)?, + counts.items(), + counts.sessions, + counts.events, store.capture_source_count()?, store.catalog_session_counts()?, ) } else { - (0, 0, Default::default()) + (0, 0, 0, 0, Default::default()) }; analytics::insert_bool(analytics_properties, "initialized", initialized); analytics::insert_count_bucket(analytics_properties, "indexed_items_bucket", records as u64); + analytics::insert_count_bucket( + analytics_properties, + "indexed_sessions_bucket", + sessions as u64, + ); + analytics::insert_count_bucket(analytics_properties, "indexed_events_bucket", events as u64); analytics::insert_count_bucket( analytics_properties, "indexed_sources_bucket", @@ -1556,6 +1595,7 @@ fn run_status( "cataloged_sessions_bucket", catalog_counts.total as u64, ); + insert_db_size_bucket(analytics_properties, &db_path); if args.json { print_json(json!({ @@ -1565,6 +1605,8 @@ fn run_status( "database_path": db_path, "config_path": config_path, "indexed_items": records, + "indexed_sessions": sessions, + "indexed_events": events, "indexed_sources": sources, "cataloged_sessions": catalog_counts.total, "indexed_catalog_sessions": catalog_counts.indexed, @@ -3677,9 +3719,53 @@ fn run_search( data_root: PathBuf, analytics_properties: &mut AnalyticsProperties, ) -> Result<()> { + let refresh_started = Instant::now(); let refresh = refresh_before_search(&args, &data_root)?; - let store = Store::open(database_path(data_root))?; + analytics::insert_duration( + analytics_properties, + "refresh_duration", + refresh_started.elapsed(), + ); + analytics::insert_str( + analytics_properties, + "search_refresh_mode", + refresh.mode.as_str(), + ); + analytics::insert_str( + analytics_properties, + "search_refresh_status", + refresh.status, + ); + analytics::insert_count_bucket( + analytics_properties, + "search_refresh_source_count_bucket", + refresh.source_count as u64, + ); + let db_path = database_path(data_root); + insert_db_size_bucket(analytics_properties, &db_path); + let store = Store::open(&db_path)?; + insert_store_analytics_counts(analytics_properties, &store)?; let query = args.query.unwrap_or_default(); + let query_term_count = query + .split_whitespace() + .filter(|term| !term.trim().is_empty()) + .count() + .saturating_add( + args.term + .iter() + .filter(|term| !term.trim().is_empty()) + .count(), + ); + analytics::insert_text_length_bucket( + analytics_properties, + "query_length_bucket", + query.chars().count(), + ); + analytics::insert_count_bucket( + analytics_properties, + "query_term_count_bucket", + query_term_count as u64, + ); let event_results = args.events || args.session.is_some(); let options = ctx_history_search::PacketOptions { limit: args.limit, @@ -3705,11 +3791,17 @@ fn run_search( ..ctx_history_search::PacketOptions::default() }; let uses_composed_terms = args.term.iter().any(|term| !term.trim().is_empty()); + let query_started = Instant::now(); let packet = if uses_composed_terms { ctx_history_search::search_packet_terms(&store, &query, &args.term, &options)? } else { ctx_history_search::search_packet(&store, &query, &options)? }; + analytics::insert_duration( + analytics_properties, + "query_duration", + query_started.elapsed(), + ); let result_count = packet.results.len(); let citation_count = packet .results @@ -3726,6 +3818,8 @@ fn run_search( "citation_count_bucket", citation_count as u64, ); + analytics::insert_bool(analytics_properties, "zero_result", result_count == 0); + let render_started = Instant::now(); if args.json { let suggested_next_query = (!uses_composed_terms).then_some(query.as_str()); print_share_safe_value(SearchDto::packet( @@ -3764,6 +3858,11 @@ fn run_search( } } } + analytics::insert_duration( + analytics_properties, + "render_duration", + render_started.elapsed(), + ); Ok(()) } diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index b56bd971f..eb7246ca0 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -8,6 +8,7 @@ use ring::{ use rusqlite::{params, Connection}; use serde_json::{json, Value}; use std::{ + collections::BTreeSet, fs, io::Write, path::{Path, PathBuf}, @@ -94,6 +95,18 @@ fn file_url(path: &Path) -> String { format!("file://{}", path.display()) } +fn read_analytics_events(path: &Path) -> Vec { + fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +fn analytics_event_properties(event: &Value) -> &serde_json::Map { + event["events"][0]["properties"].as_object().unwrap() +} + fn sha256_hex(bytes: &[u8]) -> String { use sha2::{Digest, Sha256}; let digest = Sha256::digest(bytes); @@ -1659,20 +1672,37 @@ fn provider_session_lookup_requires_explicit_provider_flags_in_help() { fn analytics_sends_coarse_cli_metadata_when_enabled() { let temp = tempdir(); let events_path = temp.path().join("analytics.jsonl"); + let home = temp.path().join("home"); + let state = temp.path().join("state"); + let data_root = temp.path().join("data"); + fs::create_dir_all(&home).unwrap(); ctx(&temp) .arg("status") + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) .env_remove("CTX_ANALYTICS_OFF") .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) .assert() .success(); - let body = fs::read_to_string(&events_path).unwrap(); - let event: Value = serde_json::from_str(body.lines().next().unwrap()).unwrap(); + let event = read_analytics_events(&events_path).remove(0); assert_eq!(event["broker_runtime"], "cli"); + assert!(uuid::Uuid::parse_str(event["broker_install_id"].as_str().unwrap()).is_ok()); + assert!(uuid::Uuid::parse_str(event["broker_device_id"].as_str().unwrap()).is_ok()); assert_eq!(event["events"][0]["event_name"], "cli_invocation"); assert_eq!(event["events"][0]["origin_runtime"], "cli"); assert_eq!(event["events"][0]["surface"], "cli"); + assert_eq!( + event["events"][0]["origin_install_id"], + event["broker_install_id"] + ); + assert_eq!( + event["events"][0]["origin_device_id"], + event["broker_device_id"] + ); assert_eq!(event["events"][0]["properties"]["action"], "status"); assert_eq!( event["events"][0]["properties"]["analytics_client"], @@ -1687,6 +1717,16 @@ fn analytics_sends_coarse_cli_metadata_when_enabled() { event["events"][0]["properties"]["cataloged_sessions_bucket"], "0" ); + assert_eq!( + event["events"][0]["properties"]["indexed_sessions_bucket"], + "0" + ); + assert_eq!( + event["events"][0]["properties"]["indexed_events_bucket"], + "0" + ); + assert_eq!(event["events"][0]["properties"]["db_size_bucket"], "0"); + assert_analytics_properties_are_allowlisted(analytics_event_properties(&event)); for forbidden in [ "command", "query", @@ -1708,9 +1748,196 @@ fn analytics_sends_coarse_cli_metadata_when_enabled() { } } +#[test] +fn analytics_device_id_persists_across_data_roots() { + let temp = tempdir(); + let home = temp.path().join("home"); + let state = temp.path().join("state"); + let data_root_a = temp.path().join("data-a"); + let data_root_b = temp.path().join("data-b"); + let events_path = temp.path().join("analytics.jsonl"); + fs::create_dir_all(&home).unwrap(); + + for data_root in [&data_root_a, &data_root_b] { + ctx(&temp) + .arg("status") + .env("CTX_DATA_ROOT", data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .success(); + } + + let events = read_analytics_events(&events_path); + assert_eq!(events.len(), 2); + let install_a = events[0]["broker_install_id"].as_str().unwrap(); + let install_b = events[1]["broker_install_id"].as_str().unwrap(); + let device_a = events[0]["broker_device_id"].as_str().unwrap(); + let device_b = events[1]["broker_device_id"].as_str().unwrap(); + assert_ne!(install_a, install_b); + assert_eq!(device_a, device_b); + assert!(uuid::Uuid::parse_str(install_a).is_ok()); + assert!(uuid::Uuid::parse_str(install_b).is_ok()); + assert!(uuid::Uuid::parse_str(device_a).is_ok()); + + assert!(data_root_a.join("install.json").exists()); + assert!(data_root_b.join("install.json").exists()); + let device_path = expected_device_path(&home, &state); + assert!(device_path.exists()); + assert!(!device_path.starts_with(&data_root_a)); + assert!(!device_path.starts_with(&data_root_b)); + let device_json: Value = serde_json::from_slice(&fs::read(&device_path).unwrap()).unwrap(); + assert_eq!(device_json["schema_version"], 1); + assert_eq!(device_json["device_id"], device_a); + let device_body = serde_json::to_string(&device_json).unwrap(); + assert!(!device_body.contains(home.to_str().unwrap())); + assert!(!device_body.contains(data_root_a.to_str().unwrap())); + assert!(!device_body.contains(data_root_b.to_str().unwrap())); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mode = fs::metadata(device_path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + } +} + +#[test] +fn analytics_payloads_omit_sensitive_command_data() { + let temp = tempdir(); + let home = temp.path().join("alice-secret-home"); + let state = temp.path().join("state"); + let data_root = temp.path().join("ctx-data"); + let events_path = temp.path().join("analytics.jsonl"); + fs::create_dir_all(&home).unwrap(); + let private_query = + "prompt text /home/alice/private/acme-secret repo@example.com host.internal 192.0.2.44"; + + ctx(&temp) + .args([ + "search", + private_query, + "--workspace", + "acme-secret-repo", + "--refresh", + "off", + ]) + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .success(); + + ctx(&temp) + .args(["docs", "search", "private prompt text", "--limit", "1"]) + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .success(); + + ctx(&temp) + .args(["upgrade", "status"]) + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .success(); + + ctx(&temp) + .args(["show", "session", "not-a-uuid-secret"]) + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .failure(); + + let events = read_analytics_events(&events_path); + assert_eq!(events.len(), 4); + let actions = events + .iter() + .map(|event| { + event["events"][0]["properties"]["action"] + .as_str() + .unwrap() + .to_owned() + }) + .collect::>(); + assert_eq!(actions, ["search", "docs", "upgrade", "show"]); + + let search_properties = analytics_event_properties(&events[0]); + assert_eq!(search_properties["query_length_bucket"], "21-100"); + assert_eq!(search_properties["query_term_count_bucket"], "6-20"); + assert_eq!(search_properties["search_refresh_mode"], "off"); + assert_eq!(search_properties["search_refresh_status"], "skipped"); + assert_eq!(search_properties["zero_result"], true); + assert!(search_properties.get("query_duration_bucket").is_some()); + assert!(search_properties.get("render_duration_bucket").is_some()); + assert_eq!(events[3]["events"][0]["success"], false); + assert_eq!( + events[3]["events"][0]["properties"]["failure_kind"], + "command_error" + ); + + for event in &events { + assert_analytics_properties_are_allowlisted(analytics_event_properties(event)); + assert_no_json_string_contains( + event, + &[ + private_query, + "private prompt text", + "not-a-uuid-secret", + "acme-secret-repo", + "/home/alice/private", + "repo@example.com", + "host.internal", + "192.0.2.44", + home.to_str().unwrap(), + ], + ); + let properties = analytics_event_properties(event); + for forbidden_key in [ + "install_id", + "origin_install_id", + "broker_install_id", + "device_id", + "origin_device_id", + "broker_device_id", + "hostname", + "username", + "repo_name", + "file_path", + "prompt", + "transcript", + ] { + assert!( + properties.get(forbidden_key).is_none(), + "analytics leaked forbidden property {forbidden_key}: {event:#}" + ); + } + } +} + #[test] fn analytics_config_opt_out_suppresses_delivery() { let temp = tempdir(); + let state = temp.path().join("state"); fs::write( temp.path().join("config.toml"), "[analytics]\nenabled = false\n", @@ -1720,6 +1947,8 @@ fn analytics_config_opt_out_suppresses_delivery() { ctx(&temp) .arg("status") + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) .env_remove("CTX_ANALYTICS_OFF") .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) .assert() @@ -1729,6 +1958,184 @@ fn analytics_config_opt_out_suppresses_delivery() { !events_path.exists(), "analytics endpoint should not be touched" ); + assert!( + !temp.path().join("install.json").exists(), + "disabled analytics should not create an install identity" + ); + assert!( + !expected_device_path(temp.path(), &state).exists(), + "disabled analytics should not create a device identity" + ); +} + +#[test] +fn analytics_env_opt_out_wins_over_enable_flag() { + let temp = tempdir(); + let state = temp.path().join("state"); + let events_path = temp.path().join("analytics.jsonl"); + + ctx(&temp) + .arg("status") + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env("CTX_ANALYTICS_OFF", "1") + .env("CTX_ANALYTICS_ENABLED", "true") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .success(); + + assert!( + !events_path.exists(), + "CTX_ANALYTICS_OFF should be a hard process opt-out" + ); + assert!( + !expected_device_path(temp.path(), &state).exists(), + "hard opt-out should not create a device identity" + ); +} + +#[test] +fn analytics_refuses_device_identity_under_data_root() { + let temp = tempdir(); + let data_root = temp.path().join("ctx-data"); + let state = data_root.join("state"); + let events_path = temp.path().join("analytics.jsonl"); + + ctx(&temp) + .arg("status") + .env("CTX_DATA_ROOT", &data_root) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .success(); + + assert!( + !events_path.exists(), + "device identity under data root should fail closed before delivery" + ); + assert!( + !state.join("ctx").join("device.json").exists(), + "device identity must not be created under CTX_DATA_ROOT" + ); +} + +fn expected_device_path(_home: &Path, state: &Path) -> PathBuf { + #[cfg(target_os = "windows")] + { + state.join("ctx").join("device.json") + } + #[cfg(target_os = "macos")] + { + _home + .join("Library") + .join("Application Support") + .join("ctx") + .join("device.json") + } + #[cfg(all(not(target_os = "windows"), not(target_os = "macos")))] + { + state.join("ctx").join("device.json") + } +} + +fn assert_no_json_string_contains(value: &Value, forbidden: &[&str]) { + match value { + Value::String(text) => { + for needle in forbidden { + assert!( + !text.contains(needle), + "analytics leaked forbidden string {needle:?} in {text:?}" + ); + } + } + Value::Array(values) => { + for value in values { + assert_no_json_string_contains(value, forbidden); + } + } + Value::Object(values) => { + for value in values.values() { + assert_no_json_string_contains(value, forbidden); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } +} + +fn assert_analytics_properties_are_allowlisted(properties: &serde_json::Map) { + let allowed = [ + "action", + "all_sources", + "analytics_client", + "available_sources_bucket", + "background", + "catalog_only", + "catalog_source_bytes_bucket", + "cataloged_sessions_bucket", + "citation_count_bucket", + "db_size_bucket", + "dry_run", + "edges_imported_bucket", + "event_results", + "failed_bucket", + "failed_sources_bucket", + "failure_kind", + "finding_count_bucket", + "has_event_type_filter", + "has_file_filter", + "has_provider_filter", + "has_query", + "has_session_filter", + "has_since_filter", + "has_workspace_filter", + "include_current_session", + "include_subagents", + "indexed_events_bucket", + "indexed_items_bucket", + "indexed_sessions_bucket", + "indexed_sources_bucket", + "initialized", + "json_output", + "limit_bucket", + "native_sources_bucket", + "output_format", + "pending_sessions_bucket", + "primary_only", + "progress_mode", + "provider_filter", + "provider_lookup", + "providers_detected_bucket", + "query_duration_bucket", + "query_length_bucket", + "query_term_count_bucket", + "refresh_duration_bucket", + "render_duration_bucket", + "result_count_bucket", + "resume", + "search_refresh_mode", + "search_refresh_source_count_bucket", + "search_refresh_status", + "sessions_imported_bucket", + "skipped_bucket", + "source_files_bucket", + "source_mode", + "target_kind", + "transcript_mode", + "window_bucket", + "writes_out_file", + "zero_result", + ] + .into_iter() + .collect::>(); + + for key in properties.keys() { + assert!( + allowed.contains(key.as_str()), + "unexpected analytics property {key}: {properties:#?}" + ); + } } #[test] diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index b9eb28abb..f19f71b35 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -279,6 +279,18 @@ pub struct CatalogCounts { pub failed: usize, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct IndexedHistoryCounts { + pub sessions: usize, + pub events: usize, +} + +impl IndexedHistoryCounts { + pub fn items(self) -> usize { + self.sessions.saturating_add(self.events) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CatalogIndexedStatus { Pending, @@ -2287,13 +2299,20 @@ impl Store { } pub fn indexed_history_item_count(&self) -> Result { + Ok(self.indexed_history_counts()?.items()) + } + + pub fn indexed_history_counts(&self) -> Result { let sessions: i64 = self .conn .query_row("SELECT COUNT(*) FROM sessions", [], |row| row.get(0))?; let events: i64 = self .conn .query_row("SELECT COUNT(*) FROM events", [], |row| row.get(0))?; - Ok((sessions as usize).saturating_add(events as usize)) + Ok(IndexedHistoryCounts { + sessions: sessions as usize, + events: events as usize, + }) } pub fn upsert_session_edge(&self, edge: &SessionEdge) -> Result<()> { diff --git a/docs/storage.md b/docs/storage.md index 00b99052a..bd59353eb 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -234,23 +234,32 @@ opt-outs such as `CTX_UPGRADE_OFF=1` or `CTX_DISABLE_AUTO_UPGRADE=1`. Upgrade metadata checks do not send provider transcript text, search queries, result snippets, source paths, repository names, or command output. -First-party analytics are default-on and may create `install.json` and send -coarse product metadata. They do not send session text, prompts, transcripts, -search queries, result snippets, source paths, repository or branch names, -native session IDs, command text, command output, or raw IP addresses. +First-party analytics are default-on and may create `install.json` plus a +separate device identity file in OS user state, then send coarse product +metadata. They do not send session text, prompts, transcripts, search queries, +result snippets, source paths, repository or branch names, native session IDs, +command text, command output, usernames, hostnames, raw IP addresses, or +hardware-derived machine fingerprints. Analytics may include: -- a generated install identifier that is hashed server-side; +- generated random install and device identifiers that are hashed server-side; - ctx version, OS, architecture, command name, success state, and duration bucket; - JSON-output and option booleans such as whether a search used filters; - bucketed counts such as indexed sessions, import totals, result counts, and validation finding counts; +- bucketed search query length and term count, but not query content; - provider identifiers such as `codex` or `claude` when selected as filters; - coarse Cloudflare-derived geography such as country, region, colo, ASN, and AS organization. +The install identifier lives in `install.json` under the configured ctx data +root and represents that local index. The device identifier is a random UUID +created only when analytics are enabled and an event is sent; it lives outside +the ctx data root in OS user state, such as `$XDG_STATE_HOME/ctx/device.json` or +`~/.local/state/ctx/device.json` on Linux. + `ctx sql` and MCP do not send first-party analytics events. To disable analytics, add: From c3bffbdc2a9cba40ce93a15a7725027e08678bc4 Mon Sep 17 00:00:00 2001 From: Luca King Date: Wed, 1 Jul 2026 13:54:05 -0500 Subject: [PATCH 20/72] Harden ctx release and archive coverage Adds focused public ctx coverage for signed release metadata parsing, endpoint validation, provider-session exclusion, and archive/blob fail-closed validation. --- crates/ctx-cli/src/net.rs | 60 ++++++++++- crates/ctx-cli/src/upgrade.rs | 46 +++++--- crates/ctx-cli/tests/cli.rs | 45 ++++++++ crates/ctx-history-search/src/lib.rs | 126 ++++++++++++++++++++++ crates/ctx-history-store/src/lib.rs | 154 +++++++++++++++++++++++++++ 5 files changed, 413 insertions(+), 18 deletions(-) diff --git a/crates/ctx-cli/src/net.rs b/crates/ctx-cli/src/net.rs index 3fad54aa5..e4eeb0c96 100644 --- a/crates/ctx-cli/src/net.rs +++ b/crates/ctx-cli/src/net.rs @@ -8,7 +8,7 @@ use std::{ use anyhow::{anyhow, Context, Result}; pub fn post_json(endpoint: &str, body: &[u8]) -> Result<()> { - if let Some(path) = file_url_path(endpoint) { + if let Some(path) = file_url_path(endpoint)? { let mut file = OpenOptions::new() .create(true) .append(true) @@ -28,7 +28,7 @@ pub fn post_json(endpoint: &str, body: &[u8]) -> Result<()> { } pub fn get_bytes(endpoint: &str) -> Result> { - if let Some(path) = file_url_path(endpoint) { + if let Some(path) = file_url_path(endpoint)? { return fs::read(&path).with_context(|| format!("read {}", path.display())); } require_https_or_localhost(endpoint)?; @@ -44,8 +44,14 @@ pub fn get_bytes(endpoint: &str) -> Result> { Ok(bytes) } -fn file_url_path(url: &str) -> Option { - url.strip_prefix("file://").map(PathBuf::from) +fn file_url_path(url: &str) -> Result> { + let Some(path) = url.strip_prefix("file://") else { + return Ok(None); + }; + if path.is_empty() || !path.starts_with('/') { + return Err(anyhow!("file URL must use an absolute local path: {url}")); + } + Ok(Some(PathBuf::from(path))) } fn require_https_or_localhost(url: &str) -> Result<()> { @@ -54,9 +60,53 @@ fn require_https_or_localhost(url: &str) -> Result<()> { } if let Some(rest) = url.strip_prefix("http://") { let host = rest.split('/').next().unwrap_or_default(); - if matches!(host, "localhost" | "127.0.0.1" | "[::1]") { + if is_localhost_authority(host) { return Ok(()); } } Err(anyhow!("refusing non-HTTPS endpoint: {url}")) } + +fn is_localhost_authority(authority: &str) -> bool { + if authority.contains('@') { + return false; + } + let host = if let Some(rest) = authority.strip_prefix("[::1]") { + if rest.is_empty() || rest.starts_with(':') { + "[::1]" + } else { + return false; + } + } else { + authority.split(':').next().unwrap_or_default() + }; + matches!(host, "localhost" | "127.0.0.1" | "[::1]") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_urls_must_be_absolute_local_paths() { + assert_eq!( + file_url_path("file:///tmp/ctx-release-metadata.env") + .unwrap() + .unwrap(), + PathBuf::from("/tmp/ctx-release-metadata.env") + ); + assert!(file_url_path("file://relative/path").is_err()); + assert!(file_url_path("file://").is_err()); + assert!(file_url_path("https://example.com").unwrap().is_none()); + } + + #[test] + fn endpoint_validation_allows_https_and_localhost_http_only() { + require_https_or_localhost("https://example.com/releases").unwrap(); + require_https_or_localhost("http://localhost:8080/events").unwrap(); + require_https_or_localhost("http://127.0.0.1/events").unwrap(); + require_https_or_localhost("http://[::1]:8080/events").unwrap(); + assert!(require_https_or_localhost("http://example.com/events").is_err()); + assert!(require_https_or_localhost("http://example.com@localhost/events").is_err()); + } +} diff --git a/crates/ctx-cli/src/upgrade.rs b/crates/ctx-cli/src/upgrade.rs index 33b1e1590..37ed6f18c 100644 --- a/crates/ctx-cli/src/upgrade.rs +++ b/crates/ctx-cli/src/upgrade.rs @@ -1,4 +1,5 @@ use std::{ + collections::BTreeMap, env, fs, io::Write, path::{Path, PathBuf}, @@ -567,7 +568,8 @@ fn parse_release_metadata( expected_channel: &str, ) -> Result { let text = std::str::from_utf8(bytes).context("release metadata is not UTF-8")?; - let value = |key: &str| metadata_value(text, key); + let metadata = parse_metadata_map(text)?; + let value = |key: &str| metadata_value(&metadata, key); let schema = value("CTX_RELEASE_SCHEMA_VERSION") .ok_or_else(|| anyhow!("metadata missing CTX_RELEASE_SCHEMA_VERSION"))?; if schema != "1" { @@ -596,27 +598,45 @@ fn parse_release_metadata( sha256, source_commit: value("CTX_RELEASE_SOURCE_COMMIT"), published_at: value("CTX_RELEASE_PUBLISHED_AT"), - self_upgrade_allowed: metadata_bool(text, "CTX_RELEASE_SELF_UPGRADE_ALLOWED", false), - auto_upgrade_allowed: metadata_bool(text, "CTX_RELEASE_AUTO_UPGRADE_ALLOWED", false), + self_upgrade_allowed: metadata_bool(&metadata, "CTX_RELEASE_SELF_UPGRADE_ALLOWED", false)?, + auto_upgrade_allowed: metadata_bool(&metadata, "CTX_RELEASE_AUTO_UPGRADE_ALLOWED", false)?, store_schema_version: value("CTX_RELEASE_STORE_SCHEMA_VERSION"), }) } -fn metadata_value(text: &str, key: &str) -> Option { - text.lines().find_map(|line| { +fn parse_metadata_map(text: &str) -> Result> { + let mut metadata = BTreeMap::new(); + for line in text.lines() { let line = line.trim(); if line.starts_with('#') || line.is_empty() { - return None; + continue; } - let (candidate, value) = line.split_once('=')?; - (candidate == key).then(|| value.trim_end_matches('\r').to_owned()) - }) + let Some((key, value)) = line.split_once('=') else { + continue; + }; + if metadata + .insert(key.to_owned(), value.trim_end_matches('\r').to_owned()) + .is_some() + { + return Err(anyhow!("metadata contains duplicate key {key}")); + } + } + Ok(metadata) } -fn metadata_bool(text: &str, key: &str, default: bool) -> bool { - metadata_value(text, key).map_or(default, |value| { - matches!(value.to_ascii_lowercase().as_str(), "1" | "true" | "yes") - }) +fn metadata_value(metadata: &BTreeMap, key: &str) -> Option { + metadata.get(key).cloned() +} + +fn metadata_bool(metadata: &BTreeMap, key: &str, default: bool) -> Result { + let Some(value) = metadata_value(metadata, key) else { + return Ok(default); + }; + match value.to_ascii_lowercase().as_str() { + "1" | "true" | "yes" => Ok(true), + "0" | "false" | "no" => Ok(false), + _ => Err(anyhow!("metadata {key} must be a boolean")), + } } fn verify_metadata_signature(metadata: &[u8], signature: &[u8]) -> Result<()> { diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index eb7246ca0..43d850c3e 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1521,11 +1521,56 @@ fn upgrade_verifies_signed_metadata_and_fails_closed() { stderr.contains("download release metadata signature"), "{stderr}" ); + + let default_signature_path = tempdir(); + let release = fake_release(&default_signature_path, "9.9.9"); + let check = json_output( + ctx(&default_signature_path) + .args(["upgrade", "check", "--json"]) + .env("CTX_UPGRADE_TARGET", &release.target) + .env("CTX_RELEASE_METADATA_URL", file_url(&release.metadata)) + .env( + "CTX_RELEASE_METADATA_PUBLIC_KEY_PEM", + TEST_RELEASE_PUBLIC_KEY_PEM, + ), + ); + assert_eq!(check["status"], "available"); } #[cfg(unix)] #[test] fn upgrade_rejects_unsafe_metadata_and_bad_artifacts() { + let duplicate_key = tempdir(); + let release = fake_release(&duplicate_key, "9.9.9"); + rewrite_fake_release_metadata(&release, |metadata| { + format!("{metadata}CTX_RELEASE_VERSION=8.8.8\n") + }); + let stderr = failure_stderr(fake_release_env( + ctx(&duplicate_key).args(["upgrade", "check"]), + &release, + )); + assert!( + stderr.contains("metadata contains duplicate key CTX_RELEASE_VERSION"), + "{stderr}" + ); + + let malformed_bool = tempdir(); + let release = fake_release(&malformed_bool, "9.9.9"); + rewrite_fake_release_metadata(&release, |metadata| { + metadata.replace( + "CTX_RELEASE_SELF_UPGRADE_ALLOWED=true\n", + "CTX_RELEASE_SELF_UPGRADE_ALLOWED=definitely\n", + ) + }); + let stderr = failure_stderr(fake_release_env( + ctx(&malformed_bool).args(["upgrade", "check"]), + &release, + )); + assert!( + stderr.contains("metadata CTX_RELEASE_SELF_UPGRADE_ALLOWED must be a boolean"), + "{stderr}" + ); + let missing_policy = tempdir(); let release = fake_release(&missing_policy, "9.9.9"); rewrite_fake_release_metadata(&release, |metadata| { diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index 9afd0563c..a932e08fb 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -2275,6 +2275,17 @@ mod tests { } } + fn excluded_filter(session_id: Option) -> SearchFilters { + SearchFilters { + exclude_provider_session: Some(ProviderSessionFilter { + provider: CaptureProvider::Codex, + provider_session_id: "provider-session-1".into(), + session_id, + }), + ..SearchFilters::default() + } + } + fn test_store() -> (tempfile::TempDir, ctx_history_store::Store) { let temp = tempdir(); let path = temp.path().join("work.sqlite"); @@ -2318,6 +2329,121 @@ mod tests { assert!(!preview.contains("secret payload")); } + #[test] + fn excluded_provider_session_matches_provider_external_id_for_hits() { + let filters = excluded_filter(None); + let hit = HitMetadata { + provider: Some(CaptureProvider::Codex), + provider_session_id: Some("provider-session-1".into()), + ..empty_hit(fixed_time()) + }; + assert!(hit_matches_excluded_provider_session(&hit, &filters)); + + let event_hit = EventSearchHit { + event_id: Uuid::parse_str("018f45d0-0000-7000-8000-000000001001").unwrap(), + history_record_id: None, + session_id: None, + session_parent_session_id: None, + session_root_session_id: None, + run_id: None, + seq: 1, + event_type: EventType::Message, + role: Some(EventRole::User), + occurred_at: fixed_time(), + preview: "synthetic preview".into(), + score: 1.0, + provider: Some(CaptureProvider::Codex), + session_external_session_id: Some("provider-session-1".into()), + agent_type: Some(AgentType::Primary), + session_is_primary: Some(true), + cwd: None, + raw_source_path: None, + cursor: None, + record_title: None, + record_kind: None, + record_workspace: None, + }; + assert!(event_hit_matches_excluded_provider_session( + &event_hit, &filters + )); + + let mut different_provider = event_hit; + different_provider.provider = Some(CaptureProvider::Claude); + assert!(!event_hit_matches_excluded_provider_session( + &different_provider, + &filters + )); + } + + #[test] + fn excluded_provider_session_matches_parent_and_root_session_tree() { + let excluded_session_id = Uuid::parse_str("018f45d0-0000-7000-8000-000000001100").unwrap(); + let child_session_id = Uuid::parse_str("018f45d0-0000-7000-8000-000000001101").unwrap(); + let grandchild_session_id = + Uuid::parse_str("018f45d0-0000-7000-8000-000000001102").unwrap(); + let filters = excluded_filter(Some(excluded_session_id)); + + let parent_hit = HitMetadata { + session_id: Some(child_session_id), + parent_session_id: Some(excluded_session_id), + ..empty_hit(fixed_time()) + }; + assert!(hit_matches_excluded_provider_session(&parent_hit, &filters)); + + let root_event_hit = EventSearchHit { + event_id: Uuid::parse_str("018f45d0-0000-7000-8000-000000001103").unwrap(), + history_record_id: None, + session_id: Some(grandchild_session_id), + session_parent_session_id: Some(child_session_id), + session_root_session_id: Some(excluded_session_id), + run_id: None, + seq: 1, + event_type: EventType::Message, + role: Some(EventRole::Assistant), + occurred_at: fixed_time(), + preview: "synthetic preview".into(), + score: 1.0, + provider: None, + session_external_session_id: None, + agent_type: Some(AgentType::Subagent), + session_is_primary: Some(false), + cwd: None, + raw_source_path: None, + cursor: None, + record_title: None, + record_kind: None, + record_workspace: None, + }; + assert!(event_hit_matches_excluded_provider_session( + &root_event_hit, + &filters + )); + + let context = RecordContext { + sessions: vec![Session { + id: grandchild_session_id, + history_record_id: None, + parent_session_id: Some(child_session_id), + root_session_id: Some(excluded_session_id), + capture_source_id: None, + provider: CaptureProvider::Claude, + external_session_id: Some("different-provider-session".into()), + external_agent_id: None, + agent_type: AgentType::Subagent, + role_hint: None, + is_primary: false, + status: SessionStatus::Imported, + transcript_blob_id: None, + started_at: fixed_time(), + ended_at: None, + timestamps: timestamps(), + sync: sync_metadata(), + }], + ..RecordContext::default() + }; + assert!(context_has_excluded_provider_session(&context, &filters)); + } + #[test] fn rich_search_matches_typed_metadata_with_citations_and_redaction() { let (_temp, store) = test_store(); diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index f19f71b35..9b1dfdb20 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -5908,6 +5908,160 @@ fn existing_history_record_link_by_identity( .map_err(StoreError::from) } +#[cfg(test)] +mod archive_validation_tests { + use super::*; + + fn tempdir() -> tempfile::TempDir { + let root = std::env::current_dir().unwrap().join("target/test-data"); + fs::create_dir_all(&root).unwrap(); + tempfile::Builder::new() + .prefix("ctx-history-store-archive-validation-") + .tempdir_in(root) + .unwrap() + } + + fn fixed_time() -> DateTime { + DateTime::parse_from_rfc3339("2026-06-23T12:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn artifact(id: Uuid, blob_hash: String, byte_size: u64) -> Artifact { + Artifact { + id, + kind: ArtifactKind::Markdown, + blob_path: object_relative_path(&blob_hash), + blob_hash, + byte_size, + media_type: Some("text/markdown".into()), + preview_text: Some("synthetic public-safe blob".into()), + redaction_state: RedactionState::SafePreview, + timestamps: EntityTimestamps { + created_at: fixed_time(), + updated_at: fixed_time(), + }, + source_id: None, + sync: SyncMetadata { + visibility: Visibility::LocalOnly, + fidelity: Fidelity::Imported, + sync_state: SyncState::LocalOnly, + sync_version: 0, + deleted_at: None, + metadata: serde_json::json!({}), + }, + } + } + + fn write_blob(blob_dir: &Path, blob_hash: &str, content: &[u8]) { + let path = blob_dir.join(&blob_hash[..2]).join(blob_hash); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); + } + + fn assert_artifact_error( + error: StoreError, + matches_expected: impl FnOnce(&StoreError) -> bool, + ) { + assert!( + matches_expected(&error), + "unexpected archive artifact validation error: {error:?}" + ); + } + + #[test] + fn archive_blob_validation_fails_closed_when_blob_is_missing() { + let temp = tempdir(); + let content = b"missing synthetic blob"; + let artifact = artifact(new_id(), sha256_hex(content), content.len() as u64); + + let error = validate_archive_artifact_record_blob(temp.path(), &artifact).unwrap_err(); + assert_artifact_error( + error, + |error| matches!(error, StoreError::ArchiveArtifactMissingContent { id } if *id == artifact.id), + ); + } + + #[test] + fn archive_blob_validation_fails_closed_when_hash_differs() { + let temp = tempdir(); + let stored_content = b"stored bytes"; + let expected_content = b"expected bytes"; + let artifact = artifact( + new_id(), + sha256_hex(expected_content), + stored_content.len() as u64, + ); + write_blob(temp.path(), &artifact.blob_hash, stored_content); + + let error = validate_archive_artifact_record_blob(temp.path(), &artifact).unwrap_err(); + assert_artifact_error( + error, + |error| matches!(error, StoreError::ArchiveArtifactHashMismatch { id } if *id == artifact.id), + ); + } + + #[test] + fn archive_blob_validation_fails_closed_when_byte_size_differs() { + let temp = tempdir(); + let content = b"size checked bytes"; + let artifact = artifact(new_id(), sha256_hex(content), content.len() as u64 + 1); + write_blob(temp.path(), &artifact.blob_hash, content); + + let error = validate_archive_artifact_record_blob(temp.path(), &artifact).unwrap_err(); + assert_artifact_error( + error, + |error| matches!(error, StoreError::ArchiveArtifactSizeMismatch { id } if *id == artifact.id), + ); + } + + #[test] + fn archive_blob_validation_fails_closed_when_blob_path_mismatches_hash() { + let temp = tempdir(); + let content = b"path checked bytes"; + let mut artifact = artifact(new_id(), sha256_hex(content), content.len() as u64); + artifact.blob_path = "objects/ff/not-the-recorded-hash".into(); + write_blob(temp.path(), &artifact.blob_hash, content); + + let error = validate_archive_artifact_record_blob(temp.path(), &artifact).unwrap_err(); + assert_artifact_error( + error, + |error| matches!(error, StoreError::ArchiveArtifactPathMismatch { id } if *id == artifact.id), + ); + } + + #[test] + fn archive_blob_validation_fails_closed_when_blob_is_not_regular_file() { + let temp = tempdir(); + let content = b"directory at blob path"; + let artifact = artifact(new_id(), sha256_hex(content), content.len() as u64); + let path = temp + .path() + .join(&artifact.blob_hash[..2]) + .join(&artifact.blob_hash); + fs::create_dir_all(&path).unwrap(); + + let error = validate_archive_artifact_record_blob(temp.path(), &artifact).unwrap_err(); + assert_artifact_error( + error, + |error| matches!(error, StoreError::ArchiveArtifactNonRegularFile { id, .. } if *id == artifact.id), + ); + } + + #[test] + fn archive_version_validation_rejects_future_version() { + let mut archive = SessionHistoryArchive::default(); + archive.schema_version = 3; + archive.version = 3; + + let error = validate_archive_version(&archive).unwrap_err(); + assert!(matches!( + error, + StoreError::UnsupportedArchiveVersion(version) if version == 3 + )); + } +} + fn expected_archive_blob_path(id: Uuid, blob_hash: &str) -> Result { if blob_hash.get(..2).is_none() { return Err(StoreError::ArchiveArtifactPathMismatch { id }); From b0d938aa45cd3375548f28029ca98247d5a26a4e Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 14:16:17 -0500 Subject: [PATCH 21/72] release: bump ctx to 0.14.0 --- Cargo.lock | 10 +++++----- crates/ctx-cli/Cargo.toml | 2 +- crates/ctx-history-capture/Cargo.toml | 2 +- crates/ctx-history-core/Cargo.toml | 2 +- crates/ctx-history-search/Cargo.toml | 2 +- crates/ctx-history-store/Cargo.toml | 2 +- crates/ctx-history-store/src/lib.rs | 8 +++++--- scripts/build-public-cli-artifact.sh | 10 +++++----- 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 641cb5f94..906814d59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,7 +256,7 @@ dependencies = [ [[package]] name = "ctx" -version = "0.13.0" +version = "0.14.0" dependencies = [ "anyhow", "assert_cmd", @@ -281,7 +281,7 @@ dependencies = [ [[package]] name = "ctx-history-capture" -version = "0.13.0" +version = "0.14.0" dependencies = [ "chrono", "ctx-history-core", @@ -296,7 +296,7 @@ dependencies = [ [[package]] name = "ctx-history-core" -version = "0.13.0" +version = "0.14.0" dependencies = [ "chrono", "directories", @@ -309,7 +309,7 @@ dependencies = [ [[package]] name = "ctx-history-search" -version = "0.13.0" +version = "0.14.0" dependencies = [ "chrono", "ctx-history-core", @@ -324,7 +324,7 @@ dependencies = [ [[package]] name = "ctx-history-store" -version = "0.13.0" +version = "0.14.0" dependencies = [ "chrono", "ctx-history-core", diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index d91a6463b..996d0ad65 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx" -version = "0.13.0" +version = "0.14.0" description = "Local CLI for indexing and searching agent session history" edition.workspace = true autobins = false diff --git a/crates/ctx-history-capture/Cargo.toml b/crates/ctx-history-capture/Cargo.toml index 4b5b658b2..9c5967f2d 100644 --- a/crates/ctx-history-capture/Cargo.toml +++ b/crates/ctx-history-capture/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-capture" -version = "0.13.0" +version = "0.14.0" description = "Internal provider import adapters for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-core/Cargo.toml b/crates/ctx-history-core/Cargo.toml index b241edd45..aeea64c24 100644 --- a/crates/ctx-history-core/Cargo.toml +++ b/crates/ctx-history-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-core" -version = "0.13.0" +version = "0.14.0" description = "Internal core types for ctx local agent history indexing" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-search/Cargo.toml b/crates/ctx-history-search/Cargo.toml index 782390d87..7464db474 100644 --- a/crates/ctx-history-search/Cargo.toml +++ b/crates/ctx-history-search/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-search" -version = "0.13.0" +version = "0.14.0" description = "Internal search projection and ranking helpers for ctx" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-store/Cargo.toml b/crates/ctx-history-store/Cargo.toml index 4833a31c6..cbf8f9660 100644 --- a/crates/ctx-history-store/Cargo.toml +++ b/crates/ctx-history-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-store" -version = "0.13.0" +version = "0.14.0" description = "Internal SQLite storage layer for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 9b1dfdb20..b832fadf4 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -6050,9 +6050,11 @@ mod archive_validation_tests { #[test] fn archive_version_validation_rejects_future_version() { - let mut archive = SessionHistoryArchive::default(); - archive.schema_version = 3; - archive.version = 3; + let archive = SessionHistoryArchive { + schema_version: 3, + version: 3, + ..SessionHistoryArchive::default() + }; let error = validate_archive_version(&archive).unwrap_err(); assert!(matches!( diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index 83430f465..16b66e52b 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -113,8 +113,8 @@ ensure_darwin_cross_tools() { } version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; data=json.load(sys.stdin); print(next(pkg["version"] for pkg in data["packages"] if pkg["name"] == "ctx"))')" -if [[ "${version}" != "0.13.0" ]]; then - echo "error: ctx package version must be 0.13.0 for this release, got ${version}" >&2 +if [[ "${version}" != "0.14.0" ]]; then + echo "error: ctx package version must be 0.14.0 for this release, got ${version}" >&2 exit 1 fi @@ -156,12 +156,12 @@ fi case "${platform}" in linux-x64) "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.13.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.14.0" "${staged}.version" >/dev/null ;; macos-arm64) if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.13.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.14.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi @@ -169,7 +169,7 @@ case "${platform}" in macos-x64) if [[ "$(uname -s)" == "Darwin" ]] && /usr/bin/arch -x86_64 /usr/bin/true >/dev/null 2>&1; then /usr/bin/arch -x86_64 "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.13.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.14.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi From 84bd1d249bde35ad48611b1b45662c7d549d08f6 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:07:55 -0500 Subject: [PATCH 22/72] Add native personal agent history providers --- crates/ctx-cli/src/main.rs | 110 +- crates/ctx-cli/tests/cli.rs | 720 +++++- crates/ctx-history-capture/src/lib.rs | 2066 ++++++++++++++++- .../src/provider_sources.rs | 325 ++- crates/ctx-history-core/src/lib.rs | 4 + crates/ctx-history-core/src/provider.rs | 17 +- crates/ctx-history-store/src/lib.rs | 184 +- docs/cli-reference.md | 31 +- docs/first-10-minutes.md | 30 +- docs/limitations.md | 11 +- docs/provider-support-matrix.json | 218 ++ docs/provider-support.md | 10 + docs/providers.md | 22 +- docs/search.md | 13 +- 14 files changed, 3679 insertions(+), 82 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 10091d1b0..62b3de0d4 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -27,17 +27,20 @@ use analytics::{AnalyticsEvent, AnalyticsProperties}; use config::{AppConfig, CONFIG_FILE}; use ctx_history_capture::{ catalog_codex_session_tree, discover_provider_sources, discover_provider_sources_for_provider, - import_antigravity_cli_history, import_claude_projects_jsonl_tree, import_codex_history_jsonl, - import_codex_session_jsonl, import_codex_session_jsonl_tail, import_codex_session_paths, - import_codex_session_tree, import_copilot_cli_session_events, import_cursor_native_history, - import_factory_ai_droid_sessions, import_gemini_cli_history, import_opencode_sqlite, + import_antigravity_cli_history, import_astrbot_sqlite, import_claude_projects_jsonl_tree, + import_codex_history_jsonl, import_codex_session_jsonl, import_codex_session_jsonl_tail, + import_codex_session_paths, import_codex_session_tree, import_copilot_cli_session_events, + import_cursor_native_history, import_factory_ai_droid_sessions, import_gemini_cli_history, + import_hermes_sqlite, import_nanoclaw_project, import_openclaw_history, import_opencode_sqlite, import_pi_session_jsonl, provider_source_for_path, provider_source_spec, stable_capture_uuid, - AntigravityCliImportOptions, CatalogSummary, ClaudeProjectsImportOptions, CodexEventImportMode, - CodexHistoryImportOptions, CodexSessionCatalogOptions, CodexSessionImportOptions, - CodexSessionImportProgress, CodexSessionImportProgressCallback, CodexToolOutputMode, - CopilotCliImportOptions, CursorNativeImportOptions, FactoryAiDroidImportOptions, - GeminiCliImportOptions, OpenCodeSqliteImportOptions, PiSessionImportOptions, - ProviderImportSummary, ProviderImportSupport, ProviderSource, ProviderSourceStatus, + AntigravityCliImportOptions, AstrBotSqliteImportOptions, CatalogSummary, + ClaudeProjectsImportOptions, CodexEventImportMode, CodexHistoryImportOptions, + CodexSessionCatalogOptions, CodexSessionImportOptions, CodexSessionImportProgress, + CodexSessionImportProgressCallback, CodexToolOutputMode, CopilotCliImportOptions, + CursorNativeImportOptions, FactoryAiDroidImportOptions, GeminiCliImportOptions, + HermesSqliteImportOptions, NanoClawImportOptions, OpenClawImportOptions, + OpenCodeSqliteImportOptions, PiSessionImportOptions, ProviderImportSummary, + ProviderImportSupport, ProviderSource, ProviderSourceStatus, }; use ctx_history_core::{ database_path, default_data_root, utc_now, CaptureProvider, ContextCitation, @@ -516,6 +519,13 @@ enum ProviderArg { alias = "factory_ai_droid" )] FactoryAiDroid, + #[value(name = "openclaw", alias = "open-claw", alias = "open_claw")] + OpenClaw, + Hermes, + #[value(name = "nanoclaw", alias = "nano-claw", alias = "nano_claw")] + NanoClaw, + #[value(name = "astrbot", alias = "astr-bot", alias = "astr_bot")] + AstrBot, } #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] @@ -538,6 +548,10 @@ impl ProviderArg { Self::Cursor => CaptureProvider::Cursor, Self::CopilotCli => CaptureProvider::CopilotCli, Self::FactoryAiDroid => CaptureProvider::FactoryAiDroid, + Self::OpenClaw => CaptureProvider::OpenClaw, + Self::Hermes => CaptureProvider::Hermes, + Self::NanoClaw => CaptureProvider::NanoClaw, + Self::AstrBot => CaptureProvider::AstrBot, } } @@ -552,6 +566,10 @@ impl ProviderArg { Self::Cursor => "cursor", Self::CopilotCli => "copilot-cli", Self::FactoryAiDroid => "factory-ai-droid", + Self::OpenClaw => "openclaw", + Self::Hermes => "hermes", + Self::NanoClaw => "nanoclaw", + Self::AstrBot => "astrbot", } } } @@ -1639,7 +1657,7 @@ fn run_sources(args: JsonArgs, analytics_properties: &mut AnalyticsProperties) - .iter() .filter(|source| { source.exists - && matches!(source.import_support, ProviderImportSupport::Native) + && source.import_support.is_importable() && source.status == ProviderSourceStatus::Available }) .count(); @@ -4008,7 +4026,7 @@ fn search_refresh_sources(provider: Option) -> Vec { .drain(..) .filter(|source| { source.exists - && matches!(source.import_support, ProviderImportSupport::Native) + && source.import_support.is_auto_importable() && source.status == ProviderSourceStatus::Available && source.source_format != "codex_history_jsonl" }) @@ -4179,7 +4197,7 @@ fn import_requests(args: &ImportArgs) -> Result> { .into_iter() .filter(|source| { source.exists - && matches!(source.import_support, ProviderImportSupport::Native) + && source.import_support.is_auto_importable() && source.status == ProviderSourceStatus::Available }) .collect()); @@ -4195,7 +4213,12 @@ fn import_requests(args: &ImportArgs) -> Result> { .collect::>(); if sources.is_empty() { let spec = provider_source_spec(provider); - if let Some(reason) = spec.and_then(|spec| spec.unsupported_reason) { + if spec + .is_some_and(|spec| matches!(spec.import_support, ProviderImportSupport::Unsupported)) + { + let reason = spec + .and_then(|spec| spec.unsupported_reason) + .unwrap_or("no native local-history parser is implemented"); return Err(anyhow!( "{} native import is unsupported: {reason}", provider.as_str() @@ -4215,6 +4238,7 @@ fn import_requests(args: &ImportArgs) -> Result> { fn validate_source_import_supported(source: &SourceInfo) -> Result<()> { match source.import_support { ProviderImportSupport::Native => Ok(()), + ProviderImportSupport::Preview => Ok(()), ProviderImportSupport::Unsupported => { let reason = source .unsupported_reason @@ -4376,6 +4400,50 @@ fn import_one_source_inner( }, ) .map_err(anyhow::Error::from), + CaptureProvider::OpenClaw => import_openclaw_history( + &source.path, + store, + OpenClawImportOptions { + source_path: Some(source.path.clone()), + history_record_id: Some(record_id), + allow_partial_failures: true, + ..OpenClawImportOptions::default() + }, + ) + .map_err(anyhow::Error::from), + CaptureProvider::Hermes => import_hermes_sqlite( + &source.path, + store, + HermesSqliteImportOptions { + source_path: Some(source.path.clone()), + history_record_id: Some(record_id), + allow_partial_failures: true, + ..HermesSqliteImportOptions::default() + }, + ) + .map_err(anyhow::Error::from), + CaptureProvider::NanoClaw => import_nanoclaw_project( + &source.path, + store, + NanoClawImportOptions { + source_path: Some(source.path.clone()), + history_record_id: Some(record_id), + allow_partial_failures: true, + ..NanoClawImportOptions::default() + }, + ) + .map_err(anyhow::Error::from), + CaptureProvider::AstrBot => import_astrbot_sqlite( + &source.path, + store, + AstrBotSqliteImportOptions { + source_path: Some(source.path.clone()), + history_record_id: Some(record_id), + allow_partial_failures: true, + ..AstrBotSqliteImportOptions::default() + }, + ) + .map_err(anyhow::Error::from), CaptureProvider::Gemini => import_gemini_cli_history( &source.path, store, @@ -4532,7 +4600,14 @@ fn import_manifested_source( } fn source_uses_import_file_manifest(source: &SourceInfo) -> bool { - source.source_format != "codex_session_jsonl_tree" + !matches!( + source.source_format, + "codex_session_jsonl_tree" + | "openclaw_session_jsonl_tree" + | "hermes_state_sqlite" + | "nanoclaw_project" + | "astrbot_data_v4_sqlite" + ) } fn merge_provider_import_summary( @@ -5005,9 +5080,9 @@ fn sources_json(sources: &[SourceInfo]) -> Vec { "source_format": source.source_format, "status": source.status.as_str(), "import_support": import_support_json(source.import_support), - "native_import": matches!(source.import_support, ProviderImportSupport::Native), + "native_import": source.import_support.is_auto_importable(), "importable": source.status == ProviderSourceStatus::Available - && matches!(source.import_support, ProviderImportSupport::Native), + && source.import_support.is_importable(), "raw_retention": raw_retention_json(source.raw_retention), "unsupported_reason": source.unsupported_reason, }) @@ -5018,6 +5093,7 @@ fn sources_json(sources: &[SourceInfo]) -> Vec { fn import_support_json(support: ProviderImportSupport) -> &'static str { match support { ProviderImportSupport::Native => "native", + ProviderImportSupport::Preview => "preview", ProviderImportSupport::Unsupported => "unsupported", } } diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 43d850c3e..1d2c61af9 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -808,6 +808,81 @@ fn import_all_skips_empty_gemini_source() { .all(|source| source["provider"] != "gemini")); } +#[test] +fn sources_lists_personal_agent_provider_defaults() { + let temp = tempdir(); + install_default_openclaw_fixture(&temp, "openclaw-sources-oracle"); + install_default_hermes_fixture(&temp, "hermes-sources-oracle"); + install_default_astrbot_fixture(&temp, "astrbot-sources-oracle"); + + let sources = json_output(ctx(&temp).args(["sources", "--json"])); + for (provider, source_format, import_support, native_import) in [ + ("openclaw", "openclaw_session_jsonl_tree", "native", true), + ("hermes", "hermes_state_sqlite", "native", true), + ("astrbot", "astrbot_data_v4_sqlite", "preview", false), + ] { + let source = sources["sources"] + .as_array() + .unwrap() + .iter() + .find(|source| { + source["provider"] == provider && source["source_format"] == source_format + }) + .unwrap_or_else(|| panic!("missing {provider} source in {sources:#}")); + assert_eq!(source["status"], "available"); + assert_eq!(source["import_support"], import_support); + assert_eq!(source["native_import"], native_import); + assert_eq!(source["importable"], true); + assert!(source["unsupported_reason"].is_null()); + } +} + +#[test] +fn preview_native_sources_are_listed_but_not_auto_imported() { + let temp = tempdir(); + let query = "nanoclaw-preview-auto-refresh-oracle"; + let project = PathBuf::from(write_native_nanoclaw_fixture(&temp, query)); + + let mut sources_command = ctx(&temp); + sources_command.current_dir(&project); + let sources = json_output(sources_command.args(["sources", "--json"])); + let nanoclaw = sources["sources"] + .as_array() + .unwrap() + .iter() + .find(|source| source["provider"] == "nanoclaw") + .unwrap(); + assert_eq!(nanoclaw["status"], "available"); + assert_eq!(nanoclaw["import_support"], "preview"); + assert_eq!(nanoclaw["native_import"], false); + assert_eq!(nanoclaw["importable"], true); + assert!(nanoclaw["unsupported_reason"].is_null()); + + let mut search_command = ctx(&temp); + search_command.current_dir(&project); + let search = + json_output(search_command.args(["search", query, "--provider", "nanoclaw", "--json"])); + assert_eq!(search["freshness"]["mode"], "auto"); + assert_eq!(search["freshness"]["status"], "no_sources"); + assert_eq!(search["freshness"]["source_count"], 0); + assert!(search["results"].as_array().unwrap().is_empty()); + + let imported = json_output(ctx(&temp).args([ + "import", + "--provider", + "nanoclaw", + "--path", + project.to_str().unwrap(), + "--json", + ])); + assert_eq!(imported["totals"]["failed"], 0); + assert_eq!(imported["totals"]["imported_sources"], 1); + + let search_after_import = + json_output(ctx(&temp).args(["search", query, "--provider", "nanoclaw", "--json"])); + assert_search_provider_oracle(&search_after_import, "nanoclaw", query, 1, "message"); +} + #[test] fn import_all_reports_source_failure_without_losing_successes() { let temp = tempdir(); @@ -885,6 +960,10 @@ fn provider_help_matches_implemented_importers() { "pi", "claude", "opencode", + "openclaw", + "hermes", + "nanoclaw", + "astrbot", "antigravity", "gemini", "cursor", @@ -898,7 +977,13 @@ fn provider_help_matches_implemented_importers() { #[test] fn provider_json_names_are_accepted_as_cli_filter_aliases() { let temp = tempdir(); - for provider in ["copilot_cli", "factory_ai_droid"] { + for (provider, expected) in [ + ("copilot_cli", "copilot_cli"), + ("factory_ai_droid", "factory_ai_droid"), + ("open_claw", "openclaw"), + ("nano_claw", "nanoclaw"), + ("astr_bot", "astrbot"), + ] { let search = json_output(ctx(&temp).args([ "search", "anything", @@ -908,7 +993,7 @@ fn provider_json_names_are_accepted_as_cli_filter_aliases() { "off", "--json", ])); - assert_eq!(search["filters"]["provider"], provider); + assert_eq!(search["filters"]["provider"], expected); } } @@ -1016,7 +1101,7 @@ fn public_subcommand_help_is_golden_enough_for_session_retrieval() { vec![ "Usage: ctx import", "--provider ", - "[possible values: codex, pi, claude, opencode, antigravity, gemini, cursor, copilot-cli, factory-ai-droid]", + "[possible values: codex, pi, claude, opencode, antigravity, gemini, cursor, copilot-cli, factory-ai-droid, openclaw, hermes, nanoclaw, astrbot]", "--path ", "--resume", "--json", @@ -3239,6 +3324,8 @@ fn search_refresh_auto_imports_discovered_top_provider_sources() { ), ("pi", "pi", install_default_pi_fixture), ("cursor", "cursor", install_default_cursor_fixture), + ("openclaw", "openclaw", install_default_openclaw_fixture), + ("hermes", "hermes", install_default_hermes_fixture), ] { let temp = tempdir(); let query = format!("{stored_provider}-default-refresh-oracle"); @@ -3581,6 +3668,30 @@ fn native_provider_cli_flow_imports_new_supported_provider_paths() { "factory_ai_droid_sessions_jsonl", write_native_factory_droid_fixture, ), + ( + "openclaw", + "openclaw", + "openclaw_session_jsonl_tree", + write_native_openclaw_fixture, + ), + ( + "hermes", + "hermes", + "hermes_state_sqlite", + write_native_hermes_fixture, + ), + ( + "nanoclaw", + "nanoclaw", + "nanoclaw_project", + write_native_nanoclaw_fixture, + ), + ( + "astrbot", + "astrbot", + "astrbot_data_v4_sqlite", + write_native_astrbot_fixture, + ), ] { let temp = tempdir(); let query = format!("{stored_provider}-native-cli-oracle"); @@ -3604,6 +3715,24 @@ fn native_provider_cli_flow_imports_new_supported_provider_paths() { let search = json_output(ctx(&temp).args(["search", &query, "--provider", cli_provider, "--json"])); assert_search_provider_oracle(&search, stored_provider, &query, 1, "message"); + let result = &search["results"].as_array().unwrap()[0]; + let ctx_event_id = result["ctx_event_id"].as_str().unwrap(); + let ctx_session_id = result["ctx_session_id"].as_str().unwrap(); + + let show_event = + json_output(ctx(&temp).args(["show", "event", ctx_event_id, "--format", "json"])); + assert_eq!(show_event["event"]["provider"], stored_provider); + assert!(show_event["event"]["source"]["source_format"].is_string()); + assert!(show_event["event"]["source"]["path"].is_string()); + assert!(show_event["event"]["cursor"].is_string()); + + let locate_event = + json_output(ctx(&temp).args(["locate", "event", ctx_event_id, "--json"])); + assert_eq!(locate_event["provider"], stored_provider); + assert_eq!(locate_event["ctx_session_id"], ctx_session_id); + assert!(locate_event["source"]["source_format"].is_string()); + assert!(locate_event["source"]["path"].is_string()); + assert!(locate_event["cursor"].is_string()); let status = json_output(ctx(&temp).args(["status", "--json"])); assert!(status["indexed_items"].as_u64().unwrap() >= 2); @@ -3611,6 +3740,95 @@ fn native_provider_cli_flow_imports_new_supported_provider_paths() { let doctor = json_output(ctx(&temp).args(["doctor", "--json"])); assert_eq!(doctor["ok"], true); + + let second = json_output(ctx(&temp).args([ + "import", + "--provider", + cli_provider, + "--path", + &path, + "--json", + ])); + assert_eq!(second["totals"]["failed"], 0); + assert_eq!(second["totals"]["imported_events"], 0); + } +} + +#[test] +fn personal_agent_provider_imports_are_idempotent_and_incremental() { + for (cli_provider, stored_provider, fixture, append_event) in [ + ( + "openclaw", + "openclaw", + write_native_openclaw_fixture as fn(&TempDir, &str) -> String, + append_native_openclaw_event as fn(&str, &str), + ), + ( + "hermes", + "hermes", + write_native_hermes_fixture, + append_native_hermes_event, + ), + ( + "nanoclaw", + "nanoclaw", + write_native_nanoclaw_fixture, + append_native_nanoclaw_event, + ), + ( + "astrbot", + "astrbot", + write_native_astrbot_fixture, + append_native_astrbot_event, + ), + ] { + let temp = tempdir(); + let initial_query = format!("{stored_provider}-incremental-initial-oracle"); + let incremental_query = format!("{stored_provider}-incremental-next-oracle"); + let path = fixture(&temp, &initial_query); + + let first = json_output(ctx(&temp).args([ + "import", + "--provider", + cli_provider, + "--path", + &path, + "--json", + ])); + assert_eq!(first["totals"]["failed"], 0); + assert!(first["totals"]["imported_events"].as_u64().unwrap() >= 1); + + let second = json_output(ctx(&temp).args([ + "import", + "--provider", + cli_provider, + "--path", + &path, + "--json", + ])); + assert_eq!(second["totals"]["failed"], 0); + assert_eq!(second["totals"]["imported_events"], 0); + + append_event(&path, &incremental_query); + let third = json_output(ctx(&temp).args([ + "import", + "--provider", + cli_provider, + "--path", + &path, + "--json", + ])); + assert_eq!(third["totals"]["failed"], 0); + assert!(third["totals"]["imported_events"].as_u64().unwrap() >= 1); + + let search = json_output(ctx(&temp).args([ + "search", + &incremental_query, + "--provider", + cli_provider, + "--json", + ])); + assert_search_provider_oracle(&search, stored_provider, &incremental_query, 1, "message"); } } @@ -3649,6 +3867,25 @@ fn install_default_cursor_fixture(temp: &TempDir, query: &str) { copy_dir_all(&source, &temp.path().join(".cursor").join("projects")); } +fn install_default_openclaw_fixture(temp: &TempDir, query: &str) { + let source = PathBuf::from(write_native_openclaw_fixture(temp, query)); + copy_dir_all(&source, &temp.path().join(".openclaw")); +} + +fn install_default_hermes_fixture(temp: &TempDir, query: &str) { + let source = PathBuf::from(write_native_hermes_fixture(temp, query)); + let target = temp.path().join(".hermes"); + fs::create_dir_all(&target).unwrap(); + fs::copy(source, target.join("state.db")).unwrap(); +} + +fn install_default_astrbot_fixture(temp: &TempDir, query: &str) { + let source = PathBuf::from(write_native_astrbot_fixture(temp, query)); + let target = temp.path().join(".astrbot/data"); + fs::create_dir_all(&target).unwrap(); + fs::copy(source, target.join("data_v4.db")).unwrap(); +} + fn write_native_claude_fixture(temp: &TempDir, query: &str) -> String { let root = temp.path().join("native-claude/projects/-workspace"); fs::create_dir_all(&root).unwrap(); @@ -3872,6 +4109,479 @@ fn write_native_factory_droid_fixture(temp: &TempDir, query: &str) -> String { .to_owned() } +fn write_native_openclaw_fixture(temp: &TempDir, query: &str) -> String { + let root = temp.path().join("native-openclaw"); + let sessions = root.join("agents/personal-agent/sessions"); + fs::create_dir_all(&sessions).unwrap(); + fs::write( + sessions.join("sessions.json"), + serde_json::to_string(&json!({ + "openclaw-cli-native": { + "sessionId": "openclaw-cli-native", + "sessionFile": sessions.join("openclaw-cli-native.jsonl"), + "sessionStartedAt": "2026-06-24T12:00:00Z", + "modelProvider": "openai", + "model": "gpt-5-mini", + "lastChannel": "telegram" + } + })) + .unwrap(), + ) + .unwrap(); + fs::write( + sessions.join("openclaw-cli-native.jsonl"), + format!( + "{}\n{}\n{}\n", + json!({ + "type": "session", + "version": 1, + "id": "openclaw-cli-native", + "timestamp": "2026-06-24T12:00:00Z", + "cwd": "/workspace" + }), + json!({ + "type": "message", + "id": "openclaw-cli-native-user", + "timestamp": "2026-06-24T12:00:01Z", + "message": {"role": "user", "content": query} + }), + json!({ + "type": "message", + "id": "openclaw-cli-native-assistant", + "parentId": "openclaw-cli-native-user", + "timestamp": "2026-06-24T12:00:02Z", + "message": {"role": "assistant", "content": "native import ok"} + }) + ), + ) + .unwrap(); + root.to_str().unwrap().to_owned() +} + +fn write_native_hermes_fixture(temp: &TempDir, query: &str) -> String { + let path = temp.path().join("native-hermes-state.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "create table sessions ( + id text primary key, + source text not null, + model text, + model_config text, + parent_session_id text, + started_at real not null, + ended_at real, + message_count integer default 0, + tool_call_count integer default 0, + input_tokens integer default 0, + output_tokens integer default 0, + cwd text, + title text, + archived integer default 0 + ); + create table messages ( + id integer primary key autoincrement, + session_id text not null, + role text not null, + content text, + tool_calls text, + tool_call_id text, + tool_name text, + timestamp real not null, + active integer not null default 1, + compacted integer not null default 0 + );", + ) + .unwrap(); + conn.execute( + "insert into sessions ( + id, source, model, model_config, started_at, message_count, cwd, title + ) values (?1, 'acp', 'gpt-5-mini', ?2, 1782259200.0, 2, '/workspace', 'native hermes')", + [ + "hermes-cli-native", + r#"{"cwd":"/workspace","provider":"openai"}"#, + ], + ) + .unwrap(); + conn.execute( + "insert into messages (session_id, role, content, timestamp) values (?1, 'user', ?2, 1782259201.0)", + ["hermes-cli-native", query], + ) + .unwrap(); + conn.execute( + "insert into messages (session_id, role, content, timestamp) values (?1, 'assistant', 'native import ok', 1782259202.0)", + ["hermes-cli-native"], + ) + .unwrap(); + path.to_str().unwrap().to_owned() +} + +fn write_native_nanoclaw_fixture(temp: &TempDir, query: &str) -> String { + let root = temp.path().join("native-nanoclaw"); + let data = root.join("data"); + let session_dir = data.join("v2-sessions/ag-1/session-1"); + fs::create_dir_all(&session_dir).unwrap(); + let central = Connection::open(data.join("v2.db")).unwrap(); + central + .execute_batch( + "create table agent_groups ( + id text primary key, + name text, + folder text, + agent_provider text + ); + create table messaging_groups ( + id text primary key, + channel_type text, + platform_id text, + instance text, + name text + ); + create table sessions ( + id text primary key, + agent_group_id text not null, + messaging_group_id text, + thread_id text, + agent_provider text, + status text, + container_status text, + last_active integer, + created_at integer + );", + ) + .unwrap(); + central + .execute( + "insert into agent_groups values ('ag-1', 'Personal', '/workspace', 'codex')", + [], + ) + .unwrap(); + central + .execute( + "insert into messaging_groups values ('mg-1', 'telegram', 'chat-1', 'default', 'DM')", + [], + ) + .unwrap(); + central + .execute( + "insert into sessions values ( + 'session-1', 'ag-1', 'mg-1', 'thread-1', 'codex', 'active', + 'running', 1782259202000, 1782259200000 + )", + [], + ) + .unwrap(); + let inbound = Connection::open(session_dir.join("inbound.db")).unwrap(); + inbound + .execute_batch( + "create table messages_in ( + id text primary key, + seq integer, + kind text, + timestamp integer, + status text, + trigger text, + platform_id text, + channel_type text, + thread_id text, + content text, + source_session_id text, + on_wake integer + );", + ) + .unwrap(); + inbound + .execute( + "insert into messages_in values ( + 'in-1', 1, 'chat', 1782259201000, 'done', 'message', + 'chat-1', 'telegram', 'thread-1', ?1, null, 0 + )", + [json!({"text": query}).to_string()], + ) + .unwrap(); + let outbound = Connection::open(session_dir.join("outbound.db")).unwrap(); + outbound + .execute_batch( + "create table messages_out ( + id text primary key, + seq integer, + in_reply_to text, + timestamp integer, + kind text, + platform_id text, + channel_type text, + thread_id text, + content text + );", + ) + .unwrap(); + outbound + .execute( + "insert into messages_out values ( + 'out-1', 2, 'in-1', 1782259202000, 'chat', + 'chat-1', 'telegram', 'thread-1', ?1 + )", + [json!({"text": "native import ok"}).to_string()], + ) + .unwrap(); + root.to_str().unwrap().to_owned() +} + +fn write_native_astrbot_fixture(temp: &TempDir, query: &str) -> String { + let data = temp.path().join("native-astrbot/data"); + fs::create_dir_all(&data).unwrap(); + let path = data.join("data_v4.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "create table conversations ( + id integer primary key, + inner_conversation_id text, + conversation_id text, + platform_id text, + user_id text, + content text not null, + title text, + persona_id text, + token_usage text, + created_at integer, + updated_at integer + ); + create table preferences ( + scope text, + key text, + value text + ); + create table platform_message_history ( + id integer primary key, + platform_id text, + user_id text, + sender_id text, + sender_name text, + content text, + llm_checkpoint_id text, + created_at integer + );", + ) + .unwrap(); + conn.execute( + "insert into conversations values ( + 1, 'umo-1', 'conv-1', 'webchat', 'user-1', ?1, 'native astrbot', + 'default', ?2, 1782259200000, 1782259202000 + )", + [ + json!([ + {"role": "user", "content": query}, + {"type": "_checkpoint", "id": "checkpoint-1"}, + {"role": "assistant", "content": "native import ok"} + ]) + .to_string(), + json!({"prompt": 1, "completion": 1}).to_string(), + ], + ) + .unwrap(); + conn.execute( + "insert into preferences values ('umo', 'sel_conv_id', 'conv-1')", + [], + ) + .unwrap(); + conn.execute( + "insert into platform_message_history values ( + 1, 'webchat', 'user-1', 'user-1', 'User', ?1, 'checkpoint-1', 1782259201000 + )", + [json!({"text": query}).to_string()], + ) + .unwrap(); + path.to_str().unwrap().to_owned() +} + +fn append_native_openclaw_event(path: &str, query: &str) { + let transcript = + Path::new(path).join("agents/personal-agent/sessions/openclaw-cli-native.jsonl"); + let mut file = fs::OpenOptions::new() + .append(true) + .open(transcript) + .unwrap(); + writeln!( + file, + "{}", + json!({ + "type": "message", + "id": "openclaw-cli-native-incremental", + "parentId": "openclaw-cli-native-assistant", + "timestamp": "2026-06-24T12:00:03Z", + "message": {"role": "user", "content": query} + }) + ) + .unwrap(); +} + +fn append_native_hermes_event(path: &str, query: &str) { + let conn = Connection::open(path).unwrap(); + conn.execute( + "insert into messages (session_id, role, content, timestamp) values (?1, 'user', ?2, 1782259203.0)", + ["hermes-cli-native", query], + ) + .unwrap(); +} + +fn append_native_nanoclaw_event(path: &str, query: &str) { + let conn = Connection::open( + Path::new(path) + .join("data/v2-sessions/ag-1/session-1") + .join("inbound.db"), + ) + .unwrap(); + conn.execute( + "insert into messages_in values ( + 'in-2', 1, 'chat', 1782259203000, 'done', 'message', + 'chat-1', 'telegram', 'thread-1', ?1, null, 0 + )", + [json!({"text": query}).to_string()], + ) + .unwrap(); +} + +fn append_native_astrbot_event(path: &str, query: &str) { + let conn = Connection::open(path).unwrap(); + let content: String = conn + .query_row( + "select content from conversations where id = 1", + [], + |row| row.get(0), + ) + .unwrap(); + let mut content: Value = serde_json::from_str(&content).unwrap(); + content + .as_array_mut() + .unwrap() + .push(json!({"role": "assistant", "content": query})); + conn.execute( + "update conversations set content = ?1, updated_at = 1782259203000 where id = 1", + [content.to_string()], + ) + .unwrap(); +} + +#[test] +fn openclaw_import_accepts_explicit_session_jsonl_file() { + let temp = tempdir(); + let query = "openclaw-explicit-file-oracle"; + let path = temp.path().join("openclaw-single-session.jsonl"); + fs::write( + &path, + format!( + "{}\n{}\n", + json!({ + "type": "session", + "id": "openclaw-single-session", + "timestamp": "2026-06-24T12:00:00Z" + }), + json!({ + "type": "message", + "id": "openclaw-single-user", + "timestamp": "2026-06-24T12:00:01Z", + "message": {"role": "user", "content": query} + }) + ), + ) + .unwrap(); + + let imported = json_output(ctx(&temp).args([ + "import", + "--provider", + "openclaw", + "--path", + path.to_str().unwrap(), + "--json", + ])); + assert_eq!(imported["totals"]["failed"], 0); + assert_eq!(imported["totals"]["imported_sources"], 1); + + let search = + json_output(ctx(&temp).args(["search", query, "--provider", "openclaw", "--json"])); + assert_search_provider_oracle(&search, "openclaw", query, 1, "message"); +} + +#[test] +fn nanoclaw_import_tolerates_partial_auxiliary_tables() { + let temp = tempdir(); + let query = "nanoclaw-partial-auxiliary-schema-oracle"; + let path = write_native_nanoclaw_fixture(&temp, query); + let conn = Connection::open(Path::new(&path).join("data/v2.db")).unwrap(); + conn.execute_batch( + "drop table agent_groups; + create table agent_groups (id text primary key); + insert into agent_groups values ('ag-1'); + drop table messaging_groups; + create table messaging_groups (id text primary key); + insert into messaging_groups values ('mg-1');", + ) + .unwrap(); + + let imported = json_output(ctx(&temp).args([ + "import", + "--provider", + "nanoclaw", + "--path", + &path, + "--json", + ])); + assert_eq!(imported["totals"]["failed"], 0); + assert_eq!(imported["totals"]["imported_sources"], 1); + + let search = + json_output(ctx(&temp).args(["search", query, "--provider", "nanoclaw", "--json"])); + assert_search_provider_oracle(&search, "nanoclaw", query, 1, "message"); +} + +#[test] +fn personal_agent_sqlite_imports_report_corrupt_databases() { + for (provider, path) in [ + ("hermes", "corrupt-hermes-state.db"), + ("astrbot", "corrupt-astrbot-data_v4.db"), + ] { + let temp = tempdir(); + let db_path = temp.path().join(path); + fs::write(&db_path, b"not sqlite").unwrap(); + let output = ctx(&temp) + .args([ + "import", + "--provider", + provider, + "--path", + db_path.to_str().unwrap(), + "--json", + ]) + .assert() + .failure() + .get_output() + .stderr + .clone(); + let stderr = String::from_utf8(output).unwrap(); + assert!(stderr.contains("not a database"), "{stderr}"); + } + + let temp = tempdir(); + let root = temp.path().join("corrupt-nanoclaw"); + fs::create_dir_all(root.join("data/v2-sessions")).unwrap(); + fs::write(root.join("data/v2.db"), b"not sqlite").unwrap(); + let output = ctx(&temp) + .args([ + "import", + "--provider", + "nanoclaw", + "--path", + root.to_str().unwrap(), + "--json", + ]) + .assert() + .failure() + .get_output() + .stderr + .clone(); + let stderr = String::from_utf8(output).unwrap(); + assert!(stderr.contains("not a database"), "{stderr}"); +} + #[test] fn native_provider_cli_requires_existing_history_or_explicit_path() { for (cli_provider, expected_blocker) in [ @@ -3885,6 +4595,10 @@ fn native_provider_cli_requires_existing_history_or_explicit_path() { "factory-ai-droid", "no native factory_ai_droid history found", ), + ("openclaw", "no native openclaw history found"), + ("hermes", "no native hermes history found"), + ("nanoclaw", "no native nanoclaw history found"), + ("astrbot", "no native astrbot history found"), ] { let temp = tempdir(); ctx(&temp) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 7e435bfd4..e656bfdab 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -22,7 +22,7 @@ use ctx_history_core::{ SyncState, Visibility, PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, }; use ctx_history_store::{CatalogSession, Store, StoreError}; -use rusqlite::{Connection, OpenFlags}; +use rusqlite::{Connection, OpenFlags, OptionalExtension}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use thiserror::Error; @@ -429,6 +429,90 @@ impl Default for OpenCodeSqliteImportOptions { } } +#[derive(Debug, Clone)] +pub struct OpenClawImportOptions { + pub machine_id: String, + pub source_path: Option, + pub imported_at: DateTime, + pub history_record_id: Option, + pub allow_partial_failures: bool, +} + +impl Default for OpenClawImportOptions { + fn default() -> Self { + Self { + machine_id: default_machine_id(), + source_path: None, + imported_at: utc_now(), + history_record_id: None, + allow_partial_failures: false, + } + } +} + +#[derive(Debug, Clone)] +pub struct HermesSqliteImportOptions { + pub machine_id: String, + pub source_path: Option, + pub imported_at: DateTime, + pub history_record_id: Option, + pub allow_partial_failures: bool, +} + +impl Default for HermesSqliteImportOptions { + fn default() -> Self { + Self { + machine_id: default_machine_id(), + source_path: None, + imported_at: utc_now(), + history_record_id: None, + allow_partial_failures: false, + } + } +} + +#[derive(Debug, Clone)] +pub struct NanoClawImportOptions { + pub machine_id: String, + pub source_path: Option, + pub imported_at: DateTime, + pub history_record_id: Option, + pub allow_partial_failures: bool, +} + +impl Default for NanoClawImportOptions { + fn default() -> Self { + Self { + machine_id: default_machine_id(), + source_path: None, + imported_at: utc_now(), + history_record_id: None, + allow_partial_failures: false, + } + } +} + +#[derive(Debug, Clone)] +pub struct AstrBotSqliteImportOptions { + pub machine_id: String, + pub source_path: Option, + pub imported_at: DateTime, + pub history_record_id: Option, + pub allow_partial_failures: bool, +} + +impl Default for AstrBotSqliteImportOptions { + fn default() -> Self { + Self { + machine_id: default_machine_id(), + source_path: None, + imported_at: utc_now(), + history_record_id: None, + allow_partial_failures: false, + } + } +} + #[derive(Debug, Clone)] pub struct AntigravityCliImportOptions { pub machine_id: String, @@ -736,6 +820,18 @@ pub struct ClaudeProjectsJsonlAdapter; #[derive(Debug, Clone, Copy, Default)] pub struct OpenCodeSqliteAdapter; +#[derive(Debug, Clone, Copy, Default)] +pub struct OpenClawJsonlAdapter; + +#[derive(Debug, Clone, Copy, Default)] +pub struct HermesSqliteAdapter; + +#[derive(Debug, Clone, Copy, Default)] +pub struct NanoClawProjectAdapter; + +#[derive(Debug, Clone, Copy, Default)] +pub struct AstrBotSqliteAdapter; + #[derive(Debug, Clone, Copy, Default)] pub struct AntigravityCliJsonlAdapter; @@ -1364,6 +1460,78 @@ impl ProviderCaptureAdapter for OpenCodeSqliteAdapter { } } +impl ProviderCaptureAdapter for OpenClawJsonlAdapter { + fn provider(&self) -> CaptureProvider { + CaptureProvider::OpenClaw + } + + fn source_format(&self) -> &str { + OPENCLAW_SOURCE_FORMAT + } + + fn normalize_path( + &self, + path: &Path, + context: &ProviderAdapterContext, + ) -> Result { + normalize_openclaw_history(path, context) + } +} + +impl ProviderCaptureAdapter for HermesSqliteAdapter { + fn provider(&self) -> CaptureProvider { + CaptureProvider::Hermes + } + + fn source_format(&self) -> &str { + HERMES_SQLITE_SOURCE_FORMAT + } + + fn normalize_path( + &self, + path: &Path, + context: &ProviderAdapterContext, + ) -> Result { + normalize_hermes_sqlite(path, context) + } +} + +impl ProviderCaptureAdapter for NanoClawProjectAdapter { + fn provider(&self) -> CaptureProvider { + CaptureProvider::NanoClaw + } + + fn source_format(&self) -> &str { + NANOCLAW_SOURCE_FORMAT + } + + fn normalize_path( + &self, + path: &Path, + context: &ProviderAdapterContext, + ) -> Result { + normalize_nanoclaw_project(path, context) + } +} + +impl ProviderCaptureAdapter for AstrBotSqliteAdapter { + fn provider(&self) -> CaptureProvider { + CaptureProvider::AstrBot + } + + fn source_format(&self) -> &str { + ASTRBOT_SQLITE_SOURCE_FORMAT + } + + fn normalize_path( + &self, + path: &Path, + context: &ProviderAdapterContext, + ) -> Result { + normalize_astrbot_sqlite(path, context) + } +} + impl ProviderCaptureAdapter for AntigravityCliJsonlAdapter { fn provider(&self) -> CaptureProvider { CaptureProvider::Antigravity @@ -3076,6 +3244,127 @@ pub fn import_opencode_sqlite( ) } +pub fn import_openclaw_history( + path: impl AsRef, + store: &mut Store, + options: OpenClawImportOptions, +) -> Result { + import_native_jsonl_tree( + store, + NativeJsonlTreeImport { + path: path.as_ref(), + machine_id: options.machine_id, + source_path: options.source_path, + imported_at: options.imported_at, + history_record_id: options.history_record_id, + allow_partial_failures: options.allow_partial_failures, + }, + OpenClawJsonlAdapter, + ) +} + +pub fn import_hermes_sqlite( + path: impl AsRef, + store: &mut Store, + options: HermesSqliteImportOptions, +) -> Result { + let path = path.as_ref(); + let source_path = options + .source_path + .clone() + .unwrap_or_else(|| path.to_path_buf()); + let normalization = HermesSqliteAdapter.normalize_path( + path, + &ProviderAdapterContext { + machine_id: options.machine_id, + source_path: Some(source_path), + imported_at: options.imported_at, + tool_output_mode: CodexToolOutputMode::Full, + event_mode: CodexEventImportMode::Rich, + include_notices: true, + }, + )?; + import_normalized_provider_captures( + store, + normalization, + NormalizedProviderImportOptions { + history_record_id: options.history_record_id, + allow_partial_failures: options.allow_partial_failures, + persist_cursors: true, + wrap_transaction: true, + fast_event_inserts: true, + }, + ) +} + +pub fn import_nanoclaw_project( + path: impl AsRef, + store: &mut Store, + options: NanoClawImportOptions, +) -> Result { + let path = path.as_ref(); + let source_path = options + .source_path + .clone() + .unwrap_or_else(|| path.to_path_buf()); + let normalization = NanoClawProjectAdapter.normalize_path( + path, + &ProviderAdapterContext { + machine_id: options.machine_id, + source_path: Some(source_path), + imported_at: options.imported_at, + tool_output_mode: CodexToolOutputMode::Full, + event_mode: CodexEventImportMode::Rich, + include_notices: true, + }, + )?; + import_normalized_provider_captures( + store, + normalization, + NormalizedProviderImportOptions { + history_record_id: options.history_record_id, + allow_partial_failures: options.allow_partial_failures, + persist_cursors: true, + wrap_transaction: true, + fast_event_inserts: true, + }, + ) +} + +pub fn import_astrbot_sqlite( + path: impl AsRef, + store: &mut Store, + options: AstrBotSqliteImportOptions, +) -> Result { + let path = path.as_ref(); + let source_path = options + .source_path + .clone() + .unwrap_or_else(|| path.to_path_buf()); + let normalization = AstrBotSqliteAdapter.normalize_path( + path, + &ProviderAdapterContext { + machine_id: options.machine_id, + source_path: Some(source_path), + imported_at: options.imported_at, + tool_output_mode: CodexToolOutputMode::Full, + event_mode: CodexEventImportMode::Rich, + include_notices: true, + }, + )?; + import_normalized_provider_captures( + store, + normalization, + NormalizedProviderImportOptions { + history_record_id: options.history_record_id, + allow_partial_failures: options.allow_partial_failures, + persist_cursors: true, + wrap_transaction: true, + fast_event_inserts: true, + }, + ) +} + pub fn import_antigravity_cli_history( path: impl AsRef, store: &mut Store, @@ -3228,6 +3517,10 @@ pub fn import_normalized_provider_captures( const CODEX_SESSION_SOURCE_FORMAT: &str = "codex_session_jsonl"; const CLAUDE_PROJECTS_SOURCE_FORMAT: &str = "claude_projects_jsonl_tree"; const OPENCODE_SQLITE_SOURCE_FORMAT: &str = "opencode_sqlite"; +const OPENCLAW_SOURCE_FORMAT: &str = "openclaw_session_jsonl_tree"; +const HERMES_SQLITE_SOURCE_FORMAT: &str = "hermes_state_sqlite"; +const NANOCLAW_SOURCE_FORMAT: &str = "nanoclaw_project"; +const ASTRBOT_SQLITE_SOURCE_FORMAT: &str = "astrbot_data_v4_sqlite"; const ANTIGRAVITY_CLI_SOURCE_FORMAT: &str = "antigravity_cli_transcript_jsonl_tree"; const GEMINI_CLI_SOURCE_FORMAT: &str = "gemini_cli_chat_recording_jsonl"; const CURSOR_AGENT_TRANSCRIPT_SOURCE_FORMAT: &str = "cursor_agent_transcript_jsonl"; @@ -4994,6 +5287,1777 @@ struct OpenCodeMessageRow { data: String, } +struct NativeSessionDraft { + provider: CaptureProvider, + source_format: &'static str, + provider_session_id: String, + parent_provider_session_id: Option, + root_provider_session_id: Option, + external_agent_id: Option, + agent_type: AgentType, + role_hint: Option, + is_primary: bool, + started_at: DateTime, + ended_at: Option>, + cwd: Option, + fidelity: Fidelity, + raw_source_path: String, + trust: ProviderSourceTrust, + source_metadata: Value, + session_metadata: Value, +} + +fn native_provider_capture( + draft: NativeSessionDraft, + context: &ProviderAdapterContext, + event: Option, +) -> ProviderCaptureEnvelope { + ProviderCaptureEnvelope { + schema_version: PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, + provider: draft.provider, + source: ProviderSourceEnvelope { + source_format: draft.source_format.to_owned(), + machine_id: context.machine_id.clone(), + observed_at: context.imported_at, + raw_source_path: Some(draft.raw_source_path), + raw_retention: ProviderRawRetention::PathReference, + redaction_boundary: ProviderRedactionBoundary::BeforeExport, + trust: draft.trust, + fidelity: draft.fidelity, + cursor: event.as_ref().and_then(|event| { + event.cursor.as_ref().map(|cursor| ProviderCursorRange { + before: None, + after: Some(ProviderCursorCheckpoint { + stream: provider_cursor_stream(draft.provider, draft.source_format), + cursor: cursor.clone(), + observed_at: event.occurred_at, + }), + }) + }), + idempotency_key: Some(format!( + "provider-source:{}:{}:{}", + draft.provider.as_str(), + draft.source_format, + draft.provider_session_id + )), + metadata: draft.source_metadata, + }, + session: ProviderSessionEnvelope { + provider_session_id: draft.provider_session_id.clone(), + parent_provider_session_id: draft.parent_provider_session_id, + root_provider_session_id: draft.root_provider_session_id, + external_agent_id: draft.external_agent_id, + agent_type: draft.agent_type, + role_hint: draft.role_hint, + is_primary: draft.is_primary, + status: SessionStatus::Imported, + started_at: draft.started_at, + ended_at: draft.ended_at, + cwd: draft.cwd, + fidelity: draft.fidelity, + idempotency_key: Some(format!( + "provider-session:{}:{}", + draft.provider.as_str(), + draft.provider_session_id + )), + artifacts: Vec::new(), + metadata: draft.session_metadata, + }, + event, + } +} + +fn open_provider_sqlite_readonly(path: &Path) -> Result { + ensure_regular_provider_transcript_file(path)?; + let conn = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + conn.busy_timeout(std::time::Duration::from_secs(5))?; + conn.pragma_update(None, "query_only", true)?; + Ok(conn) +} + +fn provider_timestamp_seconds(value: Option, fallback: DateTime) -> DateTime { + let Some(value) = value else { + return fallback; + }; + if !value.is_finite() { + return fallback; + } + let millis = if value.abs() > 1_000_000_000_000.0 { + value.round() as i64 + } else { + (value * 1000.0).round() as i64 + }; + DateTime::::from_timestamp_millis(millis).unwrap_or(fallback) +} + +fn provider_timestamp_millis(value: Option, fallback: DateTime) -> DateTime { + value + .and_then(DateTime::::from_timestamp_millis) + .unwrap_or(fallback) +} + +fn provider_timestamp_value(value: Option<&Value>, fallback: DateTime) -> DateTime { + match value { + Some(Value::String(raw)) => parse_rfc3339_utc(raw) + .or_else(|| { + raw.parse::() + .ok() + .map(|ts| provider_timestamp_seconds(Some(ts), fallback)) + }) + .unwrap_or(fallback), + Some(Value::Number(number)) => number + .as_f64() + .map(|ts| provider_timestamp_seconds(Some(ts), fallback)) + .unwrap_or(fallback), + _ => fallback, + } +} + +fn text_id_index(seed: &str, offset: u64) -> u64 { + offset.saturating_add(fnv1a64(seed.as_bytes()) & 0x0fff_ffff) +} + +fn provider_json_text(raw: &str) -> Value { + serde_json::from_str::(raw).unwrap_or_else(|_| Value::String(raw.to_owned())) +} + +fn hermes_decode_content(raw: Option<&str>) -> Value { + let Some(raw) = raw else { + return Value::Null; + }; + if let Some(json) = raw.strip_prefix("\0json:") { + return provider_json_text(json); + } + Value::String(raw.to_owned()) +} + +fn native_event( + provider: CaptureProvider, + source_format: &'static str, + provider_session_id: &str, + provider_event_index: u64, + provider_event_hash: Option, + cursor: String, + event_type: EventType, + role: Option, + occurred_at: DateTime, + text: String, + body: Value, + metadata: Value, +) -> ProviderEventEnvelope { + let (text, truncated) = provider_safe_preview(&text, PROVIDER_MAX_TEXT_CHARS); + ProviderEventEnvelope { + provider_event_index, + provider_event_hash, + cursor: Some(cursor), + event_type, + role, + occurred_at, + fidelity: Fidelity::Imported, + redaction_state: RedactionState::SafePreview, + idempotency_key: Some(format!( + "provider-event:{}:{}:{}", + provider.as_str(), + provider_session_id, + provider_event_index + )), + artifacts: Vec::new(), + payload: json!({ + "text": text, + "truncated": truncated, + "source_format": source_format, + "body": provider_capped_json(&body, PROVIDER_MAX_PREVIEW_CHARS), + }), + metadata, + } +} + +fn openclaw_agent_id(path: &Path) -> Option { + let components = path + .components() + .map(|component| component.as_os_str().to_string_lossy().to_string()) + .collect::>(); + components.windows(2).find_map(|window| { + (window[0] == "agents" && !window[1].is_empty()).then(|| window[1].clone()) + }) +} + +fn provider_path_has_component(path: &Path, expected: &str) -> bool { + path.components() + .any(|component| component.as_os_str() == expected) +} + +fn openclaw_session_indexes(root: &Path) -> BTreeMap { + let mut indexes = BTreeMap::new(); + let mut paths = Vec::new(); + collect_named_paths(root, "sessions.json", &mut paths); + for path in paths { + let Ok(text) = fs::read_to_string(&path) else { + continue; + }; + let Ok(value) = serde_json::from_str::(&text) else { + continue; + }; + let agent_id = openclaw_agent_id(&path); + for (key, value) in openclaw_session_index_entries(value) { + if let Some(session_id) = value + .get("sessionId") + .or_else(|| value.get("id")) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + { + if let Some(agent_id) = &agent_id { + indexes + .entry(format!("{agent_id}/{session_id}")) + .or_insert(value.clone()); + } + indexes + .entry(session_id.to_owned()) + .or_insert(value.clone()); + } + if let Some(agent_id) = &agent_id { + indexes + .entry(format!("{agent_id}/{key}")) + .or_insert(value.clone()); + } + indexes.entry(key).or_insert(value); + } + } + indexes +} + +fn openclaw_session_index_entries(value: Value) -> Vec<(String, Value)> { + match value { + Value::Array(items) => items + .into_iter() + .enumerate() + .map(|(index, value)| { + let key = value + .get("sessionId") + .or_else(|| value.get("id")) + .and_then(Value::as_str) + .map(str::to_owned) + .unwrap_or_else(|| index.to_string()); + (key, value) + }) + .collect(), + Value::Object(mut map) => { + if let Some(Value::Array(items)) = map.remove("sessions") { + return openclaw_session_index_entries(Value::Array(items)); + } + map.into_iter().collect() + } + _ => Vec::new(), + } +} + +fn collect_named_paths(root: &Path, name: &str, paths: &mut Vec) { + let Ok(metadata) = fs::symlink_metadata(root) else { + return; + }; + if metadata.file_type().is_symlink() { + return; + } + if metadata.file_type().is_file() { + if root.file_name().and_then(|file_name| file_name.to_str()) == Some(name) { + paths.push(root.to_path_buf()); + } + return; + } + if !metadata.file_type().is_dir() { + return; + } + let Ok(entries) = fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + collect_named_paths(&entry.path(), name, paths); + } +} + +fn normalize_openclaw_history( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + let mut paths = Vec::new(); + collect_jsonl_paths(path, &mut paths)?; + if !path.is_file() { + paths.retain(|candidate| provider_path_has_component(candidate, "sessions")); + } + paths.sort(); + if paths.is_empty() { + return Err(CaptureError::InvalidProviderTranscriptPath { + path: path.to_path_buf(), + reason: "no OpenClaw session JSONL transcripts found", + }); + } + let indexes = openclaw_session_indexes(path); + let mut merged = ProviderNormalizationResult::default(); + for transcript_path in paths { + let mut result = normalize_openclaw_jsonl_file(&transcript_path, context, &indexes)?; + merged.summary.merge(result.summary); + merged.captures.append(&mut result.captures); + merged.files_touched.append(&mut result.files_touched); + } + Ok(merged) +} + +fn normalize_openclaw_jsonl_file( + path: &Path, + context: &ProviderAdapterContext, + indexes: &BTreeMap, +) -> Result { + ensure_regular_provider_transcript_file(path)?; + let file = File::open(path)?; + let mut reader = BufReader::new(file); + let mut result = ProviderNormalizationResult::default(); + let fallback_id = path + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or("openclaw-session") + .to_owned(); + let agent_id = openclaw_agent_id(path); + let mut provider_session_id = agent_id + .as_ref() + .map(|agent| format!("{agent}/{fallback_id}")) + .unwrap_or_else(|| fallback_id.clone()); + let mut started_at = context.imported_at; + let mut cwd = None; + let mut header_raw = Value::Null; + let mut header_seen = false; + let mut line_number = 0usize; + let mut line = Vec::new(); + loop { + line.clear(); + let read = reader.read_until(b'\n', &mut line)?; + if read == 0 { + break; + } + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { + continue; + } + let value: Value = match serde_json::from_slice(&line) { + Ok(value) => value, + Err(err) => { + result.summary.failed += 1; + result.summary.failures.push(ProviderImportFailure { + line: line_number, + error: err.to_string(), + }); + continue; + } + }; + let row_type = value + .get("type") + .and_then(Value::as_str) + .unwrap_or("message"); + if row_type == "session" { + if let Some(id) = value.get("id").and_then(Value::as_str) { + provider_session_id = agent_id + .as_ref() + .map(|agent| format!("{agent}/{id}")) + .unwrap_or_else(|| id.to_owned()); + } + started_at = provider_timestamp_value(value.get("timestamp"), context.imported_at); + cwd = value.get("cwd").and_then(Value::as_str).map(str::to_owned); + header_raw = value.clone(); + header_seen = true; + result.captures.push(( + line_number, + openclaw_capture( + &provider_session_id, + agent_id.as_deref(), + started_at, + None, + cwd.clone(), + path, + context, + indexes, + header_raw.clone(), + None, + ), + )); + continue; + } + + let occurred_at = provider_timestamp_value(value.get("timestamp"), started_at); + let event_index = (line_number - 1) as u64; + let event = openclaw_event( + &provider_session_id, + event_index, + line_number, + &value, + occurred_at, + ); + if !header_seen { + header_seen = true; + result.captures.push(( + line_number, + openclaw_capture( + &provider_session_id, + agent_id.as_deref(), + started_at, + None, + cwd.clone(), + path, + context, + indexes, + header_raw.clone(), + None, + ), + )); + } + result.captures.push(( + line_number, + openclaw_capture( + &provider_session_id, + agent_id.as_deref(), + started_at, + None, + cwd.clone(), + path, + context, + indexes, + header_raw.clone(), + Some(event), + ), + )); + } + Ok(result) +} + +fn openclaw_capture( + provider_session_id: &str, + agent_id: Option<&str>, + started_at: DateTime, + ended_at: Option>, + cwd: Option, + path: &Path, + context: &ProviderAdapterContext, + indexes: &BTreeMap, + header_raw: Value, + event: Option, +) -> ProviderCaptureEnvelope { + let local_id = provider_session_id + .rsplit_once('/') + .map(|(_, id)| id) + .unwrap_or(provider_session_id); + let index = indexes + .get(provider_session_id) + .or_else(|| indexes.get(local_id)) + .cloned() + .unwrap_or(Value::Null); + native_provider_capture( + NativeSessionDraft { + provider: CaptureProvider::OpenClaw, + source_format: OPENCLAW_SOURCE_FORMAT, + provider_session_id: provider_session_id.to_owned(), + parent_provider_session_id: index + .get("parentSessionId") + .or_else(|| index.get("parent_session_id")) + .and_then(Value::as_str) + .map(str::to_owned), + root_provider_session_id: None, + external_agent_id: agent_id.map(str::to_owned), + agent_type: AgentType::Primary, + role_hint: Some("personal-agent".to_owned()), + is_primary: true, + started_at, + ended_at, + cwd, + fidelity: Fidelity::Partial, + raw_source_path: path.display().to_string(), + trust: ProviderSourceTrust::ProviderNative, + source_metadata: json!({ + "adapter": OPENCLAW_SOURCE_FORMAT, + "index": provider_capped_json(&index, PROVIDER_MAX_PREVIEW_CHARS), + "header": provider_capped_json(&header_raw, PROVIDER_MAX_PREVIEW_CHARS), + "support_level": "beta", + }), + session_metadata: json!({ + "source_format": OPENCLAW_SOURCE_FORMAT, + "agent_id": agent_id, + "session_index": provider_capped_json(&index, PROVIDER_MAX_PREVIEW_CHARS), + "fidelity_gap": "OpenClaw session JSONL is current native storage, but upstream keeps a storage-neutral accessor for future schema changes", + }), + }, + context, + event, + ) +} + +fn openclaw_event( + provider_session_id: &str, + event_index: u64, + line_number: usize, + row: &Value, + occurred_at: DateTime, +) -> ProviderEventEnvelope { + let row_type = row.get("type").and_then(Value::as_str).unwrap_or("message"); + let message = row.get("message").unwrap_or(row); + let role = message + .get("role") + .or_else(|| row.get("role")) + .and_then(Value::as_str) + .map(|role| provider_role(Some(role))); + let event_type = match row_type { + "message" => match role { + Some(EventRole::Tool) => EventType::ToolOutput, + _ => EventType::Message, + }, + "leaf" | "compaction" | "custom" => EventType::Notice, + _ => EventType::Notice, + }; + let text = message + .get("content") + .or_else(|| message.get("text")) + .or_else(|| message.get("output")) + .and_then(provider_value_text) + .unwrap_or_else(|| format!("OpenClaw {row_type}")); + native_event( + CaptureProvider::OpenClaw, + OPENCLAW_SOURCE_FORMAT, + provider_session_id, + event_index, + row.get("id").and_then(Value::as_str).map(str::to_owned), + format!("line:{line_number}"), + event_type, + role, + occurred_at, + text, + row.clone(), + json!({ + "source": "openclaw_jsonl", + "source_format": OPENCLAW_SOURCE_FORMAT, + "row_type": row_type, + "message_id": row.get("id").and_then(Value::as_str), + "parent_id": row.get("parentId").or_else(|| row.get("parent_id")).cloned(), + }), + ) +} + +#[derive(Debug, Clone)] +struct HermesSessionRow { + id: String, + source: String, + parent_session_id: Option, + model: Option, + model_config: Option, + started_at: f64, + ended_at: Option, + end_reason: Option, + message_count: i64, + tool_call_count: i64, + input_tokens: i64, + output_tokens: i64, + cache_read_tokens: i64, + cache_write_tokens: i64, + reasoning_tokens: i64, + cwd: Option, + git_branch: Option, + git_repo_root: Option, + billing_provider: Option, + billing_base_url: Option, + billing_mode: Option, + estimated_cost_usd: Option, + actual_cost_usd: Option, + title: Option, + archived: i64, +} + +#[derive(Debug, Clone)] +struct HermesMessageRow { + id: i64, + session_id: String, + role: String, + content: Option, + tool_call_id: Option, + tool_calls: Option, + tool_name: Option, + timestamp: f64, + token_count: Option, + finish_reason: Option, + reasoning: Option, + reasoning_content: Option, + reasoning_details: Option, + codex_reasoning_items: Option, + codex_message_items: Option, + platform_message_id: Option, + observed: i64, + active: i64, + compacted: i64, +} + +fn normalize_hermes_sqlite( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + let conn = open_provider_sqlite_readonly(path)?; + let user_version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?; + let schema_fingerprint = opencode_schema_fingerprint(&conn)?; + let sessions = hermes_sessions(&conn)?; + let messages = hermes_messages(&conn)?; + let sessions_by_id = sessions + .into_iter() + .map(|session| (session.id.clone(), session)) + .collect::>(); + let mut result = ProviderNormalizationResult::default(); + + for row in messages { + let Some(session) = sessions_by_id.get(&row.session_id) else { + result.summary.failed += 1; + result.summary.failures.push(ProviderImportFailure { + line: row.id.max(0) as usize, + error: format!( + "Hermes message {} references missing session {}", + row.id, row.session_id + ), + }); + continue; + }; + let provider_session_id = session.id.clone(); + let occurred_at = provider_timestamp_seconds(Some(row.timestamp), context.imported_at); + let started_at = provider_timestamp_seconds(Some(session.started_at), occurred_at); + let ended_at = session + .ended_at + .map(|timestamp| provider_timestamp_seconds(Some(timestamp), context.imported_at)); + let content = hermes_decode_content(row.content.as_deref()); + let text = provider_value_text(&content).unwrap_or_else(|| { + row.tool_name + .as_ref() + .map(|name| format!("tool: {name}")) + .unwrap_or_else(|| format!("Hermes {}", row.role)) + }); + let event_type = hermes_event_type(&row); + let role = Some(provider_role(Some(&row.role))); + let event = native_event( + CaptureProvider::Hermes, + HERMES_SQLITE_SOURCE_FORMAT, + &provider_session_id, + row.id.max(0) as u64, + Some(format!("message:{}", row.id)), + format!("messages:id:{}", row.id), + event_type, + role, + occurred_at, + text, + json!({ + "message_id": row.id, + "role": row.role, + "content": content, + "tool_call_id": row.tool_call_id, + "tool_calls": row.tool_calls.as_deref().map(provider_json_text), + "tool_name": row.tool_name, + "reasoning": row.reasoning, + "reasoning_content": row.reasoning_content, + "reasoning_details": row.reasoning_details.as_deref().map(provider_json_text), + "codex_reasoning_items": row.codex_reasoning_items.as_deref().map(provider_json_text), + "codex_message_items": row.codex_message_items.as_deref().map(provider_json_text), + }), + json!({ + "source": "hermes_state_db", + "source_format": HERMES_SQLITE_SOURCE_FORMAT, + "message_id": row.id, + "platform_message_id": row.platform_message_id, + "token_count": row.token_count, + "finish_reason": row.finish_reason, + "observed": row.observed != 0, + "active": row.active != 0, + "compacted": row.compacted != 0, + }), + ); + result.captures.push(( + row.id.max(0) as usize, + native_provider_capture( + NativeSessionDraft { + provider: CaptureProvider::Hermes, + source_format: HERMES_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + parent_provider_session_id: session.parent_session_id.clone(), + root_provider_session_id: None, + external_agent_id: Some(session.source.clone()), + agent_type: if session.parent_session_id.is_some() { + AgentType::Subagent + } else { + AgentType::Primary + }, + role_hint: Some(session.source.clone()), + is_primary: session.parent_session_id.is_none(), + started_at, + ended_at, + cwd: session.cwd.clone(), + fidelity: Fidelity::Imported, + raw_source_path: path.display().to_string(), + trust: ProviderSourceTrust::ProviderNative, + source_metadata: json!({ + "adapter": HERMES_SQLITE_SOURCE_FORMAT, + "sqlite_user_version": user_version, + "schema_fingerprint": schema_fingerprint, + "upstream_schema_version_at_research": 17, + }), + session_metadata: json!({ + "source_format": HERMES_SQLITE_SOURCE_FORMAT, + "source": session.source, + "title": session.title, + "model": session.model, + "model_config": session.model_config.as_deref().map(provider_json_text), + "end_reason": session.end_reason, + "message_count": session.message_count, + "tool_call_count": session.tool_call_count, + "tokens": { + "input": session.input_tokens, + "output": session.output_tokens, + "cache_read": session.cache_read_tokens, + "cache_write": session.cache_write_tokens, + "reasoning": session.reasoning_tokens, + }, + "git": { + "branch": session.git_branch, + "repo_root": session.git_repo_root, + }, + "billing": { + "provider": session.billing_provider, + "base_url": session.billing_base_url, + "mode": session.billing_mode, + "estimated_cost_usd": session.estimated_cost_usd, + "actual_cost_usd": session.actual_cost_usd, + }, + "archived": session.archived != 0, + }), + }, + context, + Some(event), + ), + )); + } + + Ok(result) +} + +fn hermes_event_type(row: &HermesMessageRow) -> EventType { + if row.role == "tool" { + EventType::ToolOutput + } else if row + .tool_calls + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || row + .tool_name + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + { + EventType::ToolCall + } else { + EventType::Message + } +} + +fn hermes_sessions(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "sessions")? { + return Err(CaptureError::InvalidPayload( + "Hermes state.db is missing required sessions table".into(), + )); + } + let columns = sqlite_table_columns(conn, "sessions")?; + ensure_sqlite_table_columns( + &columns, + "Hermes sessions table", + &["id", "source", "started_at"], + )?; + let parent_session_id = optional_column_expr(&columns, "parent_session_id", "NULL"); + let model = optional_column_expr(&columns, "model", "NULL"); + let model_config = optional_column_expr(&columns, "model_config", "NULL"); + let ended_at = optional_column_expr(&columns, "ended_at", "NULL"); + let end_reason = optional_column_expr(&columns, "end_reason", "NULL"); + let message_count = optional_column_expr(&columns, "message_count", "0"); + let tool_call_count = optional_column_expr(&columns, "tool_call_count", "0"); + let input_tokens = optional_column_expr(&columns, "input_tokens", "0"); + let output_tokens = optional_column_expr(&columns, "output_tokens", "0"); + let cache_read_tokens = optional_column_expr(&columns, "cache_read_tokens", "0"); + let cache_write_tokens = optional_column_expr(&columns, "cache_write_tokens", "0"); + let reasoning_tokens = optional_column_expr(&columns, "reasoning_tokens", "0"); + let cwd = optional_column_expr(&columns, "cwd", "NULL"); + let git_branch = optional_column_expr(&columns, "git_branch", "NULL"); + let git_repo_root = optional_column_expr(&columns, "git_repo_root", "NULL"); + let billing_provider = optional_column_expr(&columns, "billing_provider", "NULL"); + let billing_base_url = optional_column_expr(&columns, "billing_base_url", "NULL"); + let billing_mode = optional_column_expr(&columns, "billing_mode", "NULL"); + let estimated_cost_usd = optional_column_expr(&columns, "estimated_cost_usd", "NULL"); + let actual_cost_usd = optional_column_expr(&columns, "actual_cost_usd", "NULL"); + let title = optional_column_expr(&columns, "title", "NULL"); + let archived = optional_column_expr(&columns, "archived", "0"); + let sql = format!( + "select id, source, {parent_session_id}, {model}, {model_config}, started_at, \ + {ended_at}, {end_reason}, {message_count}, {tool_call_count}, {input_tokens}, \ + {output_tokens}, {cache_read_tokens}, {cache_write_tokens}, {reasoning_tokens}, \ + {cwd}, {git_branch}, {git_repo_root}, {billing_provider}, {billing_base_url}, \ + {billing_mode}, {estimated_cost_usd}, {actual_cost_usd}, {title}, {archived} \ + from sessions order by started_at, id" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(HermesSessionRow { + id: row.get(0)?, + source: row.get(1)?, + parent_session_id: row.get(2)?, + model: row.get(3)?, + model_config: row.get(4)?, + started_at: row.get(5)?, + ended_at: row.get(6)?, + end_reason: row.get(7)?, + message_count: row.get(8)?, + tool_call_count: row.get(9)?, + input_tokens: row.get(10)?, + output_tokens: row.get(11)?, + cache_read_tokens: row.get(12)?, + cache_write_tokens: row.get(13)?, + reasoning_tokens: row.get(14)?, + cwd: row.get(15)?, + git_branch: row.get(16)?, + git_repo_root: row.get(17)?, + billing_provider: row.get(18)?, + billing_base_url: row.get(19)?, + billing_mode: row.get(20)?, + estimated_cost_usd: row.get(21)?, + actual_cost_usd: row.get(22)?, + title: row.get(23)?, + archived: row.get(24)?, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +fn hermes_messages(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "messages")? { + return Err(CaptureError::InvalidPayload( + "Hermes state.db is missing required messages table".into(), + )); + } + let columns = sqlite_table_columns(conn, "messages")?; + ensure_sqlite_table_columns( + &columns, + "Hermes messages table", + &["id", "session_id", "role", "timestamp"], + )?; + let content = optional_column_expr(&columns, "content", "NULL"); + let tool_call_id = optional_column_expr(&columns, "tool_call_id", "NULL"); + let tool_calls = optional_column_expr(&columns, "tool_calls", "NULL"); + let tool_name = optional_column_expr(&columns, "tool_name", "NULL"); + let token_count = optional_column_expr(&columns, "token_count", "NULL"); + let finish_reason = optional_column_expr(&columns, "finish_reason", "NULL"); + let reasoning = optional_column_expr(&columns, "reasoning", "NULL"); + let reasoning_content = optional_column_expr(&columns, "reasoning_content", "NULL"); + let reasoning_details = optional_column_expr(&columns, "reasoning_details", "NULL"); + let codex_reasoning_items = optional_column_expr(&columns, "codex_reasoning_items", "NULL"); + let codex_message_items = optional_column_expr(&columns, "codex_message_items", "NULL"); + let platform_message_id = optional_column_expr(&columns, "platform_message_id", "NULL"); + let observed = optional_column_expr(&columns, "observed", "0"); + let active = optional_column_expr(&columns, "active", "1"); + let compacted = optional_column_expr(&columns, "compacted", "0"); + let visibility = if columns.contains("active") || columns.contains("compacted") { + format!("where ({active} = 1 or {compacted} = 1)") + } else { + String::new() + }; + let sql = format!( + "select id, session_id, role, {content}, {tool_call_id}, {tool_calls}, {tool_name}, \ + timestamp, {token_count}, {finish_reason}, {reasoning}, {reasoning_content}, \ + {reasoning_details}, {codex_reasoning_items}, {codex_message_items}, \ + {platform_message_id}, {observed}, {active}, {compacted} \ + from messages {visibility} order by session_id, id" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(HermesMessageRow { + id: row.get(0)?, + session_id: row.get(1)?, + role: row.get(2)?, + content: row.get(3)?, + tool_call_id: row.get(4)?, + tool_calls: row.get(5)?, + tool_name: row.get(6)?, + timestamp: row.get(7)?, + token_count: row.get(8)?, + finish_reason: row.get(9)?, + reasoning: row.get(10)?, + reasoning_content: row.get(11)?, + reasoning_details: row.get(12)?, + codex_reasoning_items: row.get(13)?, + codex_message_items: row.get(14)?, + platform_message_id: row.get(15)?, + observed: row.get(16)?, + active: row.get(17)?, + compacted: row.get(18)?, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +#[derive(Debug, Clone)] +struct NanoClawSessionRow { + id: String, + agent_group_id: String, + messaging_group_id: Option, + thread_id: Option, + agent_provider: Option, + status: Option, + container_status: Option, + last_active: Option, + created_at: Option, + agent_group_name: Option, + agent_group_folder: Option, + messaging_channel_type: Option, + messaging_platform_id: Option, + messaging_instance: Option, + messaging_name: Option, +} + +#[derive(Debug, Clone)] +struct NanoClawMessageRow { + source: &'static str, + id: String, + seq: Option, + kind: Option, + timestamp: Option, + status: Option, + in_reply_to: Option, + platform_id: Option, + channel_type: Option, + thread_id: Option, + content: Option, + trigger: Option, + source_session_id: Option, + on_wake: Option, +} + +fn normalize_nanoclaw_project( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + let project_root = nanoclaw_project_root(path)?; + let central_path = project_root.join("data").join("v2.db"); + let conn = open_provider_sqlite_readonly(¢ral_path)?; + let user_version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?; + let schema_fingerprint = opencode_schema_fingerprint(&conn)?; + let sessions = nanoclaw_sessions(&conn)?; + let mut result = ProviderNormalizationResult::default(); + for session in sessions { + let session_dir = project_root + .join("data") + .join("v2-sessions") + .join(&session.agent_group_id) + .join(&session.id); + let mut messages = Vec::new(); + let inbound_path = session_dir.join("inbound.db"); + if inbound_path.is_file() { + messages.extend(nanoclaw_inbound_messages(&inbound_path)?); + } + let outbound_path = session_dir.join("outbound.db"); + if outbound_path.is_file() { + messages.extend(nanoclaw_outbound_messages(&outbound_path)?); + } + messages.sort_by_key(|message| { + ( + message.timestamp.unwrap_or_default(), + message.seq.unwrap_or_default(), + message.source, + message.id.clone(), + ) + }); + for message in messages { + let provider_session_id = format!("{}/{}", session.agent_group_id, session.id); + let occurred_at = provider_timestamp_millis(message.timestamp, context.imported_at); + let started_at = provider_timestamp_millis(session.created_at, occurred_at); + let content = message + .content + .as_deref() + .map(provider_json_text) + .unwrap_or(Value::Null); + let text = provider_value_text(&content).unwrap_or_else(|| { + format!( + "NanoClaw {}", + message.kind.as_deref().unwrap_or(message.source) + ) + }); + let event_index = nanoclaw_event_index(&message); + let role = if message.source == "inbound" { + Some(EventRole::User) + } else { + Some(EventRole::Assistant) + }; + let event = native_event( + CaptureProvider::NanoClaw, + NANOCLAW_SOURCE_FORMAT, + &provider_session_id, + event_index, + Some(format!("{}:{}", message.source, message.id)), + format!( + "{}:{}:{}", + message.source, + session.id, + message.seq.unwrap_or_default() + ), + EventType::Message, + role, + occurred_at, + text, + json!({ + "message_id": message.id, + "seq": message.seq, + "kind": message.kind, + "content": content, + "status": message.status, + "in_reply_to": message.in_reply_to, + "platform_id": message.platform_id, + "channel_type": message.channel_type, + "thread_id": message.thread_id, + "trigger": message.trigger, + "source_session_id": message.source_session_id, + "on_wake": message.on_wake, + }), + json!({ + "source": format!("nanoclaw_{}", message.source), + "source_format": NANOCLAW_SOURCE_FORMAT, + "message_id": message.id, + "seq": message.seq, + }), + ); + result.captures.push(( + event_index.min(usize::MAX as u64) as usize, + native_provider_capture( + NativeSessionDraft { + provider: CaptureProvider::NanoClaw, + source_format: NANOCLAW_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + parent_provider_session_id: None, + root_provider_session_id: None, + external_agent_id: session.agent_provider.clone(), + agent_type: AgentType::Primary, + role_hint: Some("container-session".to_owned()), + is_primary: true, + started_at, + ended_at: session.last_active.map(|timestamp| { + provider_timestamp_millis(Some(timestamp), context.imported_at) + }), + cwd: session.agent_group_folder.clone(), + fidelity: Fidelity::Partial, + raw_source_path: project_root.display().to_string(), + trust: ProviderSourceTrust::ProviderNative, + source_metadata: json!({ + "adapter": NANOCLAW_SOURCE_FORMAT, + "central_db": central_path.display().to_string(), + "sqlite_user_version": user_version, + "schema_fingerprint": schema_fingerprint, + "support_level": "preview", + }), + session_metadata: json!({ + "source_format": NANOCLAW_SOURCE_FORMAT, + "session_id": session.id, + "agent_group_id": session.agent_group_id, + "agent_group_name": session.agent_group_name, + "agent_provider": session.agent_provider, + "status": session.status, + "container_status": session.container_status, + "messaging_group_id": session.messaging_group_id, + "messaging": { + "channel_type": session.messaging_channel_type, + "platform_id": session.messaging_platform_id, + "instance": session.messaging_instance, + "name": session.messaging_name, + "thread_id": session.thread_id, + }, + }), + }, + context, + Some(event), + ), + )); + } + } + Ok(result) +} + +fn nanoclaw_project_root(path: &Path) -> Result { + if path.is_dir() && path.join("data").join("v2.db").is_file() { + return Ok(path.to_path_buf()); + } + if path.file_name().and_then(|name| name.to_str()) == Some("v2.db") { + if let Some(data_dir) = path.parent() { + if let Some(root) = data_dir.parent() { + return Ok(root.to_path_buf()); + } + } + } + Err(CaptureError::InvalidProviderTranscriptPath { + path: path.to_path_buf(), + reason: "NanoClaw import path must be a project root or data/v2.db", + }) +} + +fn nanoclaw_event_index(message: &NanoClawMessageRow) -> u64 { + if let Some(seq) = message.seq { + let source_bucket = if message.source == "outbound" { + 500_000 + } else { + 0 + }; + let row_bucket = fnv1a64(format!("{}:{}", message.source, message.id).as_bytes()) % 500_000; + return (seq.max(0) as u64) + .saturating_mul(1_000_000) + .saturating_add(source_bucket) + .saturating_add(row_bucket); + } + text_id_index(&format!("{}:{}", message.source, message.id), 2_000_000_000) +} + +fn nanoclaw_sessions(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "sessions")? { + return Err(CaptureError::InvalidPayload( + "NanoClaw data/v2.db is missing required sessions table".into(), + )); + } + let columns = sqlite_table_columns(conn, "sessions")?; + ensure_sqlite_table_columns( + &columns, + "NanoClaw sessions table", + &["id", "agent_group_id"], + )?; + let messaging_group_id = optional_column_expr(&columns, "messaging_group_id", "NULL"); + let thread_id = optional_column_expr(&columns, "thread_id", "NULL"); + let agent_provider = optional_column_expr(&columns, "agent_provider", "NULL"); + let status = optional_column_expr(&columns, "status", "NULL"); + let container_status = optional_column_expr(&columns, "container_status", "NULL"); + let last_active = optional_column_expr(&columns, "last_active", "NULL"); + let created_at = optional_column_expr(&columns, "created_at", "NULL"); + let agent_group_columns = if sqlite_table_exists(conn, "agent_groups")? { + sqlite_table_columns(conn, "agent_groups")? + } else { + BTreeSet::new() + }; + let agent_group_name = + if agent_group_columns.contains("id") && agent_group_columns.contains("name") { + "(select name from agent_groups where agent_groups.id = sessions.agent_group_id)" + } else { + "NULL" + }; + let agent_group_folder = + if agent_group_columns.contains("id") && agent_group_columns.contains("folder") { + "(select folder from agent_groups where agent_groups.id = sessions.agent_group_id)" + } else { + "NULL" + }; + let (messaging_channel_type, messaging_platform_id, messaging_instance, messaging_name) = + if columns.contains("messaging_group_id") && sqlite_table_exists(conn, "messaging_groups")? + { + let messaging_columns = sqlite_table_columns(conn, "messaging_groups")?; + ( + if messaging_columns.contains("id") && messaging_columns.contains("channel_type") { + "(select channel_type from messaging_groups where messaging_groups.id = sessions.messaging_group_id)" + } else { + "NULL" + }, + if messaging_columns.contains("id") && messaging_columns.contains("platform_id") { + "(select platform_id from messaging_groups where messaging_groups.id = sessions.messaging_group_id)" + } else { + "NULL" + }, + if messaging_columns.contains("id") && messaging_columns.contains("instance") { + "(select instance from messaging_groups where messaging_groups.id = sessions.messaging_group_id)" + } else { + "NULL" + }, + if messaging_columns.contains("id") && messaging_columns.contains("name") { + "(select name from messaging_groups where messaging_groups.id = sessions.messaging_group_id)" + } else { + "NULL" + }, + ) + } else { + ("NULL", "NULL", "NULL", "NULL") + }; + let sql = format!( + "select id, agent_group_id, {messaging_group_id}, {thread_id}, {agent_provider}, \ + {status}, {container_status}, {last_active}, {created_at}, {agent_group_name}, \ + {agent_group_folder}, {messaging_channel_type}, {messaging_platform_id}, \ + {messaging_instance}, {messaging_name} from sessions order by created_at, id" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(NanoClawSessionRow { + id: row.get(0)?, + agent_group_id: row.get(1)?, + messaging_group_id: row.get(2)?, + thread_id: row.get(3)?, + agent_provider: row.get(4)?, + status: row.get(5)?, + container_status: row.get(6)?, + last_active: row.get(7)?, + created_at: row.get(8)?, + agent_group_name: row.get(9)?, + agent_group_folder: row.get(10)?, + messaging_channel_type: row.get(11)?, + messaging_platform_id: row.get(12)?, + messaging_instance: row.get(13)?, + messaging_name: row.get(14)?, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +fn nanoclaw_inbound_messages(path: &Path) -> Result> { + let conn = open_provider_sqlite_readonly(path)?; + if !sqlite_table_exists(&conn, "messages_in")? { + return Ok(Vec::new()); + } + let columns = sqlite_table_columns(&conn, "messages_in")?; + ensure_sqlite_table_columns(&columns, "NanoClaw inbound messages table", &["id"])?; + let seq = optional_column_expr(&columns, "seq", "NULL"); + let kind = optional_column_expr(&columns, "kind", "NULL"); + let timestamp = optional_column_expr(&columns, "timestamp", "NULL"); + let status = optional_column_expr(&columns, "status", "NULL"); + let trigger = optional_column_expr(&columns, "trigger", "NULL"); + let platform_id = optional_column_expr(&columns, "platform_id", "NULL"); + let channel_type = optional_column_expr(&columns, "channel_type", "NULL"); + let thread_id = optional_column_expr(&columns, "thread_id", "NULL"); + let content = optional_column_expr(&columns, "content", "NULL"); + let source_session_id = optional_column_expr(&columns, "source_session_id", "NULL"); + let on_wake = optional_column_expr(&columns, "on_wake", "NULL"); + let sql = format!( + "select id, {seq}, {kind}, {timestamp}, {status}, {trigger}, {platform_id}, \ + {channel_type}, {thread_id}, {content}, {source_session_id}, {on_wake} \ + from messages_in order by {seq}, id" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(NanoClawMessageRow { + source: "inbound", + id: row.get(0)?, + seq: row.get(1)?, + kind: row.get(2)?, + timestamp: row.get(3)?, + status: row.get(4)?, + trigger: row.get(5)?, + platform_id: row.get(6)?, + channel_type: row.get(7)?, + thread_id: row.get(8)?, + content: row.get(9)?, + source_session_id: row.get(10)?, + on_wake: row.get(11)?, + in_reply_to: None, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +fn nanoclaw_outbound_messages(path: &Path) -> Result> { + let conn = open_provider_sqlite_readonly(path)?; + if !sqlite_table_exists(&conn, "messages_out")? { + return Ok(Vec::new()); + } + let columns = sqlite_table_columns(&conn, "messages_out")?; + ensure_sqlite_table_columns(&columns, "NanoClaw outbound messages table", &["id"])?; + let seq = optional_column_expr(&columns, "seq", "NULL"); + let kind = optional_column_expr(&columns, "kind", "NULL"); + let timestamp = optional_column_expr(&columns, "timestamp", "NULL"); + let in_reply_to = optional_column_expr(&columns, "in_reply_to", "NULL"); + let platform_id = optional_column_expr(&columns, "platform_id", "NULL"); + let channel_type = optional_column_expr(&columns, "channel_type", "NULL"); + let thread_id = optional_column_expr(&columns, "thread_id", "NULL"); + let content = optional_column_expr(&columns, "content", "NULL"); + let sql = format!( + "select id, {seq}, {kind}, {timestamp}, {in_reply_to}, {platform_id}, \ + {channel_type}, {thread_id}, {content} from messages_out order by {seq}, id" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(NanoClawMessageRow { + source: "outbound", + id: row.get(0)?, + seq: row.get(1)?, + kind: row.get(2)?, + timestamp: row.get(3)?, + in_reply_to: row.get(4)?, + platform_id: row.get(5)?, + channel_type: row.get(6)?, + thread_id: row.get(7)?, + content: row.get(8)?, + status: None, + trigger: None, + source_session_id: None, + on_wake: None, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +#[derive(Debug, Clone)] +struct AstrBotConversationRow { + row_id: i64, + inner_conversation_id: Option, + conversation_id: String, + platform_id: Option, + user_id: Option, + content: String, + title: Option, + persona_id: Option, + token_usage: Option, + created_at: Option, + updated_at: Option, +} + +#[derive(Debug, Clone)] +struct AstrBotPlatformMessageRow { + id: i64, + platform_id: Option, + user_id: Option, + sender_id: Option, + sender_name: Option, + content: Option, + llm_checkpoint_id: Option, + created_at: Option, +} + +fn normalize_astrbot_sqlite( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + let conn = open_provider_sqlite_readonly(path)?; + let user_version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?; + let schema_fingerprint = opencode_schema_fingerprint(&conn)?; + let conversations = astrbot_conversations(&conn)?; + let platform_messages = astrbot_platform_messages(&conn)?; + let selected_conversation = astrbot_selected_conversation(&conn).ok().flatten(); + let mut result = ProviderNormalizationResult::default(); + let mut checkpoint_sessions = BTreeMap::::new(); + + for conversation in &conversations { + let provider_session_id = astrbot_provider_session_id(conversation); + let started_at = provider_timestamp_millis(conversation.created_at, context.imported_at); + let ended_at = conversation + .updated_at + .map(|timestamp| provider_timestamp_millis(Some(timestamp), context.imported_at)); + let content = provider_json_text(&conversation.content); + if let Value::Array(items) = &content { + for (index, item) in items.iter().enumerate() { + if let Some(checkpoint) = astrbot_checkpoint_id(item) { + checkpoint_sessions.insert(checkpoint, provider_session_id.clone()); + continue; + } + let role = astrbot_role(item); + let text = astrbot_item_text(item) + .unwrap_or_else(|| "AstrBot conversation item".to_owned()); + let event = native_event( + CaptureProvider::AstrBot, + ASTRBOT_SQLITE_SOURCE_FORMAT, + &provider_session_id, + index as u64, + astrbot_item_id(item).map(|id| format!("conversation:{id}")), + format!("conversation:{}:item:{index}", conversation.conversation_id), + EventType::Message, + role, + started_at, + text, + item.clone(), + json!({ + "source": "astrbot_conversations", + "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, + "conversation_id": conversation.conversation_id, + "inner_conversation_id": conversation.inner_conversation_id, + "item_index": index, + }), + ); + result.captures.push(( + index + 1, + astrbot_capture( + conversation, + &provider_session_id, + started_at, + ended_at, + path, + context, + user_version, + &schema_fingerprint, + selected_conversation.as_deref(), + Some(event), + ), + )); + } + } else { + let text = + provider_value_text(&content).unwrap_or_else(|| "AstrBot conversation".to_owned()); + let event = native_event( + CaptureProvider::AstrBot, + ASTRBOT_SQLITE_SOURCE_FORMAT, + &provider_session_id, + 0, + Some(format!("conversation-row:{}", conversation.row_id)), + format!("conversation:{}:content", conversation.conversation_id), + EventType::Message, + None, + started_at, + text, + content.clone(), + json!({ + "source": "astrbot_conversations", + "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, + "conversation_id": conversation.conversation_id, + }), + ); + result.captures.push(( + conversation.row_id.max(0) as usize, + astrbot_capture( + conversation, + &provider_session_id, + started_at, + ended_at, + path, + context, + user_version, + &schema_fingerprint, + selected_conversation.as_deref(), + Some(event), + ), + )); + } + } + + let conversations_by_id = conversations + .iter() + .map(|conversation| (astrbot_provider_session_id(conversation), conversation)) + .collect::>(); + for message in platform_messages { + let provider_session_id = message + .llm_checkpoint_id + .as_ref() + .and_then(|checkpoint| checkpoint_sessions.get(checkpoint)) + .cloned() + .unwrap_or_else(|| { + format!( + "platform/{}/{}", + message.platform_id.as_deref().unwrap_or("unknown"), + message.user_id.as_deref().unwrap_or("unknown") + ) + }); + let conversation = conversations_by_id.get(&provider_session_id).copied(); + let started_at = conversation + .and_then(|conversation| conversation.created_at) + .map(|timestamp| provider_timestamp_millis(Some(timestamp), context.imported_at)) + .unwrap_or_else(|| provider_timestamp_millis(message.created_at, context.imported_at)); + let content = message + .content + .as_deref() + .map(provider_json_text) + .unwrap_or(Value::Null); + let text = + provider_value_text(&content).unwrap_or_else(|| "AstrBot platform message".to_owned()); + let role = if message.sender_id.as_deref() == message.user_id.as_deref() { + Some(EventRole::User) + } else { + Some(EventRole::Assistant) + }; + let event_index = 1_000_000u64.saturating_add(message.id.max(0) as u64); + let event = native_event( + CaptureProvider::AstrBot, + ASTRBOT_SQLITE_SOURCE_FORMAT, + &provider_session_id, + event_index, + Some(format!("platform-message:{}", message.id)), + format!("platform_message_history:id:{}", message.id), + EventType::Message, + role, + provider_timestamp_millis(message.created_at, started_at), + text, + json!({ + "message_id": message.id, + "platform_id": message.platform_id, + "user_id": message.user_id, + "sender_id": message.sender_id, + "sender_name": message.sender_name, + "content": content, + "llm_checkpoint_id": message.llm_checkpoint_id, + }), + json!({ + "source": "astrbot_platform_message_history", + "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, + "message_id": message.id, + }), + ); + if let Some(conversation) = conversation { + result.captures.push(( + event_index.min(usize::MAX as u64) as usize, + astrbot_capture( + conversation, + &provider_session_id, + started_at, + conversation.updated_at.map(|timestamp| { + provider_timestamp_millis(Some(timestamp), context.imported_at) + }), + path, + context, + user_version, + &schema_fingerprint, + selected_conversation.as_deref(), + Some(event), + ), + )); + } else { + result.captures.push(( + event_index.min(usize::MAX as u64) as usize, + native_provider_capture( + NativeSessionDraft { + provider: CaptureProvider::AstrBot, + source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + parent_provider_session_id: None, + root_provider_session_id: None, + external_agent_id: message.platform_id.clone(), + agent_type: AgentType::Primary, + role_hint: Some("platform-history".to_owned()), + is_primary: true, + started_at, + ended_at: None, + cwd: None, + fidelity: Fidelity::Partial, + raw_source_path: path.display().to_string(), + trust: ProviderSourceTrust::ProviderNative, + source_metadata: json!({ + "adapter": ASTRBOT_SQLITE_SOURCE_FORMAT, + "sqlite_user_version": user_version, + "schema_fingerprint": schema_fingerprint, + "support_level": "preview", + }), + session_metadata: json!({ + "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, + "platform_id": message.platform_id, + "user_id": message.user_id, + "fidelity_gap": "platform history row was not linked to a conversations checkpoint", + }), + }, + context, + Some(event), + ), + )); + } + } + + Ok(result) +} + +fn astrbot_provider_session_id(conversation: &AstrBotConversationRow) -> String { + conversation + .inner_conversation_id + .as_ref() + .or(Some(&conversation.conversation_id)) + .cloned() + .unwrap_or_else(|| format!("conversation-row-{}", conversation.row_id)) +} + +fn astrbot_capture( + conversation: &AstrBotConversationRow, + provider_session_id: &str, + started_at: DateTime, + ended_at: Option>, + path: &Path, + context: &ProviderAdapterContext, + user_version: i64, + schema_fingerprint: &str, + selected_conversation: Option<&str>, + event: Option, +) -> ProviderCaptureEnvelope { + native_provider_capture( + NativeSessionDraft { + provider: CaptureProvider::AstrBot, + source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.to_owned(), + parent_provider_session_id: None, + root_provider_session_id: None, + external_agent_id: conversation.platform_id.clone(), + agent_type: AgentType::Primary, + role_hint: Some("llm-context".to_owned()), + is_primary: true, + started_at, + ended_at, + cwd: None, + fidelity: Fidelity::Partial, + raw_source_path: path.display().to_string(), + trust: ProviderSourceTrust::ProviderNative, + source_metadata: json!({ + "adapter": ASTRBOT_SQLITE_SOURCE_FORMAT, + "sqlite_user_version": user_version, + "schema_fingerprint": schema_fingerprint, + "support_level": "preview", + }), + session_metadata: json!({ + "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, + "conversation_id": conversation.conversation_id, + "inner_conversation_id": conversation.inner_conversation_id, + "platform_id": conversation.platform_id, + "user_id": conversation.user_id, + "title": conversation.title, + "persona_id": conversation.persona_id, + "token_usage": conversation.token_usage.as_deref().map(provider_json_text), + "selected_conversation": selected_conversation, + "fidelity_gap": "AstrBot preview imports local LLM context plus available platform history; it may not be a complete raw IM transcript", + }), + }, + context, + event, + ) +} + +fn astrbot_item_id(item: &Value) -> Option<&str> { + item.get("id") + .or_else(|| item.get("message_id")) + .or_else(|| item.get("checkpoint_id")) + .and_then(Value::as_str) +} + +fn astrbot_checkpoint_id(item: &Value) -> Option { + let item_type = item + .get("type") + .or_else(|| item.get("role")) + .and_then(Value::as_str)?; + if item_type != "_checkpoint" && item_type != "checkpoint" { + return None; + } + astrbot_item_id(item).map(str::to_owned) +} + +fn astrbot_role(item: &Value) -> Option { + item.get("role") + .or_else(|| item.get("type")) + .and_then(Value::as_str) + .map(|role| provider_role(Some(role))) +} + +fn astrbot_item_text(item: &Value) -> Option { + item.get("content") + .or_else(|| item.get("text")) + .or_else(|| item.get("message")) + .and_then(provider_value_text) +} + +fn astrbot_conversations(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "conversations")? { + return Err(CaptureError::InvalidPayload( + "AstrBot data_v4.db is missing required conversations table".into(), + )); + } + let columns = sqlite_table_columns(conn, "conversations")?; + ensure_sqlite_table_columns(&columns, "AstrBot conversations table", &["content"])?; + let row_id = if columns.contains("id") { + "id" + } else { + "rowid" + }; + let inner_conversation_id = optional_column_expr(&columns, "inner_conversation_id", "NULL"); + let conversation_id = optional_column_expr( + &columns, + "conversation_id", + optional_column_expr(&columns, "inner_conversation_id", "CAST(rowid AS TEXT)"), + ); + let platform_id = optional_column_expr(&columns, "platform_id", "NULL"); + let user_id = optional_column_expr(&columns, "user_id", "NULL"); + let title = optional_column_expr(&columns, "title", "NULL"); + let persona_id = optional_column_expr(&columns, "persona_id", "NULL"); + let token_usage = optional_column_expr(&columns, "token_usage", "NULL"); + let created_at = optional_column_expr(&columns, "created_at", "NULL"); + let updated_at = optional_column_expr(&columns, "updated_at", "NULL"); + let sql = format!( + "select {row_id}, {inner_conversation_id}, {conversation_id}, {platform_id}, \ + {user_id}, content, {title}, {persona_id}, {token_usage}, {created_at}, \ + {updated_at} from conversations order by {created_at}, {row_id}" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(AstrBotConversationRow { + row_id: row.get(0)?, + inner_conversation_id: row.get(1)?, + conversation_id: row.get::<_, String>(2)?, + platform_id: row.get(3)?, + user_id: row.get(4)?, + content: row.get(5)?, + title: row.get(6)?, + persona_id: row.get(7)?, + token_usage: row.get(8)?, + created_at: row.get(9)?, + updated_at: row.get(10)?, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +fn astrbot_platform_messages(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "platform_message_history")? { + return Ok(Vec::new()); + } + let columns = sqlite_table_columns(conn, "platform_message_history")?; + let id = if columns.contains("id") { + "id" + } else { + "rowid" + }; + let platform_id = optional_column_expr(&columns, "platform_id", "NULL"); + let user_id = optional_column_expr(&columns, "user_id", "NULL"); + let sender_id = optional_column_expr(&columns, "sender_id", "NULL"); + let sender_name = optional_column_expr(&columns, "sender_name", "NULL"); + let content = optional_column_expr(&columns, "content", "NULL"); + let llm_checkpoint_id = optional_column_expr(&columns, "llm_checkpoint_id", "NULL"); + let created_at = optional_column_expr(&columns, "created_at", "NULL"); + let sql = format!( + "select {id}, {platform_id}, {user_id}, {sender_id}, {sender_name}, \ + {content}, {llm_checkpoint_id}, {created_at} from platform_message_history \ + order by {created_at}, {id}" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(AstrBotPlatformMessageRow { + id: row.get(0)?, + platform_id: row.get(1)?, + user_id: row.get(2)?, + sender_id: row.get(3)?, + sender_name: row.get(4)?, + content: row.get(5)?, + llm_checkpoint_id: row.get(6)?, + created_at: row.get(7)?, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +fn astrbot_selected_conversation(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "preferences")? { + return Ok(None); + } + let columns = sqlite_table_columns(conn, "preferences")?; + if !columns.contains("key") || !columns.contains("value") { + return Ok(None); + } + let scope_filter = if columns.contains("scope") { + "AND scope = 'umo'" + } else { + "" + }; + let sql = + format!("select value from preferences where key = 'sel_conv_id' {scope_filter} limit 1"); + let value = conn + .query_row(&sql, [], |row| row.get::<_, Option>(0)) + .optional()? + .flatten(); + Ok(value) +} + fn normalize_opencode_sqlite( path: &Path, context: &ProviderAdapterContext, diff --git a/crates/ctx-history-capture/src/provider_sources.rs b/crates/ctx-history-capture/src/provider_sources.rs index de5e75df9..2c3ca174a 100644 --- a/crates/ctx-history-capture/src/provider_sources.rs +++ b/crates/ctx-history-capture/src/provider_sources.rs @@ -1,4 +1,8 @@ -use std::path::{Path, PathBuf}; +use std::{ + collections::HashSet, + env, + path::{Path, PathBuf}, +}; use ctx_history_core::{CaptureProvider, ProviderRawRetention, ProviderRedactionBoundary}; @@ -11,9 +15,20 @@ pub enum ProviderSourceKind { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProviderImportSupport { Native, + Preview, Unsupported, } +impl ProviderImportSupport { + pub fn is_importable(self) -> bool { + matches!(self, Self::Native | Self::Preview) + } + + pub fn is_auto_importable(self) -> bool { + matches!(self, Self::Native) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProviderCatalogSupport { Native, @@ -136,6 +151,38 @@ const FACTORY_DROID_DEFAULTS: &[ProviderDefaultLocation] = &[ProviderDefaultLoca source_kind: ProviderSourceKind::NativeHistory, }]; +const OPENCLAW_DEFAULTS: &[ProviderDefaultLocation] = &[ + ProviderDefaultLocation { + path_components: &[".openclaw"], + source_format: "openclaw_session_jsonl_tree", + source_kind: ProviderSourceKind::NativeHistory, + }, + ProviderDefaultLocation { + path_components: &[".clawdbot"], + source_format: "openclaw_session_jsonl_tree", + source_kind: ProviderSourceKind::NativeHistory, + }, + ProviderDefaultLocation { + path_components: &[".moltbot"], + source_format: "openclaw_session_jsonl_tree", + source_kind: ProviderSourceKind::NativeHistory, + }, +]; + +const HERMES_DEFAULTS: &[ProviderDefaultLocation] = &[ProviderDefaultLocation { + path_components: &[".hermes", "state.db"], + source_format: "hermes_state_sqlite", + source_kind: ProviderSourceKind::NativeHistory, +}]; + +const NANOCLAW_DEFAULTS: &[ProviderDefaultLocation] = &[]; + +const ASTRBOT_DEFAULTS: &[ProviderDefaultLocation] = &[ProviderDefaultLocation { + path_components: &[".astrbot", "data", "data_v4.db"], + source_format: "astrbot_data_v4_sqlite", + source_kind: ProviderSourceKind::NativeHistory, +}]; + const PROVIDER_SPECS: &[ProviderSourceSpec] = &[ ProviderSourceSpec { provider: CaptureProvider::Codex, @@ -227,6 +274,46 @@ const PROVIDER_SPECS: &[ProviderSourceSpec] = &[ redaction_boundary: ProviderRedactionBoundary::BeforeExport, unsupported_reason: None, }, + ProviderSourceSpec { + provider: CaptureProvider::OpenClaw, + display_name: "OpenClaw", + default_locations: OPENCLAW_DEFAULTS, + import_support: ProviderImportSupport::Native, + catalog_support: ProviderCatalogSupport::None, + raw_retention: ProviderRawRetention::PathReference, + redaction_boundary: ProviderRedactionBoundary::BeforeExport, + unsupported_reason: None, + }, + ProviderSourceSpec { + provider: CaptureProvider::Hermes, + display_name: "Hermes Agent", + default_locations: HERMES_DEFAULTS, + import_support: ProviderImportSupport::Native, + catalog_support: ProviderCatalogSupport::None, + raw_retention: ProviderRawRetention::PathReference, + redaction_boundary: ProviderRedactionBoundary::BeforeExport, + unsupported_reason: None, + }, + ProviderSourceSpec { + provider: CaptureProvider::NanoClaw, + display_name: "NanoClaw", + default_locations: NANOCLAW_DEFAULTS, + import_support: ProviderImportSupport::Preview, + catalog_support: ProviderCatalogSupport::None, + raw_retention: ProviderRawRetention::PathReference, + redaction_boundary: ProviderRedactionBoundary::BeforeExport, + unsupported_reason: None, + }, + ProviderSourceSpec { + provider: CaptureProvider::AstrBot, + display_name: "AstrBot", + default_locations: ASTRBOT_DEFAULTS, + import_support: ProviderImportSupport::Preview, + catalog_support: ProviderCatalogSupport::None, + raw_retention: ProviderRawRetention::PathReference, + redaction_boundary: ProviderRedactionBoundary::BeforeExport, + unsupported_reason: None, + }, ]; pub fn provider_source_specs() -> &'static [ProviderSourceSpec] { @@ -238,39 +325,142 @@ pub fn provider_source_spec(provider: CaptureProvider) -> Option<&'static Provid } pub fn discover_provider_sources(home: &Path) -> Vec { - PROVIDER_SPECS - .iter() - .flat_map(|spec| { - spec.default_locations.iter().map(|location| { - let path = location - .path_components - .iter() - .fold(home.to_path_buf(), |path, component| path.join(component)); - provider_source_from_location(spec, location, path) - }) - }) - .collect() + dedupe_sources( + PROVIDER_SPECS + .iter() + .flat_map(|spec| discover_provider_sources_for_spec(home, spec)) + .collect(), + ) } pub fn discover_provider_sources_for_provider( home: &Path, provider: CaptureProvider, ) -> Vec { - PROVIDER_SPECS + dedupe_sources( + PROVIDER_SPECS + .iter() + .filter(|spec| spec.provider == provider) + .flat_map(|spec| discover_provider_sources_for_spec(home, spec)) + .collect(), + ) +} + +fn discover_provider_sources_for_spec( + home: &Path, + spec: &ProviderSourceSpec, +) -> Vec { + let mut sources = spec + .default_locations .iter() - .filter(|spec| spec.provider == provider) - .flat_map(|spec| { - spec.default_locations.iter().map(|location| { - let path = location - .path_components - .iter() - .fold(home.to_path_buf(), |path, component| path.join(component)); - provider_source_from_location(spec, location, path) - }) + .map(|location| { + let path = location + .path_components + .iter() + .fold(home.to_path_buf(), |path, component| path.join(component)); + provider_source_from_location(spec, location, path) }) + .collect::>(); + + match spec.provider { + CaptureProvider::OpenClaw => { + if let Some(path) = env_path("OPENCLAW_STATE_DIR") { + sources.push(provider_source_from_parts( + spec, + path, + "openclaw_session_jsonl_tree", + ProviderSourceKind::NativeHistory, + )); + } + } + CaptureProvider::Hermes => { + if let Some(path) = env_path("HERMES_HOME") { + sources.push(provider_source_from_parts( + spec, + path.join("state.db"), + "hermes_state_sqlite", + ProviderSourceKind::NativeHistory, + )); + } + } + CaptureProvider::NanoClaw => { + for root in current_dir_ancestors_with(|candidate| { + candidate.join("data").join("v2.db").is_file() + && candidate.join("data").join("v2-sessions").is_dir() + }) { + sources.push(provider_source_from_parts( + spec, + root, + "nanoclaw_project", + ProviderSourceKind::NativeHistory, + )); + } + } + CaptureProvider::AstrBot => { + if let Some(path) = env_path("ASTRBOT_ROOT") { + sources.push(provider_source_from_parts( + spec, + path.join("data").join("data_v4.db"), + "astrbot_data_v4_sqlite", + ProviderSourceKind::NativeHistory, + )); + } + for root in current_dir_ancestors_with(|candidate| { + candidate.join("data").join("data_v4.db").is_file() + }) { + sources.push(provider_source_from_parts( + spec, + root.join("data").join("data_v4.db"), + "astrbot_data_v4_sqlite", + ProviderSourceKind::NativeHistory, + )); + } + } + _ => {} + } + + sources +} + +fn env_path(name: &str) -> Option { + env::var_os(name) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) +} + +fn current_dir_ancestors_with(matches: impl Fn(&Path) -> bool) -> Vec { + let Ok(current_dir) = env::current_dir() else { + return Vec::new(); + }; + current_dir + .ancestors() + .filter(|candidate| matches(candidate)) + .map(Path::to_path_buf) + .collect() +} + +fn dedupe_sources(sources: Vec) -> Vec { + let mut seen = HashSet::new(); + sources + .into_iter() + .filter(|source| seen.insert((source.provider, source.path.clone(), source.source_format))) .collect() } +fn provider_source_from_parts( + spec: &ProviderSourceSpec, + path: PathBuf, + source_format: &'static str, + source_kind: ProviderSourceKind, +) -> ProviderSource { + let location = ProviderDefaultLocation { + path_components: &[], + source_format, + source_kind, + }; + provider_source_from_location(spec, &location, path) +} + pub fn provider_source_for_path(provider: CaptureProvider, path: PathBuf) -> ProviderSource { let unknown_spec = ProviderSourceSpec { provider, @@ -307,10 +497,20 @@ pub fn provider_source_for_path(provider: CaptureProvider, path: PathBuf) -> Pro CaptureProvider::Cursor => "cursor_agent_transcript_jsonl_tree", CaptureProvider::CopilotCli => "copilot_cli_session_events_jsonl", CaptureProvider::FactoryAiDroid => "factory_ai_droid_sessions_jsonl", + CaptureProvider::OpenClaw => "openclaw_session_jsonl_tree", + CaptureProvider::Hermes => "hermes_state_sqlite", + CaptureProvider::NanoClaw => { + if path.file_name().and_then(|name| name.to_str()) == Some("v2.db") { + "nanoclaw_project" + } else { + "nanoclaw_project" + } + } + CaptureProvider::AstrBot => "astrbot_data_v4_sqlite", _ => "unsupported", }; let explicit_import_support = spec.import_support; - let source_kind = if matches!(explicit_import_support, ProviderImportSupport::Native) { + let source_kind = if explicit_import_support.is_importable() { ProviderSourceKind::NativeHistory } else { ProviderSourceKind::DetectionOnly @@ -397,6 +597,14 @@ fn empty_source_reason(provider: CaptureProvider) -> Option<&'static str> { CaptureProvider::FactoryAiDroid => { Some("path exists but no Factory AI Droid session JSONL files were found") } + CaptureProvider::OpenClaw => { + Some("path exists but no OpenClaw agent session JSONL files were found") + } + CaptureProvider::Hermes => Some("path exists but no Hermes state.db file was found"), + CaptureProvider::NanoClaw => { + Some("path exists but no NanoClaw data/v2.db and data/v2-sessions store was found") + } + CaptureProvider::AstrBot => Some("path exists but no AstrBot data/data_v4.db was found"), _ => None, } } @@ -424,6 +632,9 @@ fn unknown_source_reason(provider: CaptureProvider) -> Option<&'static str> { CaptureProvider::FactoryAiDroid => { Some("path exists but the Factory AI Droid transcript probe hit its scan budget") } + CaptureProvider::OpenClaw => { + Some("path exists but the OpenClaw transcript probe hit its scan budget") + } _ => None, } } @@ -441,6 +652,10 @@ fn default_location_import_probe( CaptureProvider::Pi => BoundedProbe::from_bool(path.is_file()), CaptureProvider::OpenCode => BoundedProbe::from_bool(path.is_file()), CaptureProvider::Claude => has_jsonl_file_under_matching(path, 10_000, |_| true), + CaptureProvider::OpenClaw => has_openclaw_session_jsonl(path, 10_000), + CaptureProvider::Hermes => BoundedProbe::from_bool(path.is_file()), + CaptureProvider::NanoClaw => has_nanoclaw_project(path), + CaptureProvider::AstrBot => BoundedProbe::from_bool(path.is_file()), CaptureProvider::Antigravity => has_jsonl_file_under_matching(path, 10_000, |candidate| { matches!( candidate.file_name().and_then(|name| name.to_str()), @@ -467,6 +682,29 @@ fn has_gemini_chat_jsonl(root: &Path, max_entries: usize) -> BoundedProbe { has_jsonl_file_under_matching(&tmp, max_entries, |path| path_has_component(path, "chats")) } +fn has_openclaw_session_jsonl(root: &Path, max_entries: usize) -> BoundedProbe { + if root.is_file() { + return BoundedProbe::from_bool( + root.extension().and_then(|ext| ext.to_str()) == Some("jsonl"), + ); + } + let agents = root.join("agents"); + if agents.is_dir() { + return has_jsonl_file_under_matching(&agents, max_entries, |path| { + path_has_component(path, "sessions") + }); + } + has_jsonl_file_under_matching(root, max_entries, |path| { + path_has_component(path, "sessions") + }) +} + +fn has_nanoclaw_project(root: &Path) -> BoundedProbe { + BoundedProbe::from_bool( + root.join("data").join("v2.db").is_file() && root.join("data").join("v2-sessions").is_dir(), + ) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BoundedProbe { Found, @@ -656,6 +894,45 @@ mod tests { CaptureProvider::CopilotCli, ProviderSourceStatus::Available, ); + + let openclaw = temp.path().join(".openclaw/agents/personal/sessions"); + std::fs::create_dir_all(&openclaw).unwrap(); + assert_source_status( + temp.path(), + CaptureProvider::OpenClaw, + ProviderSourceStatus::Empty, + ); + std::fs::write(openclaw.join("session.jsonl"), "{}\n").unwrap(); + assert_source_status( + temp.path(), + CaptureProvider::OpenClaw, + ProviderSourceStatus::Available, + ); + + let hermes = temp.path().join(".hermes"); + std::fs::create_dir_all(&hermes).unwrap(); + std::fs::write(hermes.join("state.db"), b"sqlite fixture marker").unwrap(); + let hermes_source = discover_provider_sources(temp.path()) + .into_iter() + .find(|source| source.provider == CaptureProvider::Hermes) + .unwrap(); + assert_eq!(hermes_source.status, ProviderSourceStatus::Available); + assert_eq!(hermes_source.import_support, ProviderImportSupport::Native); + + let astrbot = temp.path().join(".astrbot/data"); + std::fs::create_dir_all(&astrbot).unwrap(); + std::fs::write(astrbot.join("data_v4.db"), b"sqlite fixture marker").unwrap(); + let astrbot_source = discover_provider_sources(temp.path()) + .into_iter() + .find(|source| source.provider == CaptureProvider::AstrBot) + .unwrap(); + assert_eq!(astrbot_source.status, ProviderSourceStatus::Available); + assert_eq!( + astrbot_source.import_support, + ProviderImportSupport::Preview + ); + assert!(astrbot_source.import_support.is_importable()); + assert!(!astrbot_source.import_support.is_auto_importable()); } #[test] diff --git a/crates/ctx-history-core/src/lib.rs b/crates/ctx-history-core/src/lib.rs index 8e58d2716..cac38f6ac 100644 --- a/crates/ctx-history-core/src/lib.rs +++ b/crates/ctx-history-core/src/lib.rs @@ -176,6 +176,10 @@ text_enum! { Cursor => "cursor", CopilotCli => "copilot_cli", FactoryAiDroid => "factory_ai_droid", + OpenClaw => "openclaw", + Hermes => "hermes", + NanoClaw => "nanoclaw", + AstrBot => "astrbot", Shell => "shell", Git => "git", Jj => "jj", diff --git a/crates/ctx-history-core/src/provider.rs b/crates/ctx-history-core/src/provider.rs index c849292ed..5010d3b6e 100644 --- a/crates/ctx-history-core/src/provider.rs +++ b/crates/ctx-history-core/src/provider.rs @@ -52,6 +52,13 @@ pub enum ProviderId { FactoryAiDroid, FactoryDroid, DroidFactoryAi, + #[serde(rename = "openclaw", alias = "open_claw")] + OpenClaw, + Hermes, + #[serde(rename = "nanoclaw", alias = "nano_claw")] + NanoClaw, + #[serde(rename = "astrbot", alias = "astr_bot")] + AstrBot, Goose, #[serde(rename = "openhands")] OpenHands, @@ -69,7 +76,7 @@ pub enum ProviderId { } impl ProviderId { - pub const ALL: [Self; 27] = [ + pub const ALL: [Self; 31] = [ Self::Codex, Self::ClaudeCode, Self::ClaudeCliCrp, @@ -84,6 +91,10 @@ impl ProviderId { Self::FactoryAiDroid, Self::FactoryDroid, Self::DroidFactoryAi, + Self::OpenClaw, + Self::Hermes, + Self::NanoClaw, + Self::AstrBot, Self::Goose, Self::OpenHands, Self::Cagent, @@ -393,13 +404,17 @@ mod tests { .collect::>(); let expected = [ ProviderId::AntigravityCli, + ProviderId::AstrBot, ProviderId::ClaudeCode, ProviderId::Codex, ProviderId::Cursor, ProviderId::CopilotCli, ProviderId::FactoryAiDroid, ProviderId::GeminiCli, + ProviderId::Hermes, + ProviderId::NanoClaw, ProviderId::OpenCode, + ProviderId::OpenClaw, ProviderId::Pi, ] .into_iter() diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index b832fadf4..035983ffa 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -93,7 +93,7 @@ pub enum StoreError { pub type Result = std::result::Result; -const SCHEMA_VERSION: i64 = 14; +const SCHEMA_VERSION: i64 = 15; const BUSY_TIMEOUT: Duration = Duration::from_millis(30_000); const OBJECTS_DIR: &str = "objects"; const SPOOL_DIR: &str = "spool"; @@ -471,7 +471,7 @@ const CREATE_TABLES_SQL: &str = r#" CREATE TABLE IF NOT EXISTS capture_sources ( id TEXT PRIMARY KEY NOT NULL, kind TEXT NOT NULL CHECK (kind IN ('provider_import', 'provider_hook', 'direct_cli', 'manual')), - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), machine_id TEXT NOT NULL, process_id INTEGER, cwd TEXT, @@ -488,7 +488,7 @@ CREATE TABLE IF NOT EXISTS capture_sources ( CREATE TABLE IF NOT EXISTS catalog_sessions ( source_path TEXT PRIMARY KEY NOT NULL, - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, external_session_id TEXT, @@ -517,7 +517,7 @@ CREATE TABLE IF NOT EXISTS catalog_sessions ( ); CREATE TABLE IF NOT EXISTS source_import_files ( - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, source_path TEXT NOT NULL, @@ -1385,6 +1385,9 @@ impl Store { if user_version < 14 { migrate_to_v14(&self.conn)?; } + if user_version < 15 { + migrate_to_v15(&self.conn)?; + } create_fts_tables_if_supported(&self.conn)?; Ok(()) } @@ -4875,6 +4878,43 @@ fn migrate_to_v14(conn: &Connection) -> Result<()> { } } +fn migrate_to_v15(conn: &Connection) -> Result<()> { + let foreign_keys_enabled: i64 = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?; + conn.execute_batch("PRAGMA foreign_keys = OFF; BEGIN IMMEDIATE;")?; + let migration = (|| -> Result<()> { + conn.execute_batch(CREATE_TABLES_SQL)?; + if stable_sql_views_exist(conn)? { + drop_stable_sql_views(conn)?; + } + rebuild_capture_sources_provider_check(conn)?; + rebuild_catalog_sessions_provider_check(conn)?; + rebuild_source_import_files_provider_check(conn)?; + conn.execute_batch(INDEXES_SQL)?; + create_stable_sql_views(conn)?; + conn.execute_batch("PRAGMA user_version = 15;")?; + Ok(()) + })(); + + match migration { + Ok(()) => { + conn.execute_batch("COMMIT;")?; + if foreign_keys_enabled != 0 { + conn.execute_batch("PRAGMA foreign_keys = ON;")?; + } + Ok(()) + } + Err(err) => { + if let Err(rollback_err) = conn.execute_batch("ROLLBACK;") { + return Err(StoreError::Sql(rollback_err)); + } + if foreign_keys_enabled != 0 { + conn.execute_batch("PRAGMA foreign_keys = ON;")?; + } + Err(err) + } + } +} + fn create_stable_sql_views(conn: &Connection) -> Result<()> { conn.execute_batch(STABLE_SQL_VIEWS_SQL)?; Ok(()) @@ -5041,7 +5081,7 @@ fn rebuild_capture_sources_provider_check(conn: &Connection) -> Result<()> { CREATE TABLE capture_sources_new ( id TEXT PRIMARY KEY NOT NULL, kind TEXT NOT NULL CHECK (kind IN ('provider_import', 'provider_hook', 'direct_cli', 'manual')), - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), machine_id TEXT NOT NULL, process_id INTEGER, cwd TEXT, @@ -5089,7 +5129,7 @@ fn rebuild_catalog_sessions_provider_check(conn: &Connection) -> Result<()> { DROP TABLE IF EXISTS catalog_sessions_new; CREATE TABLE catalog_sessions_new ( source_path TEXT PRIMARY KEY NOT NULL, - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, external_session_id TEXT, @@ -5130,6 +5170,50 @@ fn rebuild_catalog_sessions_provider_check(conn: &Connection) -> Result<()> { Ok(()) } +fn rebuild_source_import_files_provider_check(conn: &Connection) -> Result<()> { + if !table_exists(conn, "source_import_files")? { + conn.execute_batch(CREATE_TABLES_SQL)?; + return Ok(()); + } + + let recreate_views = stable_sql_views_exist(conn)?; + if recreate_views { + drop_stable_sql_views(conn)?; + } + conn.execute_batch( + r#" + DROP TABLE IF EXISTS source_import_files_new; + CREATE TABLE source_import_files_new ( + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), + source_format TEXT NOT NULL, + source_root TEXT NOT NULL, + source_path TEXT NOT NULL, + file_size_bytes INTEGER NOT NULL, + file_modified_at_ms INTEGER NOT NULL, + observed_at_ms INTEGER NOT NULL, + is_stale INTEGER NOT NULL DEFAULT 0, + indexed_at_ms INTEGER, + indexed_file_size_bytes INTEGER, + indexed_file_modified_at_ms INTEGER, + indexed_status TEXT NOT NULL DEFAULT 'pending' CHECK (indexed_status IN ('pending', 'indexed', 'failed')), + indexed_error TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (provider, source_root, source_path) + ); + INSERT INTO source_import_files_new + (provider, source_format, source_root, source_path, file_size_bytes, file_modified_at_ms, observed_at_ms, is_stale, indexed_at_ms, indexed_file_size_bytes, indexed_file_modified_at_ms, indexed_status, indexed_error, metadata_json) + SELECT provider, source_format, source_root, source_path, file_size_bytes, file_modified_at_ms, observed_at_ms, is_stale, indexed_at_ms, indexed_file_size_bytes, indexed_file_modified_at_ms, indexed_status, indexed_error, metadata_json + FROM source_import_files; + DROP TABLE source_import_files; + ALTER TABLE source_import_files_new RENAME TO source_import_files; + "#, + )?; + if recreate_views { + create_stable_sql_views(conn)?; + } + Ok(()) +} + fn create_fts_tables_if_supported(conn: &Connection) -> Result<()> { match conn.execute_batch(FTS_TABLES_SQL) { Ok(()) => Ok(()), @@ -8591,4 +8675,92 @@ mod catalog_tests { assert_eq!(source_count, 2); assert_eq!(catalog_count, 2); } + + #[test] + fn schema_v15_rebuilds_provider_checks_with_referenced_sources_and_indexes() { + let temp = tempdir(); + let path = temp.path().join("work.sqlite"); + let source_id = new_id(); + let session_id; + let event_id; + { + let store = Store::open(&path).unwrap(); + let source = CaptureSource { + id: source_id, + descriptor: CaptureSourceDescriptor { + kind: ctx_history_core::CaptureSourceKind::ProviderImport, + provider: CaptureProvider::Codex, + machine_id: "test-machine".to_owned(), + process_id: None, + cwd: Some("/repo".to_owned()), + raw_source_path: Some("/home/user/.codex/sessions/session.jsonl".to_owned()), + external_session_id: Some("codex-session-1".to_owned()), + }, + started_at: fixed_time(), + ended_at: None, + sync: sync_metadata(), + }; + store.upsert_capture_source(&source).unwrap(); + + let mut session = imported_session("codex-session-1"); + session.capture_source_id = Some(source_id); + session_id = session.id; + store.upsert_session(&session).unwrap(); + + let event = Event { + id: new_id(), + seq: 0, + history_record_id: None, + session_id: Some(session_id), + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::User), + occurred_at: fixed_time(), + capture_source_id: Some(source_id), + payload: serde_json::json!({"text": "migration source reference"}), + payload_blob_id: None, + dedupe_key: None, + redaction_state: RedactionState::SafePreview, + sync: sync_metadata(), + }; + event_id = event.id; + store.upsert_event(&event).unwrap(); + store + .conn + .execute_batch("PRAGMA user_version = 14;") + .unwrap(); + } + + let store = Store::open(&path).unwrap(); + let version: i64 = store + .conn + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .unwrap(); + assert_eq!(version, SCHEMA_VERSION); + let source_refs: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM sessions s JOIN events e ON e.session_id = s.id \ + WHERE s.id = ?1 AND e.id = ?2 AND s.capture_source_id = ?3 AND e.capture_source_id = ?3", + params![session_id.to_string(), event_id.to_string(), source_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(source_refs, 1); + for index in [ + "idx_capture_sources_external_session_id", + "idx_catalog_sessions_provider_source_root_import", + "idx_source_import_files_provider_source_root_import", + ] { + let exists: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?1", + [index], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing rebuilt index {index}"); + } + } } diff --git a/docs/cli-reference.md b/docs/cli-reference.md index b29b9a572..b00859228 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -28,8 +28,8 @@ ctx doctor --json - `setup` creates the data root, opens or creates `work.sqlite`, writes `config.toml` when needed, discovers known provider history locations, - catalogs Codex sessions, imports all discovered importable sources, optimizes - the local search index, and prints next steps. + catalogs Codex sessions, imports all discovered auto-importable sources, + optimizes the local search index, and prints next steps. - `setup --catalog-only` stops after discovery/cataloging. It is useful for fast inventory or troubleshooting, but it does not make history searchable. - `status` reports the ctx root, database path, config path, indexed item @@ -57,8 +57,10 @@ machine. Current rows include: - Codex session trees at `~/.codex/sessions`; - Codex prompt history at `~/.codex/history.jsonl`; - Pi session JSONL at `~/.pi/sessions.jsonl`; -- native rows for supported Antigravity, Claude, OpenCode, Gemini, Cursor, - Copilot CLI, and Factory AI Droid local history locations. +- native rows for supported Antigravity, Claude, OpenCode, OpenClaw, Hermes, + Gemini, Cursor, Copilot CLI, and Factory AI Droid local history locations; +- preview rows for NanoClaw project roots and AstrBot SQLite history when those + paths are discoverable. Each JSON row includes `provider`, `path`, `exists`, `source_format`, `status`, `import_support`, `native_import`, `importable`, `raw_retention`, and any @@ -75,6 +77,10 @@ ctx import --provider pi ctx import --provider antigravity ctx import --provider claude ctx import --provider opencode +ctx import --provider openclaw +ctx import --provider hermes +ctx import --provider nanoclaw --path /path/to/nanoclaw-project +ctx import --provider astrbot --path /path/to/data/data_v4.db ctx import --provider gemini ctx import --provider cursor ctx import --provider copilot-cli @@ -95,11 +101,17 @@ citations, and import totals to SQLite. Import selection rules: -- with no arguments or with `--all`, import all discovered sources that exist; +- with no arguments or with `--all`, import all discovered auto-importable + sources that exist; - with `--provider`, import discovered sources for that provider; - with `--path`, import exactly that path; - with `--path` and no provider, parse the path as Codex format. +Preview providers such as NanoClaw and AstrBot are not included in `--all` or +pre-search refresh. Import them explicitly with `--provider` when discovery +finds the desired source, or add `--path` to target a specific source, then +search the existing index. + The current `--resume` flag is an idempotent-rescan mode marker. JSON reports `resume: true` and `resume_mode: "idempotent_rescan"`, but provider-native cursor resume is not a universal contract yet. @@ -166,7 +178,7 @@ results without a foreground catch-up scan; use `--refresh strict` or `ctx import --all` when you need a full catch-up before querying. Use `--refresh off` to search the existing index without refreshing, or `--refresh strict` to fail when the pre-search refresh cannot run or import -successfully. Search-only sources without native import support are searched +successfully. Preview native sources such as NanoClaw and AstrBot are searched from the existing index until they are explicitly imported through a supported path. The query argument is optional so file or metadata filters can drive a search. Default results are session-diverse: ctx @@ -201,7 +213,7 @@ optimized for agent reading; use `--verbose` for expanded text diagnostics. Filters: -- `--provider codex|pi|claude|opencode|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; +- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; - `--workspace `, substring match over stored workspace, cwd, source path, or repository-name text; - `--since d`, for example `2026-06-01T00:00:00Z` or `30d`; @@ -219,8 +231,9 @@ Filters: - `--include-current-session`. CLI provider filters use kebab-case names. JSON output and stable SQL views use -provider IDs in ctx output; multiword IDs may be snake_case, such as `copilot_cli` or -`factory_ai_droid`. +provider IDs in ctx output; multiword IDs may be snake_case, such as +`copilot_cli` or `factory_ai_droid`, while compact IDs such as `openclaw`, +`nanoclaw`, and `astrbot` stay compact. `search` reads discovered native provider files for pre-search refresh plus SQLite, and may write newly discovered native provider history into the local diff --git a/docs/first-10-minutes.md b/docs/first-10-minutes.md index 0eddca2da..daf17ea10 100644 --- a/docs/first-10-minutes.md +++ b/docs/first-10-minutes.md @@ -26,8 +26,9 @@ ctx status --json ``` `ctx setup` creates local storage, discovers supported provider history, -catalogs Codex sessions, imports discovered sources, and optimizes the local -search index. The default root is `~/.ctx`. Use a temporary root for trials: +catalogs Codex sessions, imports discovered auto-importable sources, and +optimizes the local search index. The default root is `~/.ctx`. Use a temporary +root for trials: ```bash ctx --data-root /tmp/ctx-first-10 setup @@ -41,11 +42,13 @@ ctx sources --json ``` Expect rows for supported local import providers such as Codex, Pi, -Antigravity, Claude, OpenCode, Gemini, Cursor, Copilot CLI, and Factory AI -Droid. A row with `exists: false` means ctx knows the default path but did not -find local history there. A JSON row with `status: "empty"` means the path -exists but no provider-specific transcript files were found. A row with -`status: "unknown"` means the bounded transcript probe hit its scan budget. +Antigravity, Claude, OpenCode, OpenClaw, Hermes, Gemini, Cursor, Copilot CLI, +and Factory AI Droid. NanoClaw and AstrBot can appear as preview rows when ctx +can discover their local project or SQLite paths. A row with `exists: false` +means ctx knows the default path but did not find local history there. A JSON +row with `status: "empty"` means the path exists but no provider-specific +transcript files were found. A row with `status: "unknown"` means the bounded +transcript probe hit its scan budget. ## 4. Re-Run Or Target Imports @@ -53,15 +56,24 @@ exists but no provider-specific transcript files were found. A row with ctx import --all ``` -Setup already imports discovered sources. Use `ctx import` when you want to -repair, re-run, resume, or pass an explicit path: +Setup already imports discovered auto-importable sources. Use `ctx import` when +you want to repair, re-run, resume, or pass an explicit path: ```bash ctx import --provider codex --path ~/.codex/sessions ctx import --provider pi --path ~/.pi/sessions.jsonl ctx import --provider cursor --path ~/.cursor/projects +ctx import --provider hermes --path ~/.hermes/state.db +ctx import --provider nanoclaw --path /path/to/nanoclaw-project +ctx import --provider astrbot --path /path/to/data/data_v4.db ``` +Preview providers such as NanoClaw and AstrBot are explicit-import only. Use +`ctx import --provider nanoclaw` or `ctx import --provider astrbot` when +discovery finds the desired source, or add `--path` to target a specific source. +They are not included in `ctx import --all` or the default pre-search refresh +until their storage contracts are promoted. + After upgrading from an older ctx version, the first refresh or import can re-read previously indexed provider transcripts once so the local index includes current touched-file metadata and unredacted local transcript text. diff --git a/docs/limitations.md b/docs/limitations.md index 7a637dcbe..e7726f192 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -9,9 +9,14 @@ shipped. - Codex local import is supported for documented local JSONL sources. - Pi local import is supported only when a matching local `sessions.jsonl` file exists. -- Antigravity, Claude, OpenCode, Gemini, Cursor, Copilot CLI, and Factory AI - Droid local import is supported only when their documented local history - paths exist and match the supported native formats in the provider matrix. +- Antigravity, Claude, OpenCode, OpenClaw, Hermes, Gemini, Cursor, Copilot CLI, + and Factory AI Droid local import is supported only when their documented + local history paths exist and match the supported native formats in the + provider matrix. +- NanoClaw and AstrBot local import are preview/manual-path support. They are + not included in `ctx import --all` or pre-search refresh, and AstrBot imports + local LLM context plus available platform history rather than guaranteeing a + complete raw IM transcript. - Unknown provider formats should not be parsed optimistically. ## Import Semantics diff --git a/docs/provider-support-matrix.json b/docs/provider-support-matrix.json index 1e2c928ca..a11a5ff82 100644 --- a/docs/provider-support-matrix.json +++ b/docs/provider-support-matrix.json @@ -215,6 +215,224 @@ "crates/ctx-history-capture/src/lib.rs" ] }, + { + "id": "openclaw", + "display_name": "OpenClaw", + "priority": "p1", + "status": "local_import_when_supported", + "capture_provider": "openclaw", + "implemented_paths": [ + { + "kind": "native_import", + "source_format": "openclaw_session_jsonl_tree", + "fidelity": "partial", + "proof": [ + "ctx sources", + "ctx import --provider openclaw" + ], + "notes": [ + "Reads OpenClaw session JSONL transcripts under OPENCLAW_STATE_DIR, ~/.openclaw, and verified legacy ~/.clawdbot or ~/.moltbot homes.", + "This is beta because upstream exposes newer session helpers and warns plugins not to depend on legacy sessions.json shape." + ] + } + ], + "history_locations": [ + "OPENCLAW_STATE_DIR/agents/*/sessions/*.jsonl", + "~/.openclaw/agents/*/sessions/*.jsonl", + "~/.clawdbot/agents/*/sessions/*.jsonl", + "~/.moltbot/agents/*/sessions/*.jsonl" + ], + "imports_existing_history": true, + "captures_new_runs_passively": false, + "child_sessions_supported": false, + "fidelity": { + "user_prompts": true, + "assistant_messages": true, + "tool_calls": false, + "tool_output": true, + "command_output": false, + "files_touched": false, + "artifacts": false, + "model_identity": true, + "costs": false, + "token_usage": false, + "parent_child_session_edges": false + }, + "redaction_notes": [ + "Imports are local/private and preserve source paths for citations." + ], + "blockers": [ + "Full GA needs confirmation that the local transcript contract remains stable across newer OpenClaw session APIs." + ], + "public_docs": "docs/providers.md", + "fixture_paths": [], + "tests": [ + "crates/ctx-cli/tests/cli.rs", + "crates/ctx-history-capture/src/lib.rs" + ] + }, + { + "id": "hermes", + "display_name": "Hermes Agent", + "priority": "p1", + "status": "local_import_when_supported", + "capture_provider": "hermes", + "implemented_paths": [ + { + "kind": "native_import", + "source_format": "hermes_state_sqlite", + "fidelity": "imported", + "proof": [ + "ctx sources", + "ctx import --provider hermes" + ], + "notes": [ + "Reads Hermes Agent SQLite history from HERMES_HOME/state.db or ~/.hermes/state.db using a read-only SQLite connection.", + "Preserves sessions/messages rows, parent session IDs, model/config metadata, tool call metadata, token fields, and billing fields when present." + ] + } + ], + "history_locations": [ + "HERMES_HOME/state.db", + "~/.hermes/state.db" + ], + "imports_existing_history": true, + "captures_new_runs_passively": false, + "child_sessions_supported": true, + "fidelity": { + "user_prompts": true, + "assistant_messages": true, + "tool_calls": true, + "tool_output": true, + "command_output": false, + "files_touched": false, + "artifacts": false, + "model_identity": true, + "costs": true, + "token_usage": true, + "parent_child_session_edges": true + }, + "redaction_notes": [ + "Reads the provider SQLite database read-only; imported text remains in the local ctx index." + ], + "blockers": [], + "public_docs": "docs/providers.md", + "fixture_paths": [], + "tests": [ + "crates/ctx-cli/tests/cli.rs", + "crates/ctx-history-capture/src/lib.rs" + ] + }, + { + "id": "nanoclaw", + "display_name": "NanoClaw", + "priority": "p1", + "status": "local_import_when_supported", + "capture_provider": "nanoclaw", + "implemented_paths": [ + { + "kind": "native_import", + "source_format": "nanoclaw_project", + "fidelity": "partial", + "proof": [ + "ctx sources", + "ctx import --provider nanoclaw --path " + ], + "notes": [ + "Preview importer for a NanoClaw project root containing data/v2.db and data/v2-sessions/*/*/{inbound.db,outbound.db}.", + "Project roots are discovered from the current working directory and ancestors, but preview sources are not imported by ctx import --all or pre-search refresh." + ] + } + ], + "history_locations": [ + "/data/v2.db", + "/data/v2-sessions/*/*/inbound.db", + "/data/v2-sessions/*/*/outbound.db" + ], + "imports_existing_history": true, + "captures_new_runs_passively": false, + "child_sessions_supported": false, + "fidelity": { + "user_prompts": true, + "assistant_messages": true, + "tool_calls": false, + "tool_output": false, + "command_output": false, + "files_touched": false, + "artifacts": false, + "model_identity": false, + "costs": false, + "token_usage": false, + "parent_child_session_edges": false + }, + "redaction_notes": [ + "Manual preview imports may include IM/chat channel identifiers stored in NanoClaw project databases." + ], + "blockers": [ + "Automatic refresh is intentionally disabled until more real-world NanoClaw project layout drift is validated." + ], + "public_docs": "docs/providers.md", + "fixture_paths": [], + "tests": [ + "crates/ctx-cli/tests/cli.rs", + "crates/ctx-history-capture/src/lib.rs" + ] + }, + { + "id": "astrbot", + "display_name": "AstrBot", + "priority": "p1", + "status": "local_import_when_supported", + "capture_provider": "astrbot", + "implemented_paths": [ + { + "kind": "native_import", + "source_format": "astrbot_data_v4_sqlite", + "fidelity": "partial", + "proof": [ + "ctx sources", + "ctx import --provider astrbot --path " + ], + "notes": [ + "Preview importer for AstrBot data/data_v4.db conversation context plus available platform_message_history rows.", + "This is not a full all-channel IM transcript guarantee; connector/plugin histories may live outside the supported tables." + ] + } + ], + "history_locations": [ + "ASTRBOT_ROOT/data/data_v4.db", + "~/.astrbot/data/data_v4.db", + "/data/data_v4.db" + ], + "imports_existing_history": true, + "captures_new_runs_passively": false, + "child_sessions_supported": false, + "fidelity": { + "user_prompts": true, + "assistant_messages": true, + "tool_calls": false, + "tool_output": false, + "command_output": false, + "files_touched": false, + "artifacts": false, + "model_identity": false, + "costs": false, + "token_usage": true, + "parent_child_session_edges": false + }, + "redaction_notes": [ + "Manual preview imports may include IM platform IDs, user IDs, sender names, and local chat text." + ], + "blockers": [ + "Full GA needs stronger upstream guarantees for complete per-platform raw message retention." + ], + "public_docs": "docs/providers.md", + "fixture_paths": [], + "tests": [ + "crates/ctx-cli/tests/cli.rs", + "crates/ctx-history-capture/src/lib.rs" + ] + }, { "id": "antigravity_cli", "display_name": "Antigravity", diff --git a/docs/provider-support.md b/docs/provider-support.md index ecce897a7..5f9ba1b35 100644 --- a/docs/provider-support.md +++ b/docs/provider-support.md @@ -26,12 +26,22 @@ is: | Pi | `local_import_when_supported` | `~/.pi/sessions.jsonl` or an explicit Pi JSONL path. | Static local-history fixture smoke. | | Claude | `local_import_when_supported` | `~/.claude/projects` or an explicit Claude projects JSONL tree. | Static local-history fixture smoke. | | OpenCode | `local_import_when_supported` | `~/.local/share/opencode/opencode.db` or an explicit OpenCode SQLite DB. | Static local-history fixture smoke. | +| OpenClaw | `local_import_when_supported` | `OPENCLAW_STATE_DIR`, `~/.openclaw`, legacy `~/.clawdbot`/`~/.moltbot`, or an explicit OpenClaw state tree. | Static local-history fixture smoke; beta storage-contract notes in the matrix. | +| Hermes Agent | `local_import_when_supported` | `HERMES_HOME/state.db`, `~/.hermes/state.db`, or an explicit Hermes SQLite DB. | Static local-history fixture smoke. | +| NanoClaw | `local_import_when_supported` | Preview/manual import from a NanoClaw project root or `data/v2.db`; cwd/ancestor discovery only. | Static local-history fixture smoke; excluded from `ctx import --all` and pre-search refresh until promoted. | +| AstrBot | `local_import_when_supported` | Preview/manual import from `ASTRBOT_ROOT/data/data_v4.db`, `~/.astrbot/data/data_v4.db`, cwd/ancestor project DBs, or an explicit DB path. | Static local-history fixture smoke; imports LLM context plus available platform history, not guaranteed complete IM transcripts. | | Antigravity | `local_import_when_supported` | Antigravity `transcript_full.jsonl` or `transcript.jsonl` files under `~/.gemini/antigravity-cli/brain`, or an explicit Antigravity transcript JSONL tree. | Static local-history fixture smoke. | | Gemini | `local_import_when_supported` | Gemini chat JSONL files under `~/.gemini/tmp/**/chats`, or an explicit Gemini CLI history tree. | Static local-history fixture smoke. | | Cursor | `local_import_when_supported` | Cursor agent transcript JSONL files under `~/.cursor/projects/**/agent-transcripts`, or an explicit Cursor agent transcript path. | Static local-history fixture smoke. | | Copilot CLI | `local_import_when_supported` | Copilot CLI `events.jsonl` files under `~/.copilot/session-state`, or an explicit Copilot CLI session-state tree. | Static local-history fixture smoke. | | Factory AI Droid | `local_import_when_supported` | `~/.factory/sessions` or an explicit Factory AI Droid sessions tree. | Static local-history fixture smoke. | +`ctx sources --json` uses `import_support: "preview"` and `native_import: +false` for preview sources such as NanoClaw and AstrBot. Those paths can be +imported explicitly with `ctx import --provider ...` when discovery finds them, +or with `ctx import --provider ... --path ...` for a specific path. They are not +swept up by `ctx import --all` or the default pre-search refresh. + Fidelity fields in the machine-readable matrix describe the default public CLI import behavior and normalized ctx storage fields. Supported adapters record normalized `files_touched` metadata when provider transcripts expose file paths diff --git a/docs/providers.md b/docs/providers.md index e5a8ecd5a..7f554bb5f 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -14,6 +14,15 @@ The current CLI imports local history for: supported JSONL format; - Claude Code project JSONL transcripts under `~/.claude/projects`; - OpenCode SQLite history under `~/.local/share/opencode/opencode.db`; +- OpenClaw session JSONL trees under `OPENCLAW_STATE_DIR`, `~/.openclaw`, + legacy `~/.clawdbot`, or legacy `~/.moltbot`; +- Hermes Agent SQLite history under `HERMES_HOME/state.db` or + `~/.hermes/state.db`; +- NanoClaw project history from a project root with `data/v2.db` and + `data/v2-sessions` when imported explicitly; +- AstrBot local SQLite history from `ASTRBOT_ROOT/data/data_v4.db`, + `~/.astrbot/data/data_v4.db`, or a project `data/data_v4.db` when imported + explicitly; - Antigravity transcript JSONL mirrors under `~/.gemini/antigravity-cli/brain/*/.system_generated/logs/transcript_full.jsonl` or `transcript.jsonl`; @@ -31,13 +40,20 @@ ctx sources ctx sources --json ``` -CLI provider flags use names such as `copilot-cli` and `factory-ai-droid`. +CLI provider flags use names such as `openclaw`, `hermes`, `nanoclaw`, +`astrbot`, `copilot-cli`, and `factory-ai-droid`. Structured JSON and stable SQL views use provider IDs in ctx output; multiword IDs may be -snake_case, such as `copilot_cli` or `factory_ai_droid`. +snake_case, such as `copilot_cli` or `factory_ai_droid`, while compact native +IDs such as `openclaw`, `nanoclaw`, and `astrbot` stay compact. `ctx sources --json` reports each known provider source with `import_support` and `importable` fields. A native source is marked available/importable only -when provider-specific transcript files exist. Sources with +when provider-specific transcript files exist. Sources with `import_support: +"preview"` are explicit-import preview paths: use `ctx import --provider +nanoclaw` or `ctx import --provider astrbot` when discovery finds the desired +source, or add `--path` to target a specific source before searching it. They +are intentionally excluded from `ctx import --all` and pre-search refresh until +promoted. Sources with `status: "unknown"` hit the bounded transcript probe budget before proving history exists, and sources with `import_support: "unsupported"` are detections or blockers, not importable native history. diff --git a/docs/search.md b/docs/search.md index 956668343..aecef8532 100644 --- a/docs/search.md +++ b/docs/search.md @@ -54,7 +54,7 @@ that support it. Search filters narrow both human output and JSON: -- `--provider codex|pi|claude|opencode|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; +- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; - `--workspace `, substring match over stored workspace, cwd, source path, or repository-name text; - `--since d`; @@ -73,8 +73,9 @@ Search filters narrow both human output and JSON: - `--include-current-session`. CLI provider filters use the kebab-case names above. JSON output and stable SQL -views use provider IDs in ctx output; multiword provider IDs may be snake_case, such as -`copilot_cli` or `factory_ai_droid`. +views use provider IDs in ctx output; multiword provider IDs may be snake_case, +such as `copilot_cli` or `factory_ai_droid`, while compact IDs such as +`openclaw`, `nanoclaw`, and `astrbot` stay compact. `--since` accepts RFC 3339 timestamps such as `2026-06-01T00:00:00Z` or a day window such as `30d`. @@ -107,9 +108,9 @@ refresh fails. On large discovered sources or already-cataloged indexes, `auto` serves current results without a foreground catch-up scan; use `--refresh strict` or `ctx import --all` when you need a full catch-up before querying. `off` skips the pre-search refresh. `strict` fails the search if the -refresh cannot run or import successfully. Search-only sources without native -import support are searched from the existing index until they are explicitly -imported through a supported path. +refresh cannot run or import successfully. Preview native sources such as +NanoClaw and AstrBot are searched from the existing index until they are +explicitly imported through a supported path. Use `--refresh off` for a strictly read-only search over the existing ctx index. This avoids provider imports and avoids updating the ctx SQLite store. From 6df3f7ae623ba2752e563ade68844be75043b9df Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:57:40 -0500 Subject: [PATCH 23/72] release: bump ctx to 0.15.0 --- Cargo.lock | 10 +- crates/ctx-cli/Cargo.toml | 2 +- crates/ctx-history-capture/Cargo.toml | 2 +- crates/ctx-history-capture/src/lib.rs | 351 ++++++++++-------- .../src/provider_sources.rs | 8 +- crates/ctx-history-core/Cargo.toml | 2 +- crates/ctx-history-search/Cargo.toml | 2 +- crates/ctx-history-store/Cargo.toml | 2 +- scripts/build-public-cli-artifact.sh | 10 +- 9 files changed, 214 insertions(+), 175 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 906814d59..eabab954c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,7 +256,7 @@ dependencies = [ [[package]] name = "ctx" -version = "0.14.0" +version = "0.15.0" dependencies = [ "anyhow", "assert_cmd", @@ -281,7 +281,7 @@ dependencies = [ [[package]] name = "ctx-history-capture" -version = "0.14.0" +version = "0.15.0" dependencies = [ "chrono", "ctx-history-core", @@ -296,7 +296,7 @@ dependencies = [ [[package]] name = "ctx-history-core" -version = "0.14.0" +version = "0.15.0" dependencies = [ "chrono", "directories", @@ -309,7 +309,7 @@ dependencies = [ [[package]] name = "ctx-history-search" -version = "0.14.0" +version = "0.15.0" dependencies = [ "chrono", "ctx-history-core", @@ -324,7 +324,7 @@ dependencies = [ [[package]] name = "ctx-history-store" -version = "0.14.0" +version = "0.15.0" dependencies = [ "chrono", "ctx-history-core", diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index 996d0ad65..2b7ed2b28 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx" -version = "0.14.0" +version = "0.15.0" description = "Local CLI for indexing and searching agent session history" edition.workspace = true autobins = false diff --git a/crates/ctx-history-capture/Cargo.toml b/crates/ctx-history-capture/Cargo.toml index 9c5967f2d..35751e600 100644 --- a/crates/ctx-history-capture/Cargo.toml +++ b/crates/ctx-history-capture/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-capture" -version = "0.14.0" +version = "0.15.0" description = "Internal provider import adapters for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index e656bfdab..638f26889 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -5434,10 +5434,10 @@ fn hermes_decode_content(raw: Option<&str>) -> Value { Value::String(raw.to_owned()) } -fn native_event( +struct NativeEventDraft { provider: CaptureProvider, source_format: &'static str, - provider_session_id: &str, + provider_session_id: String, provider_event_index: u64, provider_event_hash: Option, cursor: String, @@ -5447,31 +5447,33 @@ fn native_event( text: String, body: Value, metadata: Value, -) -> ProviderEventEnvelope { - let (text, truncated) = provider_safe_preview(&text, PROVIDER_MAX_TEXT_CHARS); +} + +fn native_event(draft: NativeEventDraft) -> ProviderEventEnvelope { + let (text, truncated) = provider_safe_preview(&draft.text, PROVIDER_MAX_TEXT_CHARS); ProviderEventEnvelope { - provider_event_index, - provider_event_hash, - cursor: Some(cursor), - event_type, - role, - occurred_at, + provider_event_index: draft.provider_event_index, + provider_event_hash: draft.provider_event_hash, + cursor: Some(draft.cursor), + event_type: draft.event_type, + role: draft.role, + occurred_at: draft.occurred_at, fidelity: Fidelity::Imported, redaction_state: RedactionState::SafePreview, idempotency_key: Some(format!( "provider-event:{}:{}:{}", - provider.as_str(), - provider_session_id, - provider_event_index + draft.provider.as_str(), + draft.provider_session_id, + draft.provider_event_index )), artifacts: Vec::new(), payload: json!({ "text": text, "truncated": truncated, - "source_format": source_format, - "body": provider_capped_json(&body, PROVIDER_MAX_PREVIEW_CHARS), + "source_format": draft.source_format, + "body": provider_capped_json(&draft.body, PROVIDER_MAX_PREVIEW_CHARS), }), - metadata, + metadata: draft.metadata, } } @@ -5669,16 +5671,18 @@ fn normalize_openclaw_jsonl_file( result.captures.push(( line_number, openclaw_capture( - &provider_session_id, - agent_id.as_deref(), - started_at, - None, - cwd.clone(), - path, + OpenClawCaptureDraft { + provider_session_id: &provider_session_id, + agent_id: agent_id.as_deref(), + started_at, + ended_at: None, + cwd: cwd.clone(), + path, + indexes, + header_raw: header_raw.clone(), + event: None, + }, context, - indexes, - header_raw.clone(), - None, ), )); continue; @@ -5698,50 +5702,69 @@ fn normalize_openclaw_jsonl_file( result.captures.push(( line_number, openclaw_capture( - &provider_session_id, - agent_id.as_deref(), - started_at, - None, - cwd.clone(), - path, + OpenClawCaptureDraft { + provider_session_id: &provider_session_id, + agent_id: agent_id.as_deref(), + started_at, + ended_at: None, + cwd: cwd.clone(), + path, + indexes, + header_raw: header_raw.clone(), + event: None, + }, context, - indexes, - header_raw.clone(), - None, ), )); } result.captures.push(( line_number, openclaw_capture( - &provider_session_id, - agent_id.as_deref(), - started_at, - None, - cwd.clone(), - path, + OpenClawCaptureDraft { + provider_session_id: &provider_session_id, + agent_id: agent_id.as_deref(), + started_at, + ended_at: None, + cwd: cwd.clone(), + path, + indexes, + header_raw: header_raw.clone(), + event: Some(event), + }, context, - indexes, - header_raw.clone(), - Some(event), ), )); } Ok(result) } -fn openclaw_capture( - provider_session_id: &str, - agent_id: Option<&str>, +struct OpenClawCaptureDraft<'a> { + provider_session_id: &'a str, + agent_id: Option<&'a str>, started_at: DateTime, ended_at: Option>, cwd: Option, - path: &Path, - context: &ProviderAdapterContext, - indexes: &BTreeMap, + path: &'a Path, + indexes: &'a BTreeMap, header_raw: Value, event: Option, +} + +fn openclaw_capture( + draft: OpenClawCaptureDraft<'_>, + context: &ProviderAdapterContext, ) -> ProviderCaptureEnvelope { + let OpenClawCaptureDraft { + provider_session_id, + agent_id, + started_at, + ended_at, + cwd, + path, + indexes, + header_raw, + event, + } = draft; let local_id = provider_session_id .rsplit_once('/') .map(|(_, id)| id) @@ -5818,26 +5841,26 @@ fn openclaw_event( .or_else(|| message.get("output")) .and_then(provider_value_text) .unwrap_or_else(|| format!("OpenClaw {row_type}")); - native_event( - CaptureProvider::OpenClaw, - OPENCLAW_SOURCE_FORMAT, - provider_session_id, - event_index, - row.get("id").and_then(Value::as_str).map(str::to_owned), - format!("line:{line_number}"), + native_event(NativeEventDraft { + provider: CaptureProvider::OpenClaw, + source_format: OPENCLAW_SOURCE_FORMAT, + provider_session_id: provider_session_id.to_owned(), + provider_event_index: event_index, + provider_event_hash: row.get("id").and_then(Value::as_str).map(str::to_owned), + cursor: format!("line:{line_number}"), event_type, role, occurred_at, text, - row.clone(), - json!({ + body: row.clone(), + metadata: json!({ "source": "openclaw_jsonl", "source_format": OPENCLAW_SOURCE_FORMAT, "row_type": row_type, "message_id": row.get("id").and_then(Value::as_str), "parent_id": row.get("parentId").or_else(|| row.get("parent_id")).cloned(), }), - ) + }) } #[derive(Debug, Clone)] @@ -5934,18 +5957,18 @@ fn normalize_hermes_sqlite( }); let event_type = hermes_event_type(&row); let role = Some(provider_role(Some(&row.role))); - let event = native_event( - CaptureProvider::Hermes, - HERMES_SQLITE_SOURCE_FORMAT, - &provider_session_id, - row.id.max(0) as u64, - Some(format!("message:{}", row.id)), - format!("messages:id:{}", row.id), + let event = native_event(NativeEventDraft { + provider: CaptureProvider::Hermes, + source_format: HERMES_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + provider_event_index: row.id.max(0) as u64, + provider_event_hash: Some(format!("message:{}", row.id)), + cursor: format!("messages:id:{}", row.id), event_type, role, occurred_at, text, - json!({ + body: json!({ "message_id": row.id, "role": row.role, "content": content, @@ -5958,7 +5981,7 @@ fn normalize_hermes_sqlite( "codex_reasoning_items": row.codex_reasoning_items.as_deref().map(provider_json_text), "codex_message_items": row.codex_message_items.as_deref().map(provider_json_text), }), - json!({ + metadata: json!({ "source": "hermes_state_db", "source_format": HERMES_SQLITE_SOURCE_FORMAT, "message_id": row.id, @@ -5969,7 +5992,7 @@ fn normalize_hermes_sqlite( "active": row.active != 0, "compacted": row.compacted != 0, }), - ); + }); result.captures.push(( row.id.max(0) as usize, native_provider_capture( @@ -6291,23 +6314,23 @@ fn normalize_nanoclaw_project( } else { Some(EventRole::Assistant) }; - let event = native_event( - CaptureProvider::NanoClaw, - NANOCLAW_SOURCE_FORMAT, - &provider_session_id, - event_index, - Some(format!("{}:{}", message.source, message.id)), - format!( + let event = native_event(NativeEventDraft { + provider: CaptureProvider::NanoClaw, + source_format: NANOCLAW_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + provider_event_index: event_index, + provider_event_hash: Some(format!("{}:{}", message.source, message.id)), + cursor: format!( "{}:{}:{}", message.source, session.id, message.seq.unwrap_or_default() ), - EventType::Message, + event_type: EventType::Message, role, occurred_at, text, - json!({ + body: json!({ "message_id": message.id, "seq": message.seq, "kind": message.kind, @@ -6321,13 +6344,13 @@ fn normalize_nanoclaw_project( "source_session_id": message.source_session_id, "on_wake": message.on_wake, }), - json!({ + metadata: json!({ "source": format!("nanoclaw_{}", message.source), "source_format": NANOCLAW_SOURCE_FORMAT, "message_id": message.id, "seq": message.seq, }), - ); + }); result.captures.push(( event_index.min(usize::MAX as u64) as usize, native_provider_capture( @@ -6655,76 +6678,81 @@ fn normalize_astrbot_sqlite( let role = astrbot_role(item); let text = astrbot_item_text(item) .unwrap_or_else(|| "AstrBot conversation item".to_owned()); - let event = native_event( - CaptureProvider::AstrBot, - ASTRBOT_SQLITE_SOURCE_FORMAT, - &provider_session_id, - index as u64, - astrbot_item_id(item).map(|id| format!("conversation:{id}")), - format!("conversation:{}:item:{index}", conversation.conversation_id), - EventType::Message, + let event = native_event(NativeEventDraft { + provider: CaptureProvider::AstrBot, + source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + provider_event_index: index as u64, + provider_event_hash: astrbot_item_id(item) + .map(|id| format!("conversation:{id}")), + cursor: format!("conversation:{}:item:{index}", conversation.conversation_id), + event_type: EventType::Message, role, - started_at, + occurred_at: started_at, text, - item.clone(), - json!({ + body: item.clone(), + metadata: json!({ "source": "astrbot_conversations", "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, "conversation_id": conversation.conversation_id, "inner_conversation_id": conversation.inner_conversation_id, "item_index": index, }), - ); + }); result.captures.push(( index + 1, astrbot_capture( - conversation, - &provider_session_id, - started_at, - ended_at, - path, + AstrBotCaptureDraft { + conversation, + provider_session_id: &provider_session_id, + started_at, + ended_at, + path, + user_version, + schema_fingerprint: &schema_fingerprint, + selected_conversation: selected_conversation.as_deref(), + event: Some(event), + }, context, - user_version, - &schema_fingerprint, - selected_conversation.as_deref(), - Some(event), ), )); } } else { let text = provider_value_text(&content).unwrap_or_else(|| "AstrBot conversation".to_owned()); - let event = native_event( - CaptureProvider::AstrBot, - ASTRBOT_SQLITE_SOURCE_FORMAT, - &provider_session_id, - 0, - Some(format!("conversation-row:{}", conversation.row_id)), - format!("conversation:{}:content", conversation.conversation_id), - EventType::Message, - None, - started_at, + let event = native_event(NativeEventDraft { + provider: CaptureProvider::AstrBot, + source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + provider_event_index: 0, + provider_event_hash: Some(format!("conversation-row:{}", conversation.row_id)), + cursor: format!("conversation:{}:content", conversation.conversation_id), + event_type: EventType::Message, + role: None, + occurred_at: started_at, text, - content.clone(), - json!({ + body: content.clone(), + metadata: json!({ "source": "astrbot_conversations", "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, "conversation_id": conversation.conversation_id, }), - ); + }); result.captures.push(( conversation.row_id.max(0) as usize, astrbot_capture( - conversation, - &provider_session_id, - started_at, - ended_at, - path, + AstrBotCaptureDraft { + conversation, + provider_session_id: &provider_session_id, + started_at, + ended_at, + path, + user_version, + schema_fingerprint: &schema_fingerprint, + selected_conversation: selected_conversation.as_deref(), + event: Some(event), + }, context, - user_version, - &schema_fingerprint, - selected_conversation.as_deref(), - Some(event), ), )); } @@ -6765,18 +6793,18 @@ fn normalize_astrbot_sqlite( Some(EventRole::Assistant) }; let event_index = 1_000_000u64.saturating_add(message.id.max(0) as u64); - let event = native_event( - CaptureProvider::AstrBot, - ASTRBOT_SQLITE_SOURCE_FORMAT, - &provider_session_id, - event_index, - Some(format!("platform-message:{}", message.id)), - format!("platform_message_history:id:{}", message.id), - EventType::Message, + let event = native_event(NativeEventDraft { + provider: CaptureProvider::AstrBot, + source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + provider_event_index: event_index, + provider_event_hash: Some(format!("platform-message:{}", message.id)), + cursor: format!("platform_message_history:id:{}", message.id), + event_type: EventType::Message, role, - provider_timestamp_millis(message.created_at, started_at), + occurred_at: provider_timestamp_millis(message.created_at, started_at), text, - json!({ + body: json!({ "message_id": message.id, "platform_id": message.platform_id, "user_id": message.user_id, @@ -6785,28 +6813,30 @@ fn normalize_astrbot_sqlite( "content": content, "llm_checkpoint_id": message.llm_checkpoint_id, }), - json!({ + metadata: json!({ "source": "astrbot_platform_message_history", "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, "message_id": message.id, }), - ); + }); if let Some(conversation) = conversation { result.captures.push(( event_index.min(usize::MAX as u64) as usize, astrbot_capture( - conversation, - &provider_session_id, - started_at, - conversation.updated_at.map(|timestamp| { - provider_timestamp_millis(Some(timestamp), context.imported_at) - }), - path, + AstrBotCaptureDraft { + conversation, + provider_session_id: &provider_session_id, + started_at, + ended_at: conversation.updated_at.map(|timestamp| { + provider_timestamp_millis(Some(timestamp), context.imported_at) + }), + path, + user_version, + schema_fingerprint: &schema_fingerprint, + selected_conversation: selected_conversation.as_deref(), + event: Some(event), + }, context, - user_version, - &schema_fingerprint, - selected_conversation.as_deref(), - Some(event), ), )); } else { @@ -6861,18 +6891,33 @@ fn astrbot_provider_session_id(conversation: &AstrBotConversationRow) -> String .unwrap_or_else(|| format!("conversation-row-{}", conversation.row_id)) } -fn astrbot_capture( - conversation: &AstrBotConversationRow, - provider_session_id: &str, +struct AstrBotCaptureDraft<'a> { + conversation: &'a AstrBotConversationRow, + provider_session_id: &'a str, started_at: DateTime, ended_at: Option>, - path: &Path, - context: &ProviderAdapterContext, + path: &'a Path, user_version: i64, - schema_fingerprint: &str, - selected_conversation: Option<&str>, + schema_fingerprint: &'a str, + selected_conversation: Option<&'a str>, event: Option, +} + +fn astrbot_capture( + draft: AstrBotCaptureDraft<'_>, + context: &ProviderAdapterContext, ) -> ProviderCaptureEnvelope { + let AstrBotCaptureDraft { + conversation, + provider_session_id, + started_at, + ended_at, + path, + user_version, + schema_fingerprint, + selected_conversation, + event, + } = draft; native_provider_capture( NativeSessionDraft { provider: CaptureProvider::AstrBot, diff --git a/crates/ctx-history-capture/src/provider_sources.rs b/crates/ctx-history-capture/src/provider_sources.rs index 2c3ca174a..6834e29b0 100644 --- a/crates/ctx-history-capture/src/provider_sources.rs +++ b/crates/ctx-history-capture/src/provider_sources.rs @@ -499,13 +499,7 @@ pub fn provider_source_for_path(provider: CaptureProvider, path: PathBuf) -> Pro CaptureProvider::FactoryAiDroid => "factory_ai_droid_sessions_jsonl", CaptureProvider::OpenClaw => "openclaw_session_jsonl_tree", CaptureProvider::Hermes => "hermes_state_sqlite", - CaptureProvider::NanoClaw => { - if path.file_name().and_then(|name| name.to_str()) == Some("v2.db") { - "nanoclaw_project" - } else { - "nanoclaw_project" - } - } + CaptureProvider::NanoClaw => "nanoclaw_project", CaptureProvider::AstrBot => "astrbot_data_v4_sqlite", _ => "unsupported", }; diff --git a/crates/ctx-history-core/Cargo.toml b/crates/ctx-history-core/Cargo.toml index aeea64c24..e6ffa89f5 100644 --- a/crates/ctx-history-core/Cargo.toml +++ b/crates/ctx-history-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-core" -version = "0.14.0" +version = "0.15.0" description = "Internal core types for ctx local agent history indexing" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-search/Cargo.toml b/crates/ctx-history-search/Cargo.toml index 7464db474..0e4d93b80 100644 --- a/crates/ctx-history-search/Cargo.toml +++ b/crates/ctx-history-search/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-search" -version = "0.14.0" +version = "0.15.0" description = "Internal search projection and ranking helpers for ctx" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-store/Cargo.toml b/crates/ctx-history-store/Cargo.toml index cbf8f9660..02d1155a7 100644 --- a/crates/ctx-history-store/Cargo.toml +++ b/crates/ctx-history-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-store" -version = "0.14.0" +version = "0.15.0" description = "Internal SQLite storage layer for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index 16b66e52b..0e7b6cb91 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -113,8 +113,8 @@ ensure_darwin_cross_tools() { } version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; data=json.load(sys.stdin); print(next(pkg["version"] for pkg in data["packages"] if pkg["name"] == "ctx"))')" -if [[ "${version}" != "0.14.0" ]]; then - echo "error: ctx package version must be 0.14.0 for this release, got ${version}" >&2 +if [[ "${version}" != "0.15.0" ]]; then + echo "error: ctx package version must be 0.15.0 for this release, got ${version}" >&2 exit 1 fi @@ -156,12 +156,12 @@ fi case "${platform}" in linux-x64) "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.14.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.15.0" "${staged}.version" >/dev/null ;; macos-arm64) if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.14.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.15.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi @@ -169,7 +169,7 @@ case "${platform}" in macos-x64) if [[ "$(uname -s)" == "Darwin" ]] && /usr/bin/arch -x86_64 /usr/bin/true >/dev/null 2>&1; then /usr/bin/arch -x86_64 "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.14.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.15.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi From eb48bc153e57587caf93c5e6739e1e950bfdb574 Mon Sep 17 00:00:00 2001 From: Atharva Patil Date: Thu, 2 Jul 2026 12:36:22 +0530 Subject: [PATCH 24/72] fix(cli): improve error for nonexistent import paths --- crates/ctx-cli/src/main.rs | 6 ++++++ crates/ctx-cli/tests/cli.rs | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 62b3de0d4..8bbba6157 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -4189,6 +4189,12 @@ fn import_requests(args: &ImportArgs) -> Result> { .unwrap_or(ProviderArg::Codex) .capture_provider(); let source = explicit_path_source(provider, path.clone()); + if !source.exists { + return Err(anyhow!( + "import path does not exist: {}", + source.path.display() + )); + } validate_source_import_supported(&source)?; return Ok(vec![source]); } diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 1d2c61af9..6a32efb99 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -4746,6 +4746,23 @@ fn pi_cli_rejects_directory_import_path() { .stderr(predicate::str::contains("no importable pi history files")); } +#[test] +fn import_rejects_nonexistent_path() { + let temp = tempdir(); + + ctx(&temp) + .args([ + "import", + "--provider", + "codex", + "--path", + "/nonexistent-ctx-test-path", + ]) + .assert() + .failure() + .stderr(predicate::str::contains("import path does not exist")); +} + #[test] fn codex_cli_marks_deleted_raw_source_citations_unavailable() { let temp = tempdir(); From 92d10122292fc6a664b7ca056a3493ce39920b4d Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:23:32 -0500 Subject: [PATCH 25/72] polish import path validation test --- crates/ctx-cli/src/main.rs | 6 +++++- crates/ctx-cli/tests/cli.rs | 15 +++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 8bbba6157..fef707efe 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -4189,7 +4189,11 @@ fn import_requests(args: &ImportArgs) -> Result> { .unwrap_or(ProviderArg::Codex) .capture_provider(); let source = explicit_path_source(provider, path.clone()); - if !source.exists { + if !source + .path + .try_exists() + .with_context(|| format!("check import path {}", source.path.display()))? + { return Err(anyhow!( "import path does not exist: {}", source.path.display() diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 6a32efb99..429331308 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -4749,18 +4749,17 @@ fn pi_cli_rejects_directory_import_path() { #[test] fn import_rejects_nonexistent_path() { let temp = tempdir(); + let path = temp.path().join("missing-codex-history"); + let path = path.to_str().unwrap(); ctx(&temp) - .args([ - "import", - "--provider", - "codex", - "--path", - "/nonexistent-ctx-test-path", - ]) + .args(["import", "--provider", "codex", "--path", path]) .assert() .failure() - .stderr(predicate::str::contains("import path does not exist")); + .stderr( + predicate::str::contains("import path does not exist") + .and(predicate::str::contains(path)), + ); } #[test] From 374a46c5a03a47f720f4ff1c14eca094cb86b08e Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:38:00 -0500 Subject: [PATCH 26/72] harden import source UX --- crates/ctx-cli/src/main.rs | 78 +++++++++++++++----- crates/ctx-cli/tests/cli.rs | 140 +++++++++++++++++++++++++++++++----- 2 files changed, 185 insertions(+), 33 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index fef707efe..fc30932ae 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -4213,13 +4213,16 @@ fn import_requests(args: &ImportArgs) -> Result> { .collect()); } let provider = args.provider.expect("checked provider").capture_provider(); - let sources = discovered_sources() - .into_iter() + let discovered = discovered_sources_for_provider(provider); + let sources = discovered + .iter() .filter(|source| { source.provider == provider && source.exists + && source.import_support.is_importable() && source.status == ProviderSourceStatus::Available }) + .cloned() .collect::>(); if sources.is_empty() { let spec = provider_source_spec(provider); @@ -4234,10 +4237,7 @@ fn import_requests(args: &ImportArgs) -> Result> { provider.as_str() )); } - return Err(anyhow!( - "no native {} history found; use `ctx sources` to inspect discovered provider paths", - provider.as_str() - )); + return Err(no_importable_provider_sources_error(provider, &discovered)); } for source in &sources { validate_source_import_supported(source)?; @@ -4245,6 +4245,30 @@ fn import_requests(args: &ImportArgs) -> Result> { Ok(sources) } +fn no_importable_provider_sources_error( + provider: CaptureProvider, + sources: &[SourceInfo], +) -> anyhow::Error { + let mut message = format!("no importable {} history found", provider.as_str()); + if sources.is_empty() { + message.push_str("; no default paths are registered for this provider"); + } else { + message.push_str("\nchecked paths:"); + for source in sources { + message.push_str(&format!( + "\n {} ({})", + source.path.display(), + source.status.as_str() + )); + if let Some(reason) = source.unsupported_reason { + message.push_str(&format!(" - {reason}")); + } + } + } + message.push_str("\nuse `ctx sources` to inspect discovery, or pass --path"); + anyhow!(message) +} + fn validate_source_import_supported(source: &SourceInfo) -> Result<()> { match source.import_support { ProviderImportSupport::Native => Ok(()), @@ -4660,7 +4684,8 @@ fn collect_source_import_files(source: &SourceInfo) -> Result Result> { - let metadata = fs::symlink_metadata(&source.path)?; + let metadata = fs::symlink_metadata(&source.path) + .with_context(|| format!("stat import source {}", source.path.display()))?; if metadata.file_type().is_symlink() { return Err(anyhow!( "symlinked provider transcript roots are rejected: {}", @@ -4681,10 +4706,15 @@ fn collect_source_import_paths(source: &SourceInfo) -> Result> { let mut paths = Vec::new(); let mut stack = vec![source.path.clone()]; while let Some(dir) = stack.pop() { - for entry in fs::read_dir(&dir)? { - let entry = entry?; + for entry in fs::read_dir(&dir) + .with_context(|| format!("read import source directory {}", dir.display()))? + { + let entry = entry + .with_context(|| format!("read import source entry under {}", dir.display()))?; let path = entry.path(); - let file_type = entry.file_type()?; + let file_type = entry + .file_type() + .with_context(|| format!("stat import source entry {}", path.display()))?; if file_type.is_dir() { stack.push(path); } else if file_type.is_file() && source_import_file_matches(source, &path) { @@ -5014,7 +5044,8 @@ fn codex_include_notices() -> bool { } fn source_stats(path: &Path) -> Result { - let metadata = fs::symlink_metadata(path)?; + let metadata = fs::symlink_metadata(path) + .with_context(|| format!("stat import source {}", path.display()))?; if metadata.file_type().is_file() { return Ok(SourceStats { files: 1, @@ -5028,13 +5059,21 @@ fn source_stats(path: &Path) -> Result { let mut stats = SourceStats::default(); let mut stack = vec![path.to_path_buf()]; while let Some(dir) = stack.pop() { - for entry in fs::read_dir(&dir)? { - let entry = entry?; - let file_type = entry.file_type()?; + for entry in fs::read_dir(&dir) + .with_context(|| format!("read import source directory {}", dir.display()))? + { + let entry = entry + .with_context(|| format!("read import source entry under {}", dir.display()))?; + let entry_path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("stat import source entry {}", entry_path.display()))?; if file_type.is_dir() { - stack.push(entry.path()); + stack.push(entry_path); } else if file_type.is_file() { - let metadata = entry.metadata()?; + let metadata = entry + .metadata() + .with_context(|| format!("stat import source file {}", entry_path.display()))?; stats.files += 1; stats.bytes = stats.bytes.saturating_add(metadata.len()); } @@ -5071,6 +5110,13 @@ fn discovered_sources() -> Vec { .unwrap_or_default() } +fn discovered_sources_for_provider(provider: CaptureProvider) -> Vec { + home_dir() + .as_deref() + .map(|home| discover_provider_sources_for_provider(home, provider)) + .unwrap_or_default() +} + fn explicit_path_source(provider: CaptureProvider, path: PathBuf) -> SourceInfo { source_for_path(provider, path) } diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 429331308..9f20e19b0 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -777,6 +777,15 @@ fn import_all_discovers_and_imports_providers_together() { assert!(stderr.contains(r#""phase":"finalizing""#), "{stderr}"); } +#[test] +fn import_all_without_sources_does_not_report_missing_explicit_path() { + let temp = tempdir(); + let stderr = failure_stderr(ctx(&temp).args(["import", "--all", "--json"])); + + assert!(stderr.contains("no importable provider history sources found")); + assert!(!stderr.contains("import path does not exist"), "{stderr}"); +} + #[test] fn import_all_skips_empty_gemini_source() { let temp = tempdir(); @@ -4585,27 +4594,36 @@ fn personal_agent_sqlite_imports_report_corrupt_databases() { #[test] fn native_provider_cli_requires_existing_history_or_explicit_path() { for (cli_provider, expected_blocker) in [ - ("claude", "no native claude history found"), - ("opencode", "no native opencode history found"), - ("antigravity", "no native antigravity history found"), - ("gemini", "no native gemini history found"), - ("cursor", "no native cursor history found"), - ("copilot-cli", "no native copilot_cli history found"), + ("claude", "no importable claude history found"), + ("opencode", "no importable opencode history found"), + ("antigravity", "no importable antigravity history found"), + ("gemini", "no importable gemini history found"), + ("cursor", "no importable cursor history found"), + ("copilot-cli", "no importable copilot_cli history found"), ( "factory-ai-droid", - "no native factory_ai_droid history found", + "no importable factory_ai_droid history found", ), - ("openclaw", "no native openclaw history found"), - ("hermes", "no native hermes history found"), - ("nanoclaw", "no native nanoclaw history found"), - ("astrbot", "no native astrbot history found"), + ("openclaw", "no importable openclaw history found"), + ("hermes", "no importable hermes history found"), + ("nanoclaw", "no importable nanoclaw history found"), + ("astrbot", "no importable astrbot history found"), ] { let temp = tempdir(); - ctx(&temp) - .args(["import", "--provider", cli_provider, "--json"]) - .assert() - .failure() - .stderr(predicate::str::contains(expected_blocker)); + let stderr = + failure_stderr(ctx(&temp).args(["import", "--provider", cli_provider, "--json"])); + + assert!(stderr.contains(expected_blocker), "{stderr}"); + assert!(stderr.contains("use `ctx sources`"), "{stderr}"); + if cli_provider == "nanoclaw" { + assert!( + stderr.contains("no default paths are registered for this provider"), + "{stderr}" + ); + } else { + assert!(stderr.contains("checked paths:"), "{stderr}"); + assert!(stderr.contains(temp.path().to_str().unwrap()), "{stderr}"); + } } } @@ -4743,7 +4761,32 @@ fn pi_cli_rejects_directory_import_path() { ]) .assert() .failure() - .stderr(predicate::str::contains("no importable pi history files")); + .stderr( + predicate::str::contains("no importable pi history files") + .and(predicate::str::contains(path.to_str().unwrap())), + ); +} + +#[test] +fn pi_cli_rejects_wrong_file_import_path() { + let temp = tempdir(); + let path = temp.path().join("pi-session.txt"); + fs::write(&path, "{}\n").unwrap(); + + ctx(&temp) + .args([ + "import", + "--provider", + "pi", + "--path", + path.to_str().unwrap(), + ]) + .assert() + .failure() + .stderr( + predicate::str::contains("no importable pi history files") + .and(predicate::str::contains(path.to_str().unwrap())), + ); } #[test] @@ -4760,6 +4803,69 @@ fn import_rejects_nonexistent_path() { predicate::str::contains("import path does not exist") .and(predicate::str::contains(path)), ); + + ctx(&temp) + .args(["import", "--path", path]) + .assert() + .failure() + .stderr( + predicate::str::contains("import path does not exist") + .and(predicate::str::contains(path)), + ); +} + +#[cfg(unix)] +#[test] +fn import_rejects_symlinked_provider_root() { + use std::os::unix::fs::symlink; + + let temp = tempdir(); + let target = temp.path().join("pi-sessions"); + fs::create_dir_all(&target).unwrap(); + let path = temp.path().join("pi-sessions-link"); + symlink(&target, &path).unwrap(); + + ctx(&temp) + .args([ + "import", + "--provider", + "pi", + "--path", + path.to_str().unwrap(), + ]) + .assert() + .failure() + .stderr( + predicate::str::contains("symlinked provider transcript roots are rejected") + .and(predicate::str::contains(path.to_str().unwrap())), + ); +} + +#[cfg(unix)] +#[test] +fn import_reports_unreadable_directory_with_path_context() { + if unsafe { libc::geteuid() } == 0 { + return; + } + + use std::os::unix::fs::PermissionsExt; + + let temp = tempdir(); + let path = temp.path().join("unreadable-pi-sessions"); + fs::create_dir_all(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o000)).unwrap(); + + let stderr = failure_stderr(ctx(&temp).args([ + "import", + "--provider", + "pi", + "--path", + path.to_str().unwrap(), + ])); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + + assert!(stderr.contains("read import source directory"), "{stderr}"); + assert!(stderr.contains(path.to_str().unwrap()), "{stderr}"); } #[test] From 4ac0967193fa65b72097857cdd323f682b114c98 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:54:54 -0500 Subject: [PATCH 27/72] Update README banner headline --- README.md | 2 +- docs/assets/ctx-readme-banner.png | Bin 78104 -> 116877 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f639899b2..1f3e4abab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -ctx is a CLI for searching past agent sessions. +You have months of coding agent history on your machine. Search it with ctx. ctx is an open-source CLI for fast local search across your past coding agent sessions. diff --git a/docs/assets/ctx-readme-banner.png b/docs/assets/ctx-readme-banner.png index bc4d4c346594b408f41f4c0edbf9bc1d379a5096..6ebac3e10b2df9adfc56d7042d04fcc74560cded 100644 GIT binary patch literal 116877 zcmYIv1yEbt_cgA?t=6kI(N6pRNF4CI|2ic?f5C`>3hNij{Y zoYP!*6I>aBpo3&gon%ZiS!}YFb_Ey4ffDgQv+AtOssnm->c=q(UnvG++S?IgaXcx} zM_%1lc3ajr&bv0&It`K^k{l2AEw{S71=E=n1 zQ5&Idm>WAXbzQD*uCw^o+pW`eh>PqAvYZ?pYcs(*r9Q0adm#3a0)W(l#79q(4B2nZQWB;)}#1decBTCq;?*UF|&Mpsh`WEeGPhT;Zw176SX6s zWZBsfbKRo$Mdn@__!-Z0jCL=Zw4>u3uh`dMgHf}l`#f#H7hcW_ae6^RCu$L|(%qou zy7=W0Thqv?llS&#mz(W|4L7PCvFsR!o>|i*l1w}=sGM5HsePe}Zyg5U6O%GkBU0xq zc0a?f$3lnX$=bz)=q`e$B_!mo!x>GMyYTPvB@u zxg5;1CHg#U&O^DJLYbf1ok%^ij-(S4YLoDWgVb|#PjOfC>XeHT7;3GY2!({ z@0`PMs8T46Z*@AhDp8#?f(ho@W2cFg&}Q(GroR_WeT%9K`=}>Nqe^q>x-)_0vXxjI z6ucMzN3BjGXO2Mr-)6T5X=mQ zkSzcJh*|ITJnc6Wd*6JaC!Er91SpU~$dGs4f>^pHy&h3q#bKyenEfYhiC}^jbL5!w&Y>SeIk7kaitLHSF&x zF$*0;B$JZp4#x~>p3u}pA}6pu@kL|ScM>@tC@8q?v4o?c^NPHVAzO>NV^V-vVWdNl zQ;_^#Liyc~8?|^gO4$Wz#t^DCS@98Fs3anIh0>?!a$F-Q6=KNh%(&ipAJTm?a$x|v zVul92FvzNKtOblYxNo#qg{aU#mLH;tk6X{tGLPI?lB{d+Wm?b7eFE{ojJ3A%0V)3 zRp+9LUwcH6#nap9G3!axoWp|4;#=BG8rYqwi1TKQ#@NYl**s=)DQ!xd;CGkaliTnN zV@j%L?M~uUuBB}a{ahj!eF9R={GtifNgmesh8{aKccm3`*Bm8~e>5($l9L@RzYcXr z(@9P2qG5=V@P$^LFH?;`w6>w-dhYf$`BRiYdU~EHpU+HNGi+11-J~HZ`lCI6Y|4}= zJLt2Vt3ec6Fu!vSYYo>-3pUzP{F#Bo_Tn6-FG^F*PwkLqIx!nqEK^fteNb4<*sup@ zm&2a*V%KfvQj zLt>xEkZ=gUz&*!nOidyeGWZL*ys==+np)D6i~>j(Ra=Nqusya*d{MaYPaWb&J#tNl zYSv)_(xSwYYmw1M8JVZRaoA?o0JWNgFd?G6=?y8|pxrOQJEX_^P@Yuv1+mEoIFm>z z_FLj8*6BZ42EU{Fv8IR{;01l>xa@HDW3=<-ycEa$3Z}|vje?ooT;Kg|O``^-{u>k` zKg!(SQ#W25bzsvHCa1bIVgKFi>}H`y>R{bC;bS8J>waclDX#`ftL3G(IY0F7cUN9BXXsh+(L@J z7LbrzXqg7;=VKHm*}X>0D(c_GU_OdE&Bnkn8-`(z1|&EjOM>wVPMmyY&Vz5aH3}uQk z>HzTZxtVZL!p|kG)YPyhbOJR4OEeiKCMMizoo$3QjO=_e>c`7nQ1!5U=Cy zbZx8!s=vhq861V2+rw?^GCWO|x~FKK(5C=gWajxb0Qhr409D~N!k@MD*?UbOh2@QU z@*()ejJ`aIThnLx9RL26nos#)tjIisV~pm8D4#15HOBnI-$5e|rhIznY{p^!aC@=! z`U9)F=ZxZ&elYdjE=&@;Hks;sV;j8|qqIbw+>fm`97@}4|#N#nZ^|KbEzwghcr_-OOC*0{xaFwLh+IjELVPMX4(6B z{Rj1jZ?QkyQEM_9+ZD7tGbOc*{-zTAKLi?LtEqm+P*J(gBNxZPfKenQ?qa4IFx~by zjY6C$C!4W4)Xe7DbkFy@TPcBwU$$k;R{)jKZ%Q8ENY=8wYBaD zq?}<*>~i4*g(ZO|4VW+TGXK;Aq-4s4WL++M;i-13LM&Y2H{8%9_Bck_S>=9PG9&u} zg~dqWiFGby%)T_gcP7{bd>lzM17u1TNYE#(dZy}s+cfK-ZhcsgD(Bh(z#ADyK@a3o z7SiTQpi^l88xZ%STG+zzG5wv`FTt9gdcQ#tW7P)6KVq?haI&Znv3*osl9d7p=2i^* zquD^(u6A_!-|^i4D!8O6oxN|;T3Ssgu-HFWy6>}qeVLe_>(dTo?MQT(H6-<#g^*??RpaLemaA)YrR#(x z`-sCGjKylibwG|MTBJJ77Pcd)$H&{9uH>HBhMAYwZFhT}8efSOs#w}Q6$$_w?JTOK zMQ#KOX~-4uFK-y$_))_6`ZX)F>-D-3&+opK@ST#0^I-YJOs`xt22t9R@W zU;Fd;inB}9+=}y@SQmFa+WTZ1%T?6r51))c2}Kt5y4jX-nYL;&c&HVmB}a9YNbo+ zFCxKJI3gK(<`(eKe>hBIgom&|M8OQ%o1`kKPaP{4_CJ6WMFWAPP4}fc`JSI~C|+?W zhB#oXvHJRN-_5eeskgn=Q=Vk0+-YIYd$ap!2-V{#Q{E4FrurJUV1f+idmhqYC2pa$ zx8Bv?3MS2Iqz0{hIkbE?j^$D)=X>_y5BRXXoQitF+!>MjkX~=Z6M9v*YXeQw$WaYj zdzyDud!C%((pg&AIBkqbOG^t!?A$f2sNjPGd!{If8$<> zS7b=$QmgRcmJD9Q&`=tW9~tHY+ArQxI7==x05KSm{+~r5!C?temj=f4i6fWSDi@ci zNh5D{P$m7))zo2n0Y~kitQv;FeFsof@X#j>e@=YAFCgD@4(_1l^OzS;1W*ECjAUSI z7*4ziHpqSo5P1<3dNI3Qu-t+bn}y{dnn|)e5VS8Ng)F~!GgQ4V!=!?n_&+b8hKpVo zEwlZ7Gh87=xpRZeHw@#pwIvk?&omg6iWf$|K>9MIOnb+t$Xd}*9j&{{FRKeRwhQ#5 zbVCND7wcr|2+oFLTqOT)O;`bHvDfx}W%_>AM;7opSd7`3s~i>t`(F+c9LhWaK%!uK zF`26PMY4}kH2$m*h%UL zHv@V-EsnnXZ3@6bw7j8U{)=(dZd} zJu6vpZM;7E3d|+XytW@&xEs_Y)yWYq9zRKzAf50-_Yb;fUw6#X!OYG}^ef2qKAS3D zd4lTc3Z@aiiJnX=uS%Qtv*8ZoO^_>TO|Hu*dDx^d6{ZjTumflr?$avW~Vzhp<@+%9A$s4f(B z*;O<9y()$U?4v^UV}=YDfcQ^O3~8HMHb_=Rb?hV`EUL=SMwu0-&R#_OYs;7%Wky7> z~-xpY*j4vumKwVzbi47 zE+fOu%5$_eTH|2K43NECwSjU*e3Dk)*4r{zLwg&!vb*#sBjok(lP!DwQKRjPsD%0b?7s+yq{m82`SJBx*r^+_; zdZ>K&>C=3@Ne&~gDnr`-{{DFX-xgh?hfC?fVjqynMW5-x#K53l#%-m*XKp)UYMF_a z0WQG^)yr};yiH%?$M^K^e6n7Y(m^yKG!&>mtI|<{*n-zhx8`dQWzSry&+3&mqrogA zsi&awduD-quRdZi^H({2q}E-YMd9P6wwPp-bf*Q-eY#D(@piYn91u2}=hTZkS`5 z`CbYpRtLl4N^tO9+NB`TmbTBF^8~jqSw!)Q59^?0g&RioWuti6lGQgGGv^O%;XsQ< z!91nUrFal-@8_C1U7|-`;z$bfM{o8iuwV!gF8ViaQFL(LDcTyevz2=;c9m+AGGDn^UgCE=;ZYhU+uj6AjgnlSvEUnpW8B`jcEJFaQmrWs&B7i}rBZYp zmAa=EA8b+IkKwEI3ctFvJ(j*Plp5h_e5i`H#LB3(&QJqAih^twkHa=AGT!02=79b?*EPX@2bS&kg48RXnXJS zwcT@^BMtW!MTA47dTtdn1B2NMDCj1^o;vfBRhu#UW1BeF%u7r<4-e0hP>!B(uy>~G z*i2QzX0(0OoBcJn*1|$0J@|WfIy%6$@pPQrSb>HrKS=@zz-qcxBTzj!k^eY%FkRl< z;n=mWHh#+&c)jnI&t&#xKkL|Kn4AfTpVA&1s1K?Y*3${wOb0V&I)C?@NFT&iZ5OE! z!A4Wzq*=|Y{g7tqJ>k@JT7lP;aQHh3Kf}8OB^h=;vNTq=!>DQ3`J^Jf?juOA^bj9? zJVS+J|FbY#zRjb(PT1l=`9lnT(v*k8@dTnRooAP0Jl8XoeP#Kioy+~a13 zrBmhX9?nmzbj>%;Pr;9pU=ag)y{O*V!K~q!kSWU4vnpFyDwB6zGXDjS0Y0RI64@tF zyAkm(@}v0Q^aDb(Z9kr?S=%pcTmKj1iutI+b~Dcm)XbW`CbmvOrb!GxR=%(vi_zDl z_xL_oEvV!L?t^#ul#-%;YMg2=G7DSSL2jj)~O($yfM1Ga8DcMh(jwKh*u7yO*ml@V&h#D_iX^F<{rHW zkWS{u_aDBfdk446X|vL*w;0ab$R+)-(EM?YbuOTXr}qDEdGioqem+~Eu)@EP)z`_SfDAU{c&4j!=iZki*2#>77`T;`B41D#G)(O zBCx-{3QdHkj}_qRM9)MW$ARQXB$#vsJTM{;%F*iWjfRR;Q#3y zAa;g)2ubl%@LIU+XQEI;=@l9vpY+Ft&(q)!lA2DPcsZp1GB0LvRwzkvwXA7|dWPEX zamv5F@D4;3Fv6$fG9bRG9lG4%lfmo=ecQtn?U4f|hY7V%voCw52KkFkjNf+GK%KKh zDw;5l|6Ui1HvK6P4SkS1qIRgCY51Dt%pZdN^DQTA&N#NGqZQ77H%$U^Z@lbRWJQTw zSSJ95W}0&*87TjmQO)l|1AIxRM5rlQG7?a$nw|y8lRZR^p=2(+%mNxHaAz%CKs~?WRUio zYC6GZ4q#0j``ltb6f(AyTv5fDjaWd&_s-TvJ#A?|ekL+iQQs{U;%`5EvVXqbKN+2w zSGw_YNu6QL=c&nSS^{>%mAbp{3sI=*8zv|jj-^~%#n(g3Y`}jmUQk zx40JT!DJTm=Mpj4#mrOR>(U*?_pI|=6dOvatZLZfZAPeJ^9_9F^d34#lUeXLKUZJd z^lsvOOlsV0upB+I-7JdB*M`Vg9NrF%^mSt`HLfa!-@P%v7}5SkCb*xXiDgt+7fiYp zi4%0<2GZoSP11r|KDXfMWm^jlI==3In@7k{h~Sj-0}jOEtTOAPAixrxhd8So#Z=jE z$$|5Ni7nUb*g?J!`)Ok78SkRGZNN)`2&YB6D4uM16_6=E2~`M(FR95RgA5`uPWkR_|MMAVURVV8LYhgv@ zGtC?b{42jh$f8uI{WV2q;mP)Zgj!G_!(5Z9=FJ3NBFXvF zb^$rX9?$uAr!Xk2^baz<2lDbzc}0nUqN%fKQ7374Vbh1&UFHIP>`zA6VI=d3Pciil zUnlvm!=_kqlVPu3sHA~tkVsTavi*Ex=x>A0jB z>}}l9uK~nvqVJv!oGPevLE5Af}C@?V=)zK?g-UTlHL(PX;)x zcUYbVhCh#EoX2Rlb)Cdqd5e)5RGRJYHP<40{A5QE8==OX{x$#=zH)B z9;_C22}13)`e*)ai6E0Qq52e+W&Bdw`T{O#M(yB=D;cl%^p=~PTN7s-KC;zh|s z$r88TDKMPK2j*NC%~=-jTNFmo3~*#?nsSVge1EHYl~O6p#v)1lO?FLn1{)~#4Xzt~ zgv~c|sS+hsYDI-S8Z9p2nt{183)DHc-?7?Kyk-^2#}>%{6?H`x)bpd(mS^gFh|SY9 zO|*T4{#dCq_K!3DcLo(x^pbVAPF-~%s0z_cuw@736&@x~v+7<3(pBfys*E-6SuX#VQAicaJ1}Wu_jdry4Gp#Q1MvbwHL|%3vXl*d^@$#RS{dEX8yM<( zvZAmJZroS#zYvZz9xCyGtgb7lP}M+6R#BA1#?EZq)^E{RsS@uJhFy6a@a;d!4LO9~ zkRfEMw0)iLe!q)E7y}mjFM47vGjUStcV1m=o%q1jjrH}MOgXsHBRC<>$KmYKz1WzT z_gk<}Nz7<22Rn=Lsft-{C$J(c9p^X&9{FiiSD1cTgs`ytG@SIh45{d)d$UX9)AY!svhT#6+*uf|`}TfQaMbAz}mE|9}y*-?_4lO+bK5NEXlZ(cn1Oug`VYMwZ)eZbk0sayt(A!?D5pBdY?2Ds!5EhncKHdS*e?c z^nklCq4U0eUGQ(0tjWLY@tpc`0*X{O)&%(1W3YZ&H6X0x5e3-*(w*RC4g7`<|pU+H15Q#s1^9% zft|8&En~jGFAb7!@_GlCK+s7^e7P>UOqM$KLKQ^;u$#Y4)EtBt#oo zSQCv!=3(VoY)m$4h2%dVlCMJYvp8x@YisNIJh{VFkDodR=cHoY$)vdJ0Zwl87=(Gw z{fed%)SS9D>D8ED(UvqGTG~A?o?e@~zci*>!ORSWcpIzuc!v;6bi5y-PcIT)$2C-^%8v1d}ujVESe$ly5W{O5{lWp&f;h?**%;<^!d#3 zb07+o5v7O2e4rxRRZ7?7#~ttnO>_{O*7BQIfst{72)PLg((fQz6OPxNJ{lQqCi>d= z0X7wsc=El03<|1$)+OwSy*I3?T3*CpBVTTJSoJmwd$=|n*BkRbQf(SlMelx?of`7P znId3+o1JL((uSFdoc80JHUt_~;3!KUzB5f4eo2&8o% z(Z9IW=jYY6UmgZZfQ$g!G{R}P-(WiVYUyyX)7XXWmg@YUs{0!ZUM0lutP^KiPKW%d zZT5vpBxS7Cu`+?1N{_xv7(6&Iy%iB<$%C`o> zKs8R{1SJ#n8D2^Nb?ca2z*vvAG^$*$tGIT#RV=-HRGvwp-d=0kEh>`rOvu&*{YMt< zxe3Vlt$3E*dAp#Z?hQlTb#k4RMA%0*Y8 zq7MXyeY5IIXlZgEW_bo1fBxl87n@iZjob^%8?1%wk7DWm&C5eWYZDWlk~6c`S{6=p zX;W6HW3!#U4=b&9?bsht%)lxa!k*{r+TDl?NWe;y7XX)Aa&k(*hT{^s=_?_07%ON>`foOT^_PQX(Dj#%`%n$b8zw z3~Hmzy-uet;PG=s@{~;I?xS^L*W7BO_fA8ZmjEYpqkhyI*H%$9gn>(yqsQP5LA2ND zWxFYLbVsw43t!Ti^}W4)D%9n@?B~&>+wp1UONU_Z5n$~8s)}|0@$oss{BkO9vnAl{R|)Fv?afno){*{o zUe97F&yN0=t}p)5nsv|RROm%*MZDVO2lu!Oukv9L({$j&{V+0F>LKuV#Qv(3U%RIi zx1cgtLWgJ;@B)l`xynAc;P#JEK7tKxTck!=!?$lmU@*{7x@mS?23Xb)ERoHg?!8Uh zkkMpqunO$B%pQOok+N_~lean?HBx2Z{spS+B3urf1`)4CIs4olr{w0=xCWC+-jr3N za7rDmlX(mCe9|=FdcM&;WU{45g=_$?ozL8R3tY%lSJ_Ow=-f@-iJz0wuzt+iz`nv6 z{gL_v^H*mh|7RN;__%)Mxj(PX7qca?^z9^$nh6y@Z^WIiQzsGbXxSMZ3*$T;BWR6~j@^ zAzzT>duPQ*^Cv2SresL!%xcb?d!uFHTi3rWI6ib_YTM=H=9)4m`n}$*r&YbReE`Y` z5f!hu2VV3?e}7O}XXE-DM^ZgkpI$Ap(A$pKF z^3Y!4x~95%D2@;#z2<4f^h~cw>mnccE2@&3anw&^=a<=^S5I0oMU?QjbXAtJq6=I| zK}CSg1MiPU2v80hv~ok$q^mf&4Q(sm5Uqo+kbfToDx^QD9$E)yvY7YM$$cqBq=R^g z>&J&O60`O!;iB`{<&KacL5GdaiR?;~)xl`@BAX4fm6~{?Far!MYyyA}K#VisVbJtK z#BJ;MwnEU92{n3JzVO>_fRvkwxoNll(eGHO79U{Ixa)ktd2cm)t3NziZ)ssFYvV1e zR|l8?(4_1fiYFi`N`T`AhuA?~u7^QK5UVDyX%HA*rllsWFAk1F>D=-5@qq@%L9#ll zm{3M(;()h42~+EoP+?U$=wv=t7WH*6Hu%fP+|&>-O-^6X_dZS&4)2gsuUO6Ed*g17 zu)FKRXg9yd@yp9XafoxV49M#$lC*mVPr$loyVzV_{wP;Uji0pqseJnPk)Pb#L^cmP zsj#eKS`mtE44yKi7ect##v&mB1)E~QRm?+<<8T<|zB~2o7>@%MdJ6K-?j(>@kjcr( zd9l`fFD|7TTY|xYgSqG_6Zd=16y>7!-frG1q}s5#RO{rUwzhUh-U)E6)h#2?^I1wd z!VQV5x&A$$Ybw7-`BQVqI4iQkP^bk*Y)tH8wZ783!O2bX9+94vk(HSro0C6&jLvch z(DDd8%Rzp~)m7cv8c8e+#u`I%B(P+fAph=o!PCXV1?XC7>X?+5l$Vp7oRx)5K#-lC zEfE*h8TuI_2g+$`YilPbS-yY#&ewWttnA_8q35%@(cX+)1k0UzeA_w?tS`HFE4C68 zqGaX^&+AK#x@1#tD$s(EWOxH-XFRW-zP^>Y8En||`c2mY_bzRuM&amSZV8hYU zW*|UAbKAAe`bOo}FblClu*$}!LQXEaqM||)1sCOzSjGGR3dp*L9Ah zbu~L~t+%>8-xHH5#PmT_fzFiCnE>!~x%RVzipqI!p(%MGd0|GK(Q0=f>lrjAIpF02 z$ssZ^nNj=g@fG7`?iBFybaV0gbQhVr-e|M4gnEDz3{iX?yw@cp8@|>?x<1iyaV>5S z%Bqlg!&0rF*JY3_uF!r%_ftOcieA|PO?2W1G=pq@&G!H_V(kT+%+44DW7Di zo*V(+nTdtms9!yAl=jAVM@uQf*1#LIIF94<^S5&W(qU6XUf|(Ujoridv(;wDg@`U+ ztv|`)tW$YHsHhmemzM$48v;Cz8!dqsB=R-5eH2$bcN(9zHY zJkQ6wUVXPcVU+@&_Y%7NjmZ$|ok%le77Bj6_EBoq9pfU%kZP4tUx+}u^-I&(-r2>h z_weUOYU)CksCJ+9UaxnYZYyB@M4lcA@`ZhhyKk&)Y+jd01Nr#ct)=i67$cRpI-fUz zPa#=I#ad&FV3`EF)y9e-&o+y-kdP2mf+>fUw#=?#S~KK#_8q_1ED|YeKV2;ahd#q0 zJ1y0|X5>$W&+iT8r|0wgClB}KupTtotbueQZwMxtpB=yMgFv834dZ9yI-|}YD=Kd0 zyTh-2KE4pCQ)QK3C=m(&n+J1M{O~)O$e+QQ6IzwL4%a>GOXn0CWkOCLfTVD)$sgyz zXGx}8iw(<=`~?`agLM}y`FFFp&n_nQf!vvEpj-e zm&bf3b1EUELi6i0QzF}E&6~lEDeLR&+;%!70v@UQIQ)MEU3ycy{GXxE@bO9Hon)43 z4awCx%`e$}kf%ogSXL>Mu%vvhUSo?&{ZNN^>^8Hwo1be8?v?@(8=aDOhE$3#o~vu@ z$yR4=ax}_RF5Wz?#LWaHzq;Ff1DT*)F{Ysg!$LX}( z_4l&PSz)}~Q(7dm_llvj&uSaL;zE-Qq?;;r9K7{D)d%!tsR-v8o(ndV(W>}7R`KN@ z;c?q9JJa%07OTBITC2)TP7<$$fe=m>9(*lEtAL_NGa62-qob9c7hQ&JKmots+jf+)e{d&1G;b9Ftv5TUb8(#A+(PrQma=MhhLld{zupu%b<4?P z-%M(u^5X*Tekr&gFXBPwW{ab%ZEi!gw+UUlM$M*Yls%Mx4!=@DGE0jNujBe;;pXQ0 z7W+(V;k$$Hf_38M67{tUoFCWWxslzmU7hFoyw+`+R2h!Qqt{}6VPCW3hU?9uyP3Jk z>gwvU?YX4y+Rl)&W(nEz>T1{XJ)*MmBv3C~36-D4|7c*QGMcyk?&TGx>&0&pl=}zC zoqD^#?SLhPw(IoljPl98jduya<0#%8z&efmA`yv>IRgcyY-=wfaOXRX-@%!(O}Eb2?os^VR8TZ4wBTbItPH`~pq!&;-?EIs^c zK<{4_+9(Q#CBEViT^58@U(k6Ua}Cek`!lXv!0Bt%Qr-UU9`Jm=#P7qjJkCC4Ct9Ac zo4OEB$x#5G-QsB$OYLze2MEz8S4u)W6oXWNVptb=m}rW+ANW~gwq!15q+&@!^GB<& zF9fRxK_C9SJzofddh#x ziQwrQSwt!QW`{S+pD>>X5nz4x7OIRd-a^$8KNd)U@fGx$Secjv{dN=c>e87=h3(<6 z;DyR0BGK}%t|B4S4XI84>$w$U#mpCu2Y+OAe}9kD<)4`iR{MK<*umw>CvD<+`;zPe zeqJ14_;tnhO>X8Lxyl}L#GHdE{@Rk*rOYViMjjeJ_|9jtYp?yd4`MB1yE%E5GV>N5 zNK8!rhz)^ke1kf;Y!&T+_V$WQoKB^$O!TZ)KX`7oW2~bR4PAsQTbo-JZ*{sMQi#x#UqL!$EO9;CN(L(8pLJ^L>dAhn^Xai1dT%XzG2xM% z{9cRqekbuqKN3h>G7cv5-}VI(NJa1oYKj#OYA-roHyg@S#s+Ldi8BPc19G3Xm_A9; zvT<_EA`6@>)-_} ziql1^`XinEDZ`H2wQ)t3Sh*4HXebDuOqRP#>X2C82!m@kDrI)x_$Oy z(sec=@~2X#jO223A=}Bp=5<7d)5`A}8Vfb|;S2YT2LIQamvcB?7N-E_hgH;l|4y%~ z-rsPhpePrYC*>H>OL4#Ce(`*{+71}QvbdylMg(-d{wRTf7ViV7@a(VMH<~u&hi6b0 zmI-iF25mB;+eYqEX$v4NVVWU+EbY8n+zyt|tdX59V;-o>`*W%yY}g2#nM2Xj>$FPP zbW#bV<Lp%u`#&VUuWg#l2H#1kgrgukP% zf=ZyI1hIVLBl(vZAxv%F`}>48il|6^pt z`>i7!0!pk8`jTJr)*m%hXl2S+yK^{g=igJ{F1tt>Frzrf7yJ5FVNeez$DuDSegs;d z?VgV7=Lu|$j@s@IHP}0xMTwRw2DmR)PeBxNi39mDun!?Vo#_$8EQ-{3#94-tk1xK< zV?RF}EwqS^IFZd=mH{jIr@&Xt#>S=tX3O`X?`Ur8Ykhs6ek@{;}VvWX6J1>}r*=2}_3j1Bc~$EGO^HmkN!o@{0o%ucS#~3;hyu4^sFv=$JJmXN50B; z9+;BxoX4YO7w|kbjf!`x{zD$ukMH{xPgT9Wy({?vFhG2VnU1|Fk+)FFzk_r?B9%p|pEOp%sUrZ5xa~%*F;}CLAtB7$2+C z%J=$sFiJvgwdH&$78{!uvM+ZSH}UdljPq!-SiEvc5LpjT#!SaFJU$L3o#=a1lJU0L zn1p0Lukbn8;qyWFn)lk^uYoaX^uVNHY1G+oE+#wdJJJ^B=9rmR*D&6D@Lcw?6XHW3 zC$hLH5k&Qm!eb9ujawDxg5zNh{J6KU_X#HzZuXgr%!*WUcxkNbu==DY^B}VQ!5u-p;B>;nOUcb`amV$w9cchtsBC|h)`@h2f6|$H+K9zGq&ZBPu)qaa zXzB@6?>C0B*e%(gu2*m-5&Bfj>P|y^Fr~O~2+XR8yv)`b`clv~8g=?S_+NA$C4fm9 z(ynf9U^n{)$eYLLW)9H{5$Q`AzmAn}UZG93l<_C&sj_)a>Ns^=SyqtWIxH}v%mTp( z@#h;DX7>&oMR#r(IFn~x{_PUtA>>cX0=!*+&JjoGE2YlKlY{kZ%FD(@FcMblI$n;a zME*djF)?z2=UJ03K1KH$O3%%y=hvByw<;;8&scCVarI{^OJ5yfl8`KxXz1u+mvfq* z?Ci`{$y`v%h)pU38BBgpV_}oz`U-f@j)}b93T1OZh{DaY3O-3=s0jBSJHzpSsc^}mO zJV7&OHMMxWGO(jGQkv@a&0bcKe0x0%O%VBgxKYMrHzX4lx#f~F(+lCJVd%LouQ^w; z&av_kGBR{xVN60oHJ?Coq6Z@XBP>e!GJ;jl!%t5)o|D%=g)-Ie_G>V6;V(|dAC$8` zV+Z+d0xR^Db-Qe(le!KVqOn%nb@D&f{18^F--)sIl6Q{ZjpvXr(1uiTl%|zTZl0hE z3TMMae0==bIAMck3g>JS<~i^p&wsVaA|BxOcs)>Etr(RvsFVeh6Fd4-cMZoY&9Kv> ziA^M50ykGB`)yIcbuC72&XJT@3TFjk4t@Lnm011Pilf)rp+yJ?3jzy`9+X^cAZ|MF93cFT4K|kVQv)|Vk-JbJ1p2B^! zNVTBVXCcNc{*?%Z)>Hk`YEx6u2N?i)b>Z6^k0hx?5<*<9Hk<;4{i<#=0HyoEr`8+f zamO`J$F-J#hQUmW&WT3r8P}z50r5WZEuWjSeP+Ewcl?DXUZ=MlE2*jRyIf;tAK=B~ zdHaQgIS1)c`jT#{X9j7&Z|}oPu}6zsn>)2eKN&3+Fk4-MUXu`$!b`q;B+n=|@a8{D zOK-ls&~v-^Au8nh6_!YJzW!^217;`tI?eyQfa##&o*pp?AS{8+(c|EsQZxY97bJ3e zc6KH~CUSOi5!UAKWd#0SJZ1s4u$E727fOM?Sw{YJ$a2UVZ4W`igzVyYZV5^t$G6|e zDM4sEQ)z+LkFmBRe{K@A#QkYB=F~YkIr4-VtBsdb%E263&-*nWARdAJlQWe4SIEOp zJ_q!#X}P!^Psj5CuTNt$Z^rpggF1Ut(tk8Xr`zHfH^st8xD3#IIXEBM0 zcb?A9Wa>B9sX7EKdIeQwSVeKRt3TG(5X~ZYQ+GYiR@-@FAKlq`=FzuU-lQ2ds-)rp zZ@;$5C6K?wjmJ^SpwGx{r_QH#ya;wbz1`NSfg=I}pqCCUT)VKYfZy$K>A-6Of>k2EiQfAOEs-vKqqwW&xS{p}`V~?LwLX9o4nV%gdlf{)OaU zkr>}PtqMD?o*EB?Q%+sK+~&4>D(49*|KXyZue(_4u(=WTe_p84C-qWBL<9IZJ(^K` zxN>osCaTyGO5dB4+n7(qMWij86@zI>L->Duy>(PoQP(%BAfZS}gQS!oAl==a0@Bjm z-QAti-6`E5NOyNPNFTcH@_FCq-tYc#?-*wc9Ru;4z0cZf&H1az@@{x&V8%_u6_a6d zGToG8&fXD&H179QE=!)p?HydL*%GQVo$loMTN?`5^zg{Y&~W%My8fFRE4f0M6`6F7 z1QxZQAFb$4LHnh-cX^|jj_z$)$IT5u1Bc6+jB&;|arJ7^<3;5}WdRGZDXk1G*VAaL zwmV*gT_pWK)nEVz)VTwm$pRjCCyS?B+ICu3qWcb`P0}!tg+aO=Ps$+?M4{FD@OM zJWVXqs8J~0p;YRife^AfatS>zI+UV)_yDQrYxaOwWmd72Dr61o_lY%ll|PMRB<`z^ zFhzZPVStOM?8suSu~cEbv3+CgGHS*__@zcIxiX3A#{PRSC9Mp7uz9<84r$7^%i%t5VO^? zfJ!U;;M#|qQ?Hk;>UjMp>el?i!YP$kPNy&F zkvUSfFr6+sM`y`%vb8lq8Ar)k3P9Ln6Y+y&*|Wfs6lM{wtxZn9!pe%Y6db!=4 zW;=yM@jT9h=1fHmAPdh6T69!YvHkvM^(k?2cAK}qpo5;n=+u=KS67L@ezjh1d`qPW zP1-nXKQ$;WCBX$il3#j#)zuQG!T{GANnuI-`h%pZoIzbwl;XbH@1|ka-W<7-^SJMH zrNL;b05Ro5g=UMui}9Xk|N0LATUcA$zmag;^0LwjdVKYsNE)NJ?9nBgj_CUDN_tva z9_H5fr6FD-Z<&GSgZVT)$+hh9mOm?5oN5v)jm2%F(TAF^qztW9&${8d9+q?rq-cc9IM!S1!PBZ;-Jz1-~J_uO`UIE~sZdML$!?7q>B%uD{asEu;TX3~DMbDo{94We)~Ye7V|H~iq6TR`Xmd7I~8U}z}gwKtO5P;rS( zvp+g{npAeHSJ%?Qxv@31km<#?#Z7|DcPhlZvGGIO{qFNtPf3N|#*_fZCkepQ-uJ?t zdQd_(!Ojn5@dA1BEf{|EZ8c-9nCw92`APpY(y%cJ=&WJX(rAIb{r&lbP5@TBncct8 zYBvol7w7gxjXzu_N-t?>mMoGC4|TQ-SD43LJH5m($cch zZ7H~z1og%nn;G1$K07h(Pg>6C7r_mxfQ;06$gVW>XL+w!CCFx;kX5YV~u8w&_ z?`T$<)oOLUuCst9Rjt;PTX$aUM0a6fq3r>(`pXV325vFPNBwy^S=x@W*QqhhyCZs445=vcPHc~1c!5)FI!?wuzAZ^%48>IVz~9`Y()hP;6W?C+iKd%S-yadt zJ^|9xr4FN!#c#_{2R7O+1{VzoSZ=Pa^yhlhAAWvAtJA-~zg3&98Xh%7oJmiP(b~xk z4-5(v$O$iLIGU|oXmi5~>Qra7gn-7eA@c=teg1l|wzg`!XnikpOsnmXE^0BOHo?`| zxp~g^()z~Q(Pkp{a`nLqy1drmXctQhdtGfRL8T!+ULPqLkG1S*K0D6zY>c81Y{`5p z?rk2#<-Rh@#cJfKC~M6C=SE>KRW5I|KL;T26021^y~Uxg%i7r>E$Hc<2 zX@~${DD(F7N1YZ|m>wV4@|zd?rSn_hevX^j~!_sOc^Pl#BOq^jNlII>(g~ zqsl9GfO5eqdrfq~tk$)CVRon8Pb)m!A$r2XyJQR1qK~;4K~!*W^*BC0_DS@*J$p`b z(LPtUIBBJ`wav6YSVnYaQLi(#+qn=9uKzp$B&t7SGKd>8X?$P(P$Lsn6Mx?$mn8oW z9l;YG%P3cGxv!D!Q1XzAg8ha$b-R*&&(o8%+TfL0u!g{@*hh zJA^FRu37MT@6NcY)&2hc(M0Z^5A;J5Y1bibj?d@c$_cE4MbS0x7u#=7CbCklyec2k zSg2I-4gM9+JUu!%92^-!qLY)GqoJwc;$XL{ zp`799ygzi?85rTRWBumK%%8!*0`VzBU`jnez-kn+dAh$2NSoTv#ARew+ngFi%NK3V>0@H->p4kyh%>Nbcs%ZtKMxaKQGNN6XC>vGTbAj0Uu?2v zC^I@5Jv*6ehP>`wRon8)#79U8K`1(u?}xP;jRQ_O6i+5bQS3&`+S}&1#F*7mBRYpj z*Y65qXR;0)#PsJKeh&-EIddGw(pX6odc7&BsdflPs%@4DiO40*-UZOmmoYLK<61hW zs~+=tCJcu?6QYs=8m8rc28oykDN%i?28|Jn!%JurtdkqFQqoV+k4!56lD$6?yl?SB1>G$_|tv!Lrgf{#A{3OuU zm975%Z!D>c-y$R9jT3I)yS+Sn+`scy*{#VjlAnkIatYB_{(%AJYfTZyOFn+E_0UXj zB|JS>-3Z$ed9y{M2-E5H3W9>*^3KjsKdF=(;=e7Gc{d;BG_e@~BGiI27TDQUDm6WB z(r?~+<0di_F#x5^tEH&88prlULOZc`Ri@*f;;BGP#;a&9n1i3{1j=Q1N zXtG#)*Oo;T{1{!e{`7E-LhN2+>=PD-;9*wOtdjDXZF^~PagtqOB&LKM*xJ8e&q#%Z zGnCL-7Fu$A7MDqqCm*C-4<`gNv}jyBmI^y1I^Leoyud8h-%cNJcNfhl8KkuwYOzwk z1GK0n+VSd0(yZ@|o$sf`yTsd?uZSmbt=5__@X)%+{TeSDHH(*LDoq2@+A3&}ofwa$(;c$j91<+VWuuTzD^kt0g7 zxVT0`BnOX!&(%tU7jpUyxz0X67TtHqBDEsBe#sO;9T!{tm#Zz9ewU9JZ!^T?f7xu^ z6RfxjoXGGYj|l;2=WL1bKLRMqKS*Tob=@h8G#l z0ZNdQnT8+Te8HBjfnKpBCdmKA0%8+f_C{&6*zK1Jq}46uA&B3>p+>y7`50jL(Y86U zyrYXI$T9ofyNh7!6W+!#>a~G|CV3+J0f!&nl_{dXABjb$9~+Wr(0OBEVexNIy)`m3!&By8-|K;yM^>I8rJ&f^-sRioWM>zA({{XQ~7ocdmWqP8duF8 z4~8Nl%_s}oCO${%U#9wlW*-}k)-oDa zT?>`V6_80-yyV-CTBkzd1Hw9fC>z32aT%XQ1AUyy+c|_|weRtDXwoll>KK{{#l>se z63f+4UjbyZ&EX-j;A0(S!Vo3fV38be@*svmd`9*7nH<0Ac!+e?Rhfl(_?E$RzU15h zS|Q{B`eH|K{oCK)-EfM97jE(jicA)ZQg#!LG_H=P3;AX$U80^^C;8M#Y_*3kU9ZGi zINVOc*;4m*b_BMOaoI>oNPNW*xdtgcq+!Z*1QbZ*z(z!pHT?6Zan`W^A2ycc+G>v< z7R=n-G_*X*`)_HeXmc874M$Q0-#HyFCm$!3TJx^1c|LqEibX<-1YogWeA&)M%8<5=G0PPfP79&%Iq*XB45=F6vf$rSrMhzr1__4$mJ#4)9|hD6SvFZpDQz!H~6&7%w*JzaW|Ao)HBsrUF`O^I$^ss#C6Wfx8XE= zMT7rAoQs@9cS0HNx5X3X@Qv@b`l6aznl5M0ZS?bMF>0Nx?H%oHv)DN9F&iZS2qC@*q0&d#n>78( z8El*;Q56t@We~5K=`Jr@FkOp2;~lNi{$s8_wZ#oQbx@Cw++5sdO>;S_>IZgKTQ!Vk zJQTH(LP2OH$sc*xEx>`{hIW)NSt0Ef%+ne}Ad&QLJMN#b;$Fp4w*LNbWG~Lj$Vzhi zl{;*L03Dv6x-Ux=br>|AES-*ro~~{PWll0-E)=9pvxTJ;{VX9NDNgWBY)b6=#4)ef z`1R{gBRCZ1j{Rg%;@OU`#Ap|6i#`ADPw+p%A+*CKn$ytGJUu=h5FV6=>S^36G4q7W zp?aSFZUeG1U3&?=yNd&D$lveIs+6cQT3VL25}{PrA2K~mn_A4+MSf*%CtkW!7rZ42 zE+&HfGU2CZWi_HDzJE;oDlL(=jO`?}OI40;qOj>Xz(zY6Rb6mZ?eMUR|MIt>#~CY2 zB{!7payw*E8{L-n5$aLB)_maTi-fq0bj1&o;(WhFCnf5UGP%kx#5CN9hq&8+^#SOp zjGr6@M2C|l^b|403{K7DZ0mUV8*+#nu`u~Q@*-PXn?n2JL|Lze8tcj<YmR3TlZKL9y70Xnc$ z;$-E?E;S(!q)~iL?Np-T4;F^q^22jI1r$5>-Wo%z#K2usT49YGFED|x&Pgd z;U<$D6?;h2m2^5@9O(^wnkzRL=PcYeqU0{;Gg@iBI_&H1wH^XMW#=2 z?M&P^{H(Ay2{R%qqI(rtDHip0`Rp=UsURRp_6t2TxLp2VoI#YEd(n) zkkavRJ49hml_$Aet@a(_#3f@_Z$t{8+M?IX`5&0N>gw0^s!danQKGP+qDAT`**M_y zIm5(5x1XEsO;j;do<(aMcDwQYd;HhMATTKC*T5GG9Y>IGmPXMxY!BwtUJ#*JRDHako~IeuW~oh?=j8bJt7c2**U z^iOpE(o@S}ex*Lo*9^ zJU_kMUN)?Gz6A99an3(6Fw+P_j{AFH?uSsVtTN@=2^DaO2PyFi&^+SBRgyEjFz}d_ zlu{@Li(H9v-sGXaoxpQZz*kgCr=_L+us4ZlA|_?~vpnqE5E)IGtL@1%x3q+3O#il_Q*EsU ztJZGr8um4!i$T5d>gHLDV9EmvlKK1MxwTZ5GdOuEi4W+4-`XuqO(l9na4@#hoo`Mw z#o{N*$|$Up`JGO|b&CiIRSmeGj18!)Z5j!0mK`4+zN_8q4vKmo#XvP^zyw=F>|`$u zODr$KxE;nowd$Zu!YL;p=of@Ycng{c(eVcPSLr<#pee{FGF*G9itp!h<$&PeQjqdg zVdhH?Ku5-=#lE-3>7mROOYk+Rimr`SMCu0zIJSh0PH}G5?!XtxQu6aZ_Y9zBI2fGf z#>BT>P&;;v#}N%` zwZ^YwyOSa#LpVGi=MzekvL1Zdy`Db`6`UMx><*Vf?X?5=1)tY5UKWN!ci`(dPi%gV z2HLCjl;G$unlk?{29YMY_%b^p{d+TfaQ*<^!qh3$R`_rodMBpcYc^;UrYphA6B#;v zH1_r!|9Jja%|FD_ZU1a~r1_;pK`KI&?%M`|~Tv=};OsylJZ>7_$UheWWVg>-%;WOki z>M#m2iOp*XRoSuG6NY7LYeziDWIEmdZv5{KIX%5RW|F*1>&3df$Oc2Uya?LXf_@kD zhso00k|N>mqnT2;S|fAwFG)j1?>$?=6V-mUpCT16kM!?DUgfiT(bMKuFJ=emJC2i* zle?byRz4m+AQC?70we^N*<2_uFoleibo=@^RY)N|@26Iiq2Z$<0(eextVC(l>tHjK z1Cnu0<(04;5VD4f!ag?0F&Av{haS!8rRd6MSqW&!KU;hujSxdXb5UR|puDs=h!)Sj z4@h83l24#UM`MdL9!5+gL}6&pb!l?h)DYq3ZUYio-R&c<3FSn>517L{3?O zQ}!ypOH-3;&fb9-Ihm(718`xA;Lvz!NXN$Z>}Q2Cm=-YdO~#Dj8`y83VP$5*$gsZ$ z?QIkR_xrbqA;Iv}Uns#}zKx_NP_U50IvZ}($w){oH`-{buT;@E6(5(mO) zmM=`%1}Emq92L<0yWxfL&Qc7 zrYJPcn$HgfTc}D@_ob@jeDn5<_rX0>nVX=xL6v8+ zTFpsgy0%cz{g6cD{igXFjRrM}iVAC%yPgov&J6$b0__Tdt{^596z=Rl(iD^A(e)tC^|(%_&fc2lc|$CZ#l+x%AtB9x=f@?Zw}HweWkGe?hN>Kt56_F z+d%ynO{ipbErX9mcw0V$*VKc_!oeSE@Jbgr!pd#R6_`opp7jVYH+TC_67O zUVVR~s!iXZ>2gd|xQqMekI3mf|I2f>iPsCP4@s}lP%}kp*?u}c5OS+ zXP-*6*Pi)Uqr#Hts@M&SFbjfQobrtFDm5Dd;j;~f64xnnZ9)FCWBf!?VJlSaFX|l-&j7`s#JYcmBF|n9-@`5va8Tk_IRodiU-7h@;@V z`NhT87OSOBr=rAB!F>0S6S#P4nHqY$4OAD{Jnof-Fw2?rLaX!JxVS1YUhTRMDt32$#2z}juPZgL z3->_YaDFMuGDw4s}zyjviWC&WH=ktzeqwc`Eahk8GGy;(ES38~Z zEHh+sd-S3gwBg(rySsZqKQJIb zqupJ;PL3+8y^fnJH-{ty#oXNdAOgX7tW>2k#W{YWwbfl5_^+IB%DE<#du(M(mnf!-C=D@5$s`7U;|I*=THsIq*ugLI(W?H)U9%Ol;=6{-C@~?==tmUw| z;VD$1)o9;r59f6wLc_QQP;hZeQea69KJi7=&i>+XI*@p?d8FPTv*Ie0Iof;xsI*@L zfZ+Y&;nzbE)Q(r7G8W~14uP1WW5Z96W6O)UIG{LGB)rjY+iQTsH*IdbI236J_e1;0 zaWF`I>7oX2G@7=XVDi1Y+C_2ZU9!F9`0{XCi@U4qq66`{fboXHi`p!Pr%u}m!wjCYK|?yeN>u?YiIV#Iqc#Op=j;Ctrd7B)tR|GbDK zHzG{HhoVxck-~X8w6jJRkolBCODkmRO9%r!W>|_@9&kSZd{+%7Qv{T}q)czfG297y zSkOUO2zVA$yv8py|G2NXE9^Rpi++YE=W`xWeQVG3;=9D8`6*SNu*p0HY$-LA?0#yt z(>Xu8)X@Hf>+x9|TZbjmxI77qi+fl51tDv+I`fx@tI<;>elFhO$AWko`rgHA4AS0519S+2Gc=RDOKuU-Jc_w3@$D>>hN zzR2YZP!?~f=uR0{QRa%De{?0!CylUTk9l==d9&8wbqbm<*PZ>r!X=>l+}+xeeDzr4 zk-AD9FOIj_H&j~aGr?T9?bfL&G9-!p6cledybDFJjb3vSBw|C$T;qoWDJ+XuoBoFijZ2Ij1dYMW5 z&$oc2g!NZe77;BAIR`m8)uD}$(0uOt`rqp8;4-e8K|P(y^q=7TXQ-*p=u}s0qRjov zz@8JX;As9*s4})1eLu-`+zJd{$({RCE!t|PxKDl>Fd%bbQ?8l`Ld_S+Kf!#r4V<4{ zi(@ri0-z3Sb);O7K(R=8qHJ}na>o~+PG2BPQBedu>nLKP zey8mZKC5f`)r;R}!~QkYcIAr*}slhSMyV`&PA@a$x) z#wdJ}_jfaUJg&~Fs`~nDsQtw?g>c9?u*NE(02klOjpFV1jD?Ap(yj}~W{Ev9Gcfqp z(eCBJ>s42~y}do(__M!17+}Ko(SmTHLP(t|5np0qhyyZ!&$701fhdbuXfb%>699Mw zw9n54=*s5ye8fk-?&|MJN3F*`4Bw+IZpfP z;u?1Gae0C9&AZ2B<9--Fq6FB2HkAU(+!?ClwJe{4B&5nA$_7=IF`unMnc*Ylhr>4ADb$xsyx+9)BNls-o zn+Umz@Xl%hGS)&URk)YbkXKW4ndw#T+qh_` zZ)ktG9NEt|6UKm>*>7jC(rELPFRuq5E{)d^iO)+#A4A>Pk)FO1h^W4P6!mg(B)f}= z35hD_BMYrCF|a_4zaA`AluhH1@z)`lP+s`Eu25rmp#F+S`LXVP-`PktFo(@ z+@KxjXw43rk%gWnne^u9E~Rga>RHk$E!ZB%%vP&VFn=}8MMUloYK*+UF^*8?n(IL1 zrSNFVLm5ii*O|1_e&wTXy$a|0uA!kHO#R6tu1&Wm6Oi|MVRR*R>DTwEEVK1NNKb_- zKw&jBt)hK%gYc^NP3{t*m&WK(>@Sncwc{nnSM9G2>MhjjRvInQ@H2({gJ6REL3pgS zP4V|inKmGzny~Ug?PY41VIKq#P~W~eY^A@A9{KQ#D^ev-cMuXGgEMw(016DM(Oun< za>vC*Dc5nHg}RpX^Rt^D8%!dd_WjPDowq|Cnb+M;A?M$T1WF6FhVAF)*yUF%@!BW>`zGQ*48(I$ZisR|*CvF-GAbT;N)N;3h{G8YV z!X@Y!k%pJ&7RFE95uKni5M%Z?nO-(Q@J`zUw?Es(iloehj6QzGOqt1SQEKPHNSP{< zW|negabc;0*~0ZytIg@Dg&I&T+}sJ92M0Twe>YE0S?}|0WbAdi3u`z$P9E9&D+@_J zb$ucfEt(a9>sKlS0RVV#EZJUu7`TR$p9$?xEkWRgOFqUBE;Ewyr*nA`A0z*O=~{h2 z4~$&^Pg^qU%@GFmv(8cJ%*|!y{o+fCqM4t%n28#Cp$vYZV$AXdG#<6=XK-bsM+L;{o>x<>;Hwu-)P}NJ| zbPSgAuH>hW%};nc6W0+lfn}uaL64>CL`QfEbBQ&n=(4$O!|^Q@l<5fx8%OGMJw5pz zE!W+FktkWIvF)+*{TSLxh-aAdoAeIWW0s~C6ib9}O^+H%D;Pv61k zv|YO^|4V~=fH7z181B()%ax{ua!IG)G=9H8U?N<;J4JF8)QQ_Y(K_0~$$Z0N3dPQu zjDrIy5Zd18{OtiDn8_1ITTjO4G1uRt)oid>0^tmCF&P=r*s@0gMsnPyGq{|Ftw8pe z^-W#f;jM7uosgMVeq#GCe(+5<|g08(Fx6lO*3}OB7Q8#CE64 zZI?kiyq`Ne;&MYHP){6+VbV|4I&2Wf2|tmu7y_%BkSP*AhnNu(@#-f)XzSbKV1VN& z_@AtTUS4Eo923gED!ZpQBGvk2`2ZZeaBy%_zv~+skUeJb*H3wMZ^HmUPmzOSbeFcg zOtB!VWY-0IHg;Ven05omblGHvp59(0)eKzg{qc=IIx=Z&i76T6ieU)@)W`$|=(IH^ zimLGwf1I_F0phakg~H_dND390G2*o5k%`?4K$S=W-mcP&{qaf^A`#9e7$GB{J*e+*ZMOa3koTn%BGUol ziHOetQmmlg{scvF+{|m+d*9eBvK`r`-Jhs7zkMO0>Fke#SQ7XYgzqU2;}a3?-UOCAegI=>vtQoW=1_v&lUFXi?!mZR6{OPRRVtAwH=tv9PJ zryV!h#`r>cfE{VSKeW3`F*6|&^(NQN3BMRX-)3;7qZDKA-;Nu=H&7aRgt<)rm~P?( zGCQ{q$|$}QdO~Q08Wec7yoW<0Dh8^&)>b_|v{5Vw@5iZKuftby?8*H8$^jjj`bwLd zN|Yt{H^uZfYGxFo6yH$fJq*0>ak1vR|cIB3L$9rH3!6nM~in z0E2;Uf_>>*LAK`tu=cYlt9-`#GfXRI|1c$#4Ifx6K?_2K5;fGFi=0af? zs>NSltS0b|)H9)`a#y1?8v>>ALW}zB1X1_Kr=SY07V}zHVIluw4p?7#z9<7ZLH=k7 z(9wcy-x*TrLFLjUxiZ6Taj?Ze4sv!F-f2WDQ9nTzrXo^V$~49j-4c9Qk6t=;7^u{%(F9bs^&Y=JN+FlgX#R|y{^Y%80oy>wEFkRixj;(5@;bb@HMDOG@408EoB zh}dQbRbm4WOOFE0PtWV&;w0@1$UnP^?o-GrxKFg-?=6(ci9~#nPFp#j9(H&aRhMv! zpz-y-e6PQqjlfePpMm?3SKut?tj`zh458@naHV29~-I ze?8tKnjK9Se-h~4p5D$CMMev`UGH|v$suq$T^0Orbg%-{lv>YE4E$)PoBsfs?oVAC z3`sQVAF=RQO{XPGn#>kPch9%_p4INaVnGY3NKQ&ZMsGLX@U6u9^1*@-4^O=fuH?&v zve(NK7Dwd4R7<7Qv8bKf(d>xJWd}(w2@kC!WIrZz^uE>6LiX-3AtR>V{jM4g`2=l4 z2So13cQZM{0{if{pDiCI8U<0)%jf&!%rzRFEwBH|O3c;la#b|C-^X4|jTw@Yk$sT= z)7M9%aqA#r*z9v(rrohSxDysWyqQB1{N}q~_#t~UkH;TT9I{t;?`VnbmFl-!$cVFl z$JY7>2cNBZQh#^+Uc60XrGo%^22%-%P&_Cwn{PndE;GrB1-Wp8pB1$HDq$VWv*4;gbwQI_ z|5}C6aBCwssFqRlwl;c}tV$eBD@k5qR!3-worR@dHnUxb$Zps3$GVNf6dtQZmX&(@ z^S%0FTCkA5kvQPj&vxN>%R)!cd(3!_CRtnZBth$!NmLtV>#wW^$qD0vfNm5>2=J)bX7;=Hk;mR-==o zW`+&Ns+tC@d(sc+7R+?KQ7XHmno9?ctIIIT47X$jc?D&v=Y4?4nha_gXS8&_|2|Oz z>J*l%x)+bz-|r5yK<`8*@&_z=;Rswpe!=4<8DW1GZ*(v2-O%E&4I* zR->8tsdzB~2}J*yY>f1^u99~Q1xNwP5Py<99}b5^8Mb=e_GgD|Q54ToY)|>H;(xr@ zzv1=Fw?Yg-_YVkokI%9W7{Qa7(gq@rcNdqe<{B?+zdceWNj{MfP29!C#%2s$hq>zr zeYfEN2}UXEXKEq92$FK!IKS~J5Ey~-txnrZGA-cE-$D)?ZqIy}kLF4~)p50YY430i z&vhZ%FMe;abdW6IM6jd*%6h6kMjSTRzx_MZDq0`!bTd~y($mrrTdP4KA)sx|w-qV& z$F7nKvwZsCKP&dPDZ<4`~yXT3Xo zFhLiA!)9HzuZ_{$^T%@O*AB=}6EgLDbYb^s<8bbe*5YCisj8|vTMnoJa79S? z`?**}CCGbyqo!dE-?TloI;=DI*ZqTow@)|P##VZR+etvGjcE1u=5+1F=pj+P!7@=L zQkzu}Buo7#3uOee37jm zu0gZZcD>o36o`a(a=7`fOjt{c7GLlX3ErT=Pr;RQywv$TR@lc!+L+pOwl+EVI`=NG z#b#?UuwxE1)gPwTPga&ry>4YX+MoKu3HDA-Xe+d2GxU#FI-JJZR|4)HiNjuXW`>a9 zz4KKPL3flvk{W#9{a2~0b`eHt)>`378+#tOlsR4*zqS4-AuHSBaWh(;$_fs9+oyYG z5=JB>q+D%Wy!l2Ong%smW;566k{t!iwJ)5U4~r3ISYw$>FKeqxiQnThR@@v97SnGi z-ENKxY7m78)ap#Pc6WQ#zwq*&E;rQ(5gjkqJ2~2H&?KfmZ38ix5PU_2Wp^Ob=|YXu zo=p0}AP_%lci1>A@v0xFTd!EHrLe5z7f^2q6@MvcBh$qO0g-%oUt1qj>ZsAsgr6tB zQ4$ak2*M(pE#@URL^-W+IA}tI_j2cy)Lhk+l%{7&RU$Llu5WEfi`f8vHJj^3CpG}A zK4Q|+WM2*4At4C~%vQg?mA>Regm&JZ`9)cnn>&YbyWjF%xv_L-?ftUrqs8ZRFkNjG z21Lf{g}i)}pRX@4qzKVPP_{wi5yP{8_9`eSKp}=4OjYUy$wbEjk7Xcs=QwdU;HT~% z=5ET4kN^HPx3W6@tKoV&*e@n4CY5*UXt~zrv9*iH5k5_wt<`q_`t@UP;YUtKfA~Cp zK0iQ%2}*>bjE)!U6>7CT)SJw}?&l8iS-~e&YP4H#EN6Gag}fEiP8r{mPo1?mU=|=; zUJkglwL9f)bUL~&r&g&fn*AIT3GRKJ@x(iYi~Flm&|~H)$p4xUJ-}?)NnkCns^0N} z{S=%6$OJf8*yg6DrGg&i9Qhq}7_Css~% zbQeD+|T~c7@wb*QWWJ-Ey2U0e=AffYGHci z1pZ~={(BW?fZqTSe*g2DLAS>#*Eg|8```R#=VH|x`M=F7IZVyr|C<&M;*J%^K`i9| z{s(y}x<4#`{BDFY9U;cV!1&P~vTZifaikfp)$YFgX{0a(1)s|8|42dqZyNgl{Mjs8 zcyeL#jNT7_cSu`7o}#HeTsiL^mSZ&be}5!+g{N0-y#MAc{Lia|0NY^Ne#%voBO;mn zfBu!&w`?fC8Aj2*lQxqA`a#P7UX2(tjHbxHVDA6CNM?wr)Z~6>^3y{NKqcLQKQ&E-+?Zgq)<&{{Mb^Oj9H# zRN(m6!0@7EMge12%8`$0K{A7Vd*foke+DTW9DX;G(rb2|#hv;Y7_W4LymOLtSilUj zrZJ%YUpIdE|F|tQzp84hse1Q0Fk+O)eyWIBQ-pt^Vh3B!>3Fst^b0I)MFn7<1O2%; z95+=3+`0bU2C87!tHYbag%$v2@bI*QZ0YQ5;yv25HjqWekKzxbQPqxHfT3JJ3+@=S zwWH(R+O&ygi-RJ{46necA=wgu1X!%@9@c-q{QDrf8el|4UnCAMkR(w{Qjc)j$=gh1 z`@LA)y6J&N?B5$&^92*Z0@&&5xQlEQ@K#S@#sz!wi9BxC!xyJXAHduE33hKWHreWx zu00(#xY?VZ|CwCFJ32B#4Blaez#fN2d8x}dD9#$u)i)yt;~jUQ*HhC!diJ9M5W6JD zCRQ5^{by1%cEA|#gDLQBeuPo0H5VrgW#qZ@CW6mF2G^vPPxB7p9ay=HyYB3jAO>!X z*KG8_1O*N4>EvN*p(i>LzXcr&W39;}Rb3`YMsu!G{ojJ4V%O^pPi6WcaR8UdNiZTS zA(3*utE9lMs=mZzF&Td}su8=jb-Z<`z{vC3>mUMz(Zd1Uf98Ym9mstVZQIHF7KVw! z4saN-ZGdGMJsA1HdarCBqX+naRJwCulQIn0EtuE-jZjijp_&9s>`TCR;-HFXrppOq zw39d7!VBk@$i@@ZWLETUZf>qV*67&mruMmk5%AvG-GP;BRf+%7#|>Ls+th2elX^1AABE#2naMqnpw^acX)UFMph&Bf3i8>UF^ z9$(8Q5B*|V(d&J=%m@C6Csl5Dtv65uM&s2(~z3g6x$ zBRV+R(%$U;+!ycrbJiICe7{6@bvWBlQ?r+7w{w^eKWV~5>&+GOJKt;4)^f=xcTmVV-S&8er{qA+*g@`xVSKS zE4l=dK}xbujCvcJm3zbA!J$sEVg z6bN%~NX<>Yf|~c=0icl8{M5PW@*%%dJ%Myk+-Wa9HMcKvoL6Dz>j5U1E@2=TK14~J zU>iAe`|a!avEmR6dLzPpm;exhTD%TnFpbJ!f%nBGj9Mh-9``2H4*ncxCn zShOirg|c?fBuH9O5(O8CpPuQZ93M7NAae;KdBFg3KcQaYXp~jw43S7;94svL66}ph zn$N|Xnj)nv|NDglXdzE6hqB{abspG2sGwlDT=Qbu1ax9(s96S$9XjpR!9^sX03!u!luzN%UNiY~#vkf0Yd=L(jdH*e?=tEeMkL9=~0I#4) zVKLcNs40lcjZ1H9OD~u9^yIg+v;^QJV8#KHk475CvN+_>iBerpIdmK%F`#?4m zd>9uSYx{65tsBc^vOGPqvNEyydv)S>N#HkE5sn!aW^OWi;$Ki!t{=w{%K7>DMp7yw zWn>)d>v58k|AoTFY8i`%oMFcvkYgJ#v9l+uQ-7Kt?`Pv|nYMSxk0Qo0G7_aQf^&nr zhm7_Qi^5V4pb?r8gon4bot_dD14*7p75xOhTBb7>XOO%bF*+yK z8{85XRI+0OVNp@-W_uqZvAQ>4C4YsmEY{i0$>4KNRNx(dKa3hvzLxav??VOi4O;CU zWk^z@*NWo`Q6yiqG0djmgP4Ac+>zeslKQ@1Nl8$Nvs| zaBZjbTIN-G^aReq6$PNI1SN-prt!&%`L(!|R36P^$BVzFF-W$uwy6-eznWM)9MW#J z##y>^-5Mz#7!VgyL-*z^^tFVn@ntTJM_jeydDgrG$*9f~3@mpk>;U$5SYPKMNeqmD zQfD$Fm~RvZdWy(M84b}dCc_)35Kk1`kGQs{o^sc5U!eXkn$9{Xs`qW9YY;LRFunqM_(m`h=y_e--w?htnI8|x0eEouq$P^>b9)z>Fc z3cFg*-SVGarzH!SsXcjVU&G6}2zGIG%N_+_dXk|TUrNhJOmw~M`JL%=&(&h>A)~iQ zhW6SGkR&h=)aGVp!(}QU&AjZs1>-SO=g>#+dOxx11N59n0`nk1QY31Fuzd8iUj8*o z0vR26ZfNUfx-Pr>0t)E(Z<50Uj^+{HgN3V`^Fv6(ZUqRsu?Ys}-QC?Xsk@uZM6aHo zi!l9=4Ns_!0jvw-r7wcG)M68H>zea~MuE4tB8+^$0+-4rniE})Aao|GpI!wpChK(%0KYjZW$=Nvnw#jqauJwU}kWiWhf>m@x zg>n>@oF1!cRj^d&yjKgLl~ctguWCO`bvbFLGFN)!zzrxfx=kdvw_Ppb3m~iedEZ8W=o5S6>>iF7}tJ zJO-6&(mDg(6cFuSZoTM*8^?)P0eN3lNXp!gxS0Xhh|?{LLa2gWs5 zY(@A%XWUxPpJ$(P}Yn&(djZAZHdp-*r-3v2YgQ80np_odsdE{UX&_sCJ7Xxenm6| zJhMBX2YEj%s?*Cy=HyC_P8pAZ%o9L6o#zB8?}++`B7J7X3O^y9yR}0h=S4p#Nd`I6 z947Ni&Z8jHcY6emTlg4-=_2CIzrpviE!Yx!_EW+zzLPMRQEG0RsO4Ky?#9O?A+iwYW zLnY{8-qlSlMku_j{OV-(F9%>|)0GXgc_%Kzn9 z^rpI1Ph|Q}*BJ>2DXaOZ-dPi%g$ne&wwW3Lh;@Z;=`d;21V8FxNjx%>33I-QKpaVb z*G)YtsqjiLK}mYJ$d{c1Lplu+-~@Gehz)tu zdIP6t;FT@m67fPD!4shjNO3X^W{0gZ&tjQWkqv3YZ>twNQM-1S+} z1G$LXI|YTxFYfNeF9>X{rccsD_Z;W`c(E)b`iTjf4c=B;#O8wyWQ@)$)0=aIt2uzx zm+#9wD}&TrZN$zN#l^ir<+T_qRBYng=g=1+rXpfdj2j)}O+l95vL_7!#$%Nafz4-| z!mlw1_M$wMnJ|Z;0>t=Ka%lg`4u(yAW8#xd>T#xYrT{`}q0+t#WTI)xvMSpiGNiS)6LXlVDk3+9!fK^UBbP22-cCzsDXfHl;?d zbU_RY`?rB$`*3kxowS{lR7lcmbJ-h_+LFT+%<#L54(fI4-xbHZK@B9MjoW-6@;z%qXFDoQGWNrfi;Q&nR3uW7}^ zXlV`S9P^f1PR~8nu`)A+yrn8op3g`{j7WcWa9Lu=3qkTS;%3jK4~~k6ipsTPcIz)F zD6n`B*r&q5Ps!F+Rvs6H=jP^ya?X{Pmp=(2-AGYVP!jOoeIbeG4)}q_5{Cs)j!T!G zmDOSknaTJv`*h_*+u3IL04YzH1dyazA&qHEdkoX^>3sZkN!(ZQ>Sn&SCMMEoYwuHW zoWx(94aZ1j@-HQs8XK2**THAP$uCELL>ud@z2)|r2^9mhpQ!%ie2zmi9}#5}JlgvD zdavWFw6SOp^gW3>dbxp{@sYF){neW528fIbCu1(N;=7KN1D1=zOs3+zpB~gdI3>39dk0@*wdEx z4GOjHhIa~x1}0kIeoTm_LeUV@97%P*sonn#ugSKX0d&}>vxzPTjZV8tf#X$X)bXv% zZ8{){0M6FrG+-NQF&&x+VLgp;74n?yo3#0H*^9@)rdR8A{dZO!;8j`GnGR}g7FXA_ zVta5W%XIeVGJROAdm2CdZpWb%##g*r35Ws*U1Tb<1)mI=H*w%$qu<~%Tc?(8((hBK z=`u@(BWi2^(*eZB+FX-`FZS3CIz_Go-Zi?-0=I>PrrfCm{5<)}b`}>E1BX%W<0^kR zI9^IiWru5BPF{d{A35fk<#~T12fxMG04jmWv%lPW6^3_EpL^`N0DeDSE57E}rk|9a zJT$pq25qkIHT@M1Y4fg4#b88}-q zxz-7L4%V4egq${L@Ow6TX?#b))R->l+0}U`OPW+(5n~`1&w%HIESspQZ3;-)ysWGM zUUIf@m9amV_UtNM*jdo`fc0~BJ18!Yk(brp7?-4Va z)t?BK6O}<~!GZ2lpVdl z^2RVN-2l+BgE;9W&vX>49rOANa&P&U+z3y|ikwUMiv38r_i~ffLpsOvVUN82yzS_- z+?{yn%?13W&$^VD-`@rrjV3!=rVC2C2_)XlbIcgUu`-61HZuuPy0p*Q89Xa zNE*qpN{Yby!h4{rCvreVK7W1yAc(`ICdeEu(tNi4IwLmO>laAZh#i2!bbPrFO<4cC zh7yltyj!CE7C9u)Tmib(-Q67|9|+e#eNw6!%czQuLn1|kg!oxqyQ|FcT9H*>tW|wj zjAk)Ui2XLu|K%OF$m(c;9CkR^tSOlA53)sJlO{ExWx-FE)C|dZ<)Z(Y`j^4bxI5{~ zc4x~~wR^WOT=xVE>mBcCFLf8b)>|!Qhq~x3Q8dBG5)P;YE;6HdTkG%BnOX^Ng#AJL z`_OwQz_|WUI=TeAX0D!N3BQeH$Zo}x5o1kyE0By!FeE~71yf-V2{hN-J>!}+{YmU| zYLi&bvL$57<$}PTph(2GNQ<#*3qM~8aFE2n!CTRv&XMh(trCO$LO_-HC_1e1DIUGm?MZxCdfB?Hvz+&jCUtGCOIY{MemKi3N#bT zlF4%0v#*ukZiDTghXR{|btdt9OFrPbnAU&g1t%wON~3ASnR!ewRgF>?&bLV%-_$tVc0}KL`Fb`pOG9x zI+Eqzn&}HrI#yH9NKXmZxP?GV-)vPY4JWY8xE##=G~N;i`^wz`p%!OIou#PL_R+!0 z`0=b+Cbh7AVAIod{V#52?jD|vO*=mwJ>)z*^vGZ%yO)=GVND|o`uw&_S*fiS|7_W;vQOS^twTh0o?mVwbflTOadcvDAS-diuFD6YVzP#L zcrN1}8GD-?nV+9u^2}x;YZTx&17mB;<}tq#T_P-2{lQ*q=~qJ*jk@w1;D!dW!SpZL zSCVFdJ)n5(W$EOoN0}b%g!5)tHj+Y2;i2WyNbSajk_Lnm5SwKaSR4p-1z>TE3YlkN zxjv@$y+2>^uZ{Ydo_<&Pu_NRu#qP?s0@|E-ymbWgl^<6OhTwACJmU;u z+M$#*Yhs?rHR4Wz$aN4%}+;?;EL9gaB^TVRR=fjM}6oQBaDV63BBwwRl zTkUae4rpUiN{Z651e|z7ShxY8@r8ti$06ThV{@0f@bIZ>-B0s%*sd~l&TCzs80FtI zk1N~Ys?Sk|=&NGjQuhqvuY`T2pgr6tJV z*Qc0G)Feq>BZbgb;HkMege@n8m7~X!&eUUG_*ELryA)5)e-H&;-A!K}4iVNj6|b+U z((k`fiMpJ0CO(>?bTI0a=pHLZW|0WV^tn1-5WEknB4XD+I#q8S?PWJic9No}$&zF? zGfxI%CZ;7*H7-Czx9&ZUkBx-^1|+c+1$fmWS+`=b@# zg)t+e+}5<4KZ#JU=QEXD6$7gs@(#FjBqnF6Ku& z7Ah*8!3%rp4)aR%AvZ2sjn+aECGh^l9$vK^O_KYMH^z$?RI>E^p*D{d^$rBKjr2 z+tuW>Uq%k7I=;U=y+E*+CdlFDoy(haj_m_7)(}*HANW}x`$-01U@W#O!#TS~Kb_>g zT=YQDa005a@AeV$oSG9nnjn?uO8oZnZke2Ky5bNcX0}o@7Z8|xdj>}0J2>szKh;b3 zfUF)6lk#8uiFn{i%@B6E9|x?fk?!E%OTX3j^Rjih7d*j?XqiP3Ovn`JeGE5y6IqUn zNiWVJ?%Mn7&obNH$L*1i63Mt!q9gi1P#HcZWc;nJq@y!sRnZEhxxqz;^MiRpk{P#M zs|WfIC#U@l5e{kg*;hq7765Ep0^*l|Z?}D8P0c~A&Y^xiH&fgk1FTrC@LiLoS6^1| z18$^4zUU8;y|#Y#obnH2_u!!S!X+l!K#RBMbU9?uUKTID?%k&AGRGtK+0x*pFF==s z2V96D5GQN#(uXYF>J;lOrpdSM?I<4pfX^^6I~%0?ccZt${L#ip9LL0XgrPZf@aYP&?%>yKB%(!oz3^)){n-P?M@%-F7|YDE2tg#?>=oMg8L!Ge_Fll1kHq>VJ} zBsq2R#6-N^M!`-807J*V=Z5`1voX@D!87V0x^Jn)F?C zo({0gBcEboVPj!u*AYj0Hsx<}*Wlrcq8VNlQ?*|Yp537#@W#f*XApti7dln0hZ$GU z;cTACg<+@KTPyyGtLrIy`_&Lqh|!Z`paS42C4eaY8k} z*a~R8KTTdw8M@5*wc#jB%I0ydGId+5+ElxLCiB}!vGw8iyb}%{oy+X6Kh+oKx2oh# zDi|0TeCkO^55W?PYevRfgE!xiLZiad?}t(Wc=Y`C8*zb`WhN@dMU`SzZ7n&yLiPzb9Vh)@>`?8M06e^7RHts8SD z7BnGyGKG)ft6@EiRU7cNW@}eydD^&PFUTiYu zRP9aQ&+syC*iL&tj)g=~U9U$$!;m#lft1sjDKz!$<|hRu#m?aE%VpHY+7PCFn8e>^ z(|wj(8)73g2|Ii|{A$}py)*GM!cbA~wF%6@WMsqI!+_Z8z8S6BlaFMX%NDn7~A3b%od4EiB(?G z=lTpoNEsbZX>4o^0J>4D^ImNkY1b%qNlAzXNK;b+>zd!}*`eBpMddNP03}RHe`tf3 z#c+nNZt1P#J-;J3w65MT7UUuAy^S5eFW$i^^zNBEv!cnB3L3ic}lTYFG`S4IiZ*B=4-T7JT|i4__^BJ z^bj-`?P?-|BM5h_w_^enA_Y}gbMqy+At|WdJtgw0@fhp_Tk%jTY=T)m1x4I!d=1ke z1fJW)q*6#oNSWRlF;@~;7fME29e!BO$9`xW2TVgBDrlU+CZJU^eHQiTWPZn~hV#W} z6e;{|*1PwvnwGblery{tiFFq}F}aT;aK(Kx?TIG8%hl;olY;p2&gOxf8JiQx+=${} z;pWzf3-uo#tp=FM4m&MA4aO#4`g)@Sk<%DiESU~`R6FPTVY#HD1;<_&gs`DNBKK%2 zlMcoSxKSUvyM84}rnM*Pty6eyPl=QZz+e5CIGB;4p?{v39>l?@{4rw8i5uEX^}5pXhK za*r1AFahoir#^Z9^&G05I9(!Q!> z*y7+m+RhOD-Kmz8t~LQ1ySlD*T8UV+Z^ab=LE&+j%2X&d2TU!k(TA-hwaY zXJ(LipJHSEYj4Nzf%@LxgDMW%6O(V;ENs9>AAs=`eU6P$D7eh<;Gkx0eQnKnoyI|@ z^isRf8-$0r94^DbP89Eof;|`DJd_LaemE)vXR?U>35geu_TZ9e)82I1aLGr|N_w{R zE;RaaGNjg);8R!VQDrzW$IR@Ur1j&Nw7BTU)`DQs+tp0BQT5SF|1VH3DgIJEt)s08 zd!Z=@kOnq(X1FoZb;`uPbwK3>G10!M_aSmPpDbzfm!)Z>GLnE5NtYe^g5am++1F!Z zrW2~^vV8J76H8CGu@?T$A*<rZu9&Q|zj6#dr)^_AFCrXCbVlplg#T+A4@)Zfc>&2>c z$hcy_)*b_1i^os?XLcNRE6+9c^_@;PvFXy}e@9*NZt!@W<`2^HehZDj<08ugpSr{b zKlX;#Ax_mNbA`wF3SM3m8e-i+`9=9VhWNi`Ik~uynKQeCBUBLw5V4SXH^lEJVpxGV zCQRyH?t{h7P#?iWHIt^sull=$p(_84fzjTRJ2aX~zwCjlYqC+ttJ~SZ#dh|4KQ6$O zh~eF7k7<<~!ldvznZY81_j7SfPvukCv1`b)b%U@BuymV(nUv_DwdF8iv6Yq9FKRfi z$K~h!AV|-$0eq8vtM^1haj4fBnJs~Y_Mw4!GFcqm>xJFIF1NX<%P6N{c4tI3Jb5im4d8e%W@J+Q3-{QWl-El}WWk;D?>`CKZz&}{N*h4kr@ zMo1|u(Z@_ggwqPGfWF*(wG|fr_ry$b^P`b1e*Pc3A^IwMjwPvfwx;MYhkyUx?*EeY zXeDsSe*c)4J{=GJNnK5qNz=$gp^-u@9-*|B?WdaTva<9Se1KzUgZYiSFZH~r5r7sXU9_*OmRT|x03v_{mP^|sA$$p?%MmK^a&dC$? zPZTJat*Q}#T%$v72=kcj@9+OiUYry0t7hxJLBwIZ^l*3bm5|FeYhk=)J_t+-RUgh| z8+7XysDRc4P!IT=ii(JW!0X~*e?dPWfh;jM=*?Fu6nuijo33wrV}_JgQ{^59vI9N~ zz*IIHsZ33+j`hU-tbo!9K`a}A$*#hIr4|_pZ`OWS8>Xn$!V#zuYj1?S&3LVg$C63y z=hf)!Fm`xqY-5ZXBk>T%_hBfOiAMMcrKHFO+KD7c*W3|@vQec}m(BEuCZr{e&|Aw~ zoGPDbqf-Hs^5$%Nw8|#c1r+}j^J2!M3?)Vrwd-+Md1xf1F&S~pPavLlKz^bA)eUv9 zD=VE{E((B$HjX^|aCK`_UmYqlmM5b_&Jy;T?~TOpql#lbo27}L#%B2Y8m@av)NSnL z#h1wpg5VLzZ@VDie)7&qUElD^K^^|(u>ighRL@e}$Bxnn3yq^o}{d*TC&O}|QOJfEu>VGvLNT445jU$(b zf&Y`DOhikIbJi%Ubb#y)%ZN`E3?^#Fb^S#30uPjem+AR`$uc!HErwKRST;6D{`~;n zJMOS|&H%>o$YZ+F{o{XUtAy>?wG9+K<+&tHIgo`sxo@Hd17JcAu6*1m7%)(VS9-OX zI!T$NpCpdzutB8PHc62o!cAt^Q>{UsDPfHxURf=(d{a(2sZ9~lB;SYfN&nq7a}IOS zb29%tj)9$*#@$gvm)nGNyb}?C`Ip9T_ZtKu6Iczrfa^fAsee=|Vp963s?K$VEx#_L zjb@K9HnDS~vwX4fl!%9IW#4@>zkGXva)JR8nVEPg-(G0A&#rPaqQKX7S`K8$6JDWdwGLfvtYGD)fj1){-xP#=@m23 zS8pPHN1)HRfWPjI$9Agn zrzJ&Ks3VV~VH4MUyVvnW- zM2R2vMfyP_OhOnGeM(2dKr3;jWXPrO(C6S+7c2g7>%4f)tDby8DU*;O;xuRm(ktW3BUC7DV#TuhGtoaEoj3x+mx{{kihuQ8yXC|nea%sOk}?lwnP+65 zdF1i%-z7obar%TgbyXG?D$sUR;7^=>cUqcSSkTu<+3om?qi!|%==(xwD>$! z<52k~PQv#^O-a;aVPJ@SW`qIQj0V`e<8(O@eJt{JdLX(?Xlrw~fTgS**{*_&J*(@W z)mbwJyads?8(Ujpw#m^>55(x~S1v{0%GkKrr)~93N)ihaiAV0S$@uEsu2Aw?8U?`! zT88}jLezJa0)kZvU9OOnKs7=U5E9ykIRM@unt}Njy~xMQ);P(j7q7rhP35966J!| z3$_1eZtC+y0BVg_&72jmkX#SlO|E4g zyO74Z8WEv*VL+;=q=MeH7asME=U2b&r%x|5>IM)_qm80I6v^?)f7miBOA}Jl($FHP z5@J*7J;Yep*kEbH=>3}Da(|pVktdCpiZoObot-f$9=~#T_XsM;%ST#ohr_qNioHxG zO~w<&Iyr?mSvHHn7$Y)C+DkLh3pwOd)ZBuDgU4KZL5p@^TtPutnIY|OT>tZeiv2&c z%S*qHf8X5%9!*Em_igV(_)i@i1kZW(t}**f2&joMYVv4_$uhk_s&xe>M8B9c9HaD6 z8pQDVz%j5eeRZnIZS}Uz-@7|1Q^4uyv(Q*{(OC=c{yr$@l%)<@===Z6N5kAwN1@vd zUT*-Fpv3g1-uW{}s}v0Iy+6|Wdi}HHC$byW(!sXrM0PYb9l+#-e0b*2zZz{P;r*8BNBwpKKgZ2f*W6Tl`&x3LtB{iz(ZAPU=tkMjVX;VZ z{D<)P<*}FH+uO{2HCDRp`Xa+U6yh#)e+Pa|m1G^g)WE=G*pzosg! z3}1$r=M$%DHys3`PTZHay$Dig*mS;6%MkG8U>7?F4yj}JrXxTCKoywd1s}fJp_n(Y zfObQk=0;dPKDKvVw;PMA-4|MDc29p?09w(aYdqlL+(OpdN4>H6kLMU(yneyFJp{yz z@v*TtTvV_1a`hBflHs8MMreg95?b(qV(KzK4wCxTexVT~-nLQDP z3?<;+N@RIvGNGHpoXf5yOKP$X&;f`jMzDdLs4}Yny%iD*H2xZJ^-MwMt3H!RZinh< zrBP{`=aeRKtG_#LI9hL?*!?sV^&Hpk&FhX#ZS&k8nJ(JmNUQjMEL+3rP_Ma}8`I4Z z|H{T+Gh$<9qIaEz-@c8UmDnut1%0`k8*sLQCI3hmui;__k=XNTzTN3Ej0m6|F9^ay z1iY_51^5NyVS){_6HD;p&dEvMS3HCSx!~#)gK~mTx#<3Q*xoR2b|}H$gSC}y8K{~x zB95bvnKBY(7-jKb;pbgj_FYj?QL~Xhw;&lg%+%j(q+Ml$4VKV^p`jkhoVzuY{?Z^h zU0cf1(nD6%6?_F&-*hnNF8l@7nfn-m);&|z)T^6Emp3<)WxAOy)c)_nZ01`Gx~t3@ zOuvGcUwlr81d9-ONw#{rV3Ue~MNSL^gPOxm4jEKlU^zN2H9}!74D5WtG@+xCiJI>f zR1~Z!X7UhEn1mqTZqN6HZe2f6<3FafnX1?#roz8JTJ`pC`kuG(LLFC~&wBy_8jQfK z^9aOgQ7lF(@#VdF>o75|i49cM)GSw)HR_tn)7si3=+Xeq9hr9ngIWF~t6*@SnW%(= zmQpS{jo>%3Fu|8>km}dkNxu<+X(YeFv$@WE^y_K8$$E121VeudwhuzX&q)CWrsU5! z-me|E(_DZN5$IWA5=zs5P^u+xOpMo!Xm|nc52uJ3>0)CGLJB{n1|GtbUfq+gtKYQ~ z=vVh&+$c`{P$p)Nhv7K7R!mk~oT4KmgW`#QjnE~GtXG36{Oe+ADr|YxA#DW(*`jEE zlrtWWt>I@|{c<0D?&Du2-kf>+Lo(4#lyPb;7V|-2!P2ShGK>JN=rQn@e8amMR~cf? zvdo*zj2q11wqR7ihsbD@ZD*MhnM(Ku&C7qu(JJKg7cEDxe$B$k z>5YucyU*t43O%`=l9HgE*zs_0d*$&&nS7LCOced>M&%C-Trw2Q9NF3HVN;lWbtx;G zKiT!=HS5EwaQ0}qT%;R5Gc~y3#))_b{H1m$%A~uMrPiZhmut*fUdcHTE`X|GARJ9U z-u9bAMozBH*{><$SFehK64I0Gr<6jsiBR9QNmU+TdG_4s(L_TB<7?#+XI2I%oo)7l z6VwGeFi&~4< zPHOU+k8!j0;XIx_uj?gKoA}D#+40V8umtZY8UkG>glCt>7t0)whLZ~51{PA}Y8-~AS8Hw1nn9haaxKPM`EmW8h>jMMjzw3gkto( zrTG>1vhCywHw$UkVkKRYf_n~+9r~Nz-)8yb-}@5?d|>rX%C+{#b@|Ru?fSQC9_$O) zAgL&;qh`G4*VVgn_76Qqyf(? z4D1Prh={hkil0P1K1G~3xH>W+nGW7M?LHXNa)QiDfx0iouUcNsZ^?F5e@0oDS+fRz zy{|Gw>uR?n^1Yfgi~bIbWge$xJ9<>lD8L=T3qou;YdN)SbND2!_vc&lcIxBJr86X4 z)}6wK%^2b@xq)IcUn|sUo|Mw9%EOw8^e6IL+?Ck24u@7PX^!fAWcQqc~ zxxb(d5-XwHN?}n^0o#hIckZ{kp!G>9aO9_Sx3kql{x=pOMj;Vagr>sh&qp2P{N+rH zQtJ-M3UQhc2stKb19>@nboAMXRfVCCxd59Xf?LT>^&3#BC}!deb$}A!q&EOqBHpc_ zvrWY3grxb=Sct>wI7*f+Zz-#wQ*6<{*~LEzbPE78HwU|3eNvN3Yo41Gxz$oh@DO#^ zpFeLxE^|^i#T&RyJ2kWE6L({4IZxWRCqvHmrl-o`MRz*iO>V9y_%{cxDCk^3HKx-S zECT4>%I`bt1ELb0d%;gfpF?e(WF~Swhe43yh{g2dN3MNeJ5jgK)LKvv`*MT2l#cmN zVh#9wHSxS79=yPr1v{P^ z7Jq0fgUs{Ubbm`Fak^kXiogS;QBZ&(S&BgH@LY}p{Lj|)KNre-no|1;R80h6S^MH7YW4v=S^k?_@9du-UA2&NN{eMZ=We9a zu|h6j+Wq*VzM-+Hp}}>3;;cs3aCo36^jo<1tpFX1P>PEgwdkj_9p3oI$qnw!?dwL_ zV*`T&CB|`@YUDtyAfYlrbl{2=X?KGLg6^;9qy2paT3TAX42bP1D*)I*8`y```FjTd zz_G*qh(!5+I-YhYcOM=pzM8Nn81Pm{vg;Lg);8KbCrkBea&p=^=pTcT;0l=GrBqZ9 zrzPolkxrluX~{jr_hUJr2B>@QJ7*2>{KYT1*q@g-^YjC#96rhuF}G{TA>XLJMS%7u z__1{K4TkSP!1Xbme7etmd{IY8YY%Oua|wus0OPAo8I0plvmV*X{Dq)HpSa7K+YZP0 z2eSrsz>@Jge?C8_Ut8-!$f%Mbc&d%%h*dxEy_z=o0|NuYqa11D%6_`^+)JlTGnCyw z?kL+>%{*p9ZNs^+^gCVX0KA?VU|OmRIw03FBs^xKYd%N=|44F&SvD|Ci`MI9L;?kFN028}NCa$N|U!cU3XVjE}WHYL%_J3l6#bi2TJQIQr^&QeI`bkKh}n0A{MDwOI+ETQ-sWjO6c z0f_~=#&c2v{To~Gbb+w@90ggeU2dg_HC~owb{X^#E&PleJ?e_(W-+%O4U~W-t<={R zC-Zi#pw(}e?FcM1#O?usRRMKnYCrYk>PPDq3=2{VOiWBL3o&x?+0OjB4KV>{iK!d5 z^qM5m(<)%Tz~gMayC3jWcL)O543BbUo1L~%u(_a(W4c^R057CYY7U~-M zyseoC{Okr(m65@Se;aAIV)zcg9jw`NWoihHt1bgCW9ZRt#lF?$=yS~X5NRPcV<<^YkQ5_y~r_{vpc$=;XSv_cE2>gvP2y)Le<1?P4^sJ%E44P~x3kW*B= z&3>QhRjikRB@u!{*=o=jy1&#e!9)Po;*H+IBf~g5WgH@OY0jcnV8Oizz=zf@_myH_EC7MMx~ zpw36D^q79&QhtW0w-6bjhz4d}Wj6o1V#9*(ek*a_odmh+P3DeyGF&PViOkB$LFgOR zACNpMfOYr?YNCpPPEh;^v*v^Q^ua#7YCDy|YzV+Ie1LD_IT_ja)YSErm6;l=3*>-zu@JU<+&*N1-gWTy%H9&G>24UDR2%{T;A=q4tEjPiE(N7?i{P)Q#jeJOpw zN>ekr*xvd$i0Jzm!wUQWtE2zI3!H);FF~`Gw%DP^EEX~Q=aQYx?QNM$P{V%l`6AC( z?`kwEcRQ0t@F|FAJ6uiz_rAREUtvn`{MM`5x7a20P6Vkz`1FPp;w$klpF8(rJ3S!CbY)e73jzTQ|s~QQHn72a;`JULSQHKZ*daDnL%oiCAiI zl#eQAJ%4&+cLd6E7z-H9KIK8CDnIGKfEN1~I8kV|w1fjB7t3O>n>Y_QlutrU^`Lj&t7l{teK#Q? zrq6LNK$$Q=s-5q4LlGoFVx0FT1NQ!cyzAFTS_$ir?!+Gd1-=(rGaAV89khU<&2nNd z11v_g4%=|?vl@Q6C^FCUg`Mr0ch|RHLqh>Bz(C)!Q9zjAdZG}NCjh3nB zDO=}+=i?_RBCTDk8@=nr$k{V`<5x4}{1t>~MH;M0&pDHEc;?d+o0o2D&fC`MdaQlh%j~&5;#*#MNHgkI(#+keDWzi-> z@Gg_2#iIqmNro4l(iC68IeQ)PFvfs&OVV`5-iq zPh=bShCfj=EiWMzEDk1*&uo^kedv)10p;rnjHi-H%vRiBr7@N;kP zP~7_apP9rJrn%e~Lkh|R*}l!@=jGKm9Lrcf7j zfGxrP_)$XCx+AGcJ;;++1Kav(>}+wku9oMoDWkK#0E}}-D-$|=Zg=l0(y0mv`O0z| zUh2eg+09Y5pFla@r`nIzFZc!qRQN#?rO8EW)9vmR`F0h(ld@T_PWkQW%att=hdR92 zecB{7Fj$x9RDdd-oKU@Siw&UN8R=3KeFmR0*X3Cq6qqL8OpA}!6Zb?%hCYS1eq=+A zO-#Hwtq=zd9N)6YVFBZcsC3aGxYQJ)J~C3?fbvK!=%LW-wXzsad^!4`*q?>^=~JF> zmARyzNAVlBPx~ovf550U+}9aWai^GWxO+amiPU*w-*;2iM3 zsr!|-vLcUQ`@T!Bg0gT;%#i{8dBRb|+M6!+`NPA~o#&-9!3!v-Ql;0(KM-QJ%^2Rz zWqA6p6)8h<0t@TH4ZACoY06d8KYikWGVsX;rnmF9L`Nj(k!y5QP?9aKRe#P2>=2#3bMHXn(ew9JyepCnpy=Xq_k5pwO2IIR1q6Z&SGU@85^8NIf*aN zh_LlIqh+v_^*GZkc+OB*5N{#HvsQThO0K1R-TY_E^L>!qY`O8-;Shf?3>gydk=t1B zq4vF(>^J_pYn*S_Yv+6qcF3TD+OYcb&(a}-i-Sc-qZeik=J?X%Cb7n=)Y{v{7U^z4 zjl&NB`=^YDL4D;{ua^diZ!3-@~3eSo_G2 z=c|T&ygk3+0Q!G!Jc=C01d#3PP+$1&iKjl?8vg(r_!yMAlGKZ{wESXHEwc5VPLK3o$|i9sFVN?2^SzJ40<0JKMhcf)UK z%z`in-G2e{^!W99S%rVgtOuN0*u@Q4K4ntX^J5{|(!uK<(JDJ=#!4nt-Z78WuBH|P5e z&ZnED$(hAp-u7h8gEH8u()U+ytVQNQkxWsAs{8tx1#n^VYh?cDZ-Kc>pN;Qt7k0y& zGv1vO;F-i7ZjUhM2KSCCH?(=Dotz@=nUuIw;B#3bj?_&D06ev7n@4IZAN*o}^4{kX zC#Vx|Z1ry$I0Ay;!`)#`lz4`Tt*PyY{sPo^sL$OFV6PsOA4Gc`Uo%<#wB5IyOEUAh zwJ6Fg&=6JjdQa>n$|1z^wYP=%*(W=@;a9Xoo7}~w{Ryha_Q!TGL6}n#+YT(KW$g;& zq{|H(R6fKRnj>AI8<=~YfKMtfPck7m;kgO5&_3(cAF#SGyXNzvGI;~-pB_+DQP}ua z;{!kW6GI>3eLtJ%L_zm)Z&GPI(LOxU%kwm$a)nOLoEE%L8@&6gJsuC=C0uQc%b8cd zySADt{bd$BxY8)z>+|b$nNoceED+f}x>W(yOT@v^vE|03Me`kHN6sM5j+ZXP@W23%5%j(V}!OLPuaa|^4=0(vxe_!hxGVI@V)VazjFJtyQ9I5t*v)xf{*R$62OnU$3jGb>%m1p-Qu*xi zEKqabyi`0s^u15Xh&lg#LoFmc{Q_MYy27S+JJ=zQ`{PNb^LEeg=H`Xz{FYGw`jHgY zQcdt_-2huGOI5GkiQmk*jS@K#yT8nq55T|wMsmLfiV$e2@~U6?ZuG1ITfBdy;_E#W zMCxEH*j_VY;8HSsT_ZrKM%0&{J#c1*L};qf3(lQe5F&4*0Q1w7t5Hx<%hP=y;D=t( z%=7~J^#zl;(HV0`-dQri~BqlB81g@K@o+4lZh?j5sHA%XF zO|0eNxC!8xLfdXK2JpFmhZoP!f&XLiY>P3(T7Mv?f_VBI@d5>OpTAcye;k8D7{=Xa zBhAD_te6vF2|Pd-gE?1#;_S8VYd2Z^He8_-n90KvS)1LCQMguUZ3BsNMx`%%fa97Z zf>snwJDgkZt5$Mc+DsY5^^yu9>dYI(QT(V}1L_Z)d@U#f&L?Ph0IZ+6GAmazVqzvH zz_7U+BX%^Ja-h`J@|Vj#%k^}FV`7G=M~&^|)YtRCzy%0k z3sFB37xDPJ4VD3+%qlPX%**kX|2|Z?68YJXFH?*6;+a~G@hNJnX{)&ZsiAZlfOHQG z9Fg;k`wom~hWv3ha5quWe!}CJmQ?o8>+tMPZC)(xNVB@=p+iDOhT5eQ6t>=$7?E6+ zpW%A&$SlL1uSxoQ4T;YeqAYC0{_z;oTxCN;FVlOu^fqk(^)0w`>;+A=BI~79eMs(%XvXzNJ^5yeE$qt3nXpg#!(fOyd5Jr?-xa^831mQ9$5}f`XuQ4P6ovlG2?@ zhjd7HqXN=3bdHLYlync>0@B^xFvO6s!lLjVVStrM9i8+$D88E}Cg3MzFO)4A4 zfj8A@WACrWV)XNrvP3H1Q84FD5CjaOT@zf>aQw(Vo2b?i4`JdAD2?c{$ z3Co!ZSUhJcP}6LW=GA!W5{t&^>*?9So$^2EMi-87f_ zyWN7SQQG;u|D^8Y^&V#ny+GUV4M+H$-t{ANv<5;J2yGPWmDa=E=Ap4b1$C~V(wVn? zSUD+KBL}c|Q&TUxqbXo%Ad8!xo`serkzT5de>ppec)oUYG-CKWP`Z{D?WLV{@>pf_ zI=BCPfKC7cx;rE~(9zNL^qhNQOA8CRT3!Kff03SBr>zHPtcUeCD18U`dk^kgyUX3!=SjkxY|@Dz+QBz(yCE03hkxp`N?@^TfqW>{rgcIRcpM= zbBCo~>&QV3!}ssibUvYd<9zi-<_fof&J3H7vS$7Jm6gOC|qXGU+;=oNiO-qp~(Z7B#Z?|yH52Sl|R z7lF7PM@ffZN{3*hU6TXf(;lkv*v~>3F>&8Mv}Y~~dD@p1mdc?0{rHJ32Mz7)zejlF ze@>cap7&0D0z^Lm1TmdgNe=hK4o|e3P7@l}xBbuP+xw=>9KW^Ee7K40YlmNXbYerwG$T?mzEYpghg`Hg+Z2POGSie*faEH^7DZQ%*udZ zxpr~vv@Pk;U+(LV^%0$+VOrs_&{5jlXssyS| z`1tcfv`_iLU%y6EiP4ztPmj>a=0ozy1-zEitYA|8f?JuVAPhhFNnV0kN;_Ym+dl2QnZRTa3OKb;{! z{)xHIGq8O+8@P=pNJI$ffoMkgbiOU0UpQR!N0CH@8_8*;e%qBu$jjT3%3Do@Cu>=qTbl3lyxE%uuI zASYIgx(X&Nu#pfeMn_)kx3(GWBiX2zvlDs{IHU`#z*|1dPdm$_W6?hAj;4ZTLa$rV zk()2p<>%HvI8mcLw8= zlgY2BSE@8sWINgJR)m7ohqL5NhEtdt7sYN0*~lu-61bxzvC+_;)lnTK#EN>}(xl*? zZI{!5{dl81M*|^!TXkNc$YEG_JdS3M7Ownf?g(BDY)%S{U3w@yIfcFB*YNvl+PO|6 zh*;sq*U?pNfBrCo?OIxIRt=imvjK&!HXgz$z%n>fqzv5eW7f=rX0mtn6IX6b&N&vB z9HgCepou?>i+YGAE~Tt|9ynK%$?MA3mk|K=zn0tfkD;;TLjxk&2`QS|##3rbYWn&- z5PNF_13y2z`A-kfH;2-|2CQNleZ#@ZdQdw;ZP^vkLd>BfHLCUYbK46;x0Ch|+{y~P zjjfHufrj>%bHth#h?FCS(n^m6x*~g@lHl8<$np9i=`ge~v(PW_0v+-*xz%mKHVG zMNkTxKb2yKYqg3t7Lbb^ENtRvfAzmh(6-k>tz6XBv%ZaMi_ypn7M!Uz`Dl7P^&UpK zeI$7x4zpML(9MzWaW`2*;oGz88Ze@G0aRjO4Ss8Z zrU_o%I0i~lnsBF4`dz|X?u~-bOfPGTuII7Ov1w-S5h{U+Gy_a%Xb~jBOjqz5%9q@b zCtQaCo3L0P!@%R^wk>0?*~k4Cr3PMY(kTWNQaK}Db3;(g^gnspM&2ez@aBJ?L*+U+ zF_q;zbd?oI2s0ZbKK#P6n30Ebr``Hx6e2zt|JJ=@ zY4c`7Z!c5s)-*cS#ftT@(*+IqBuW-GmLFnF(($s)GIwW;ax$L@zse^tEP1UBx>^nA z+Q6FQ<|p%RR(f(_eh78NkGKyzad2@PR}n=Vjb{juhr2FRQCYdf*)`pcZEPJbwFKS_*<)ULV;?69x^7ySM|B;Dc{x?s zWt_fl&{}}sfq~}*bZ4*O~iA(cZ7z9=JSIGZ`s;De}2GFnVXwiURYRP7{2gGSveYP1#>lbTVkr;D0jh5 zeXh1dw(HvxT6%u#Uv~`G^9odH^xYabhiCm>oP4jh!xtm)dGCAZ(|`Z%zrSxViL>Wm zCH=oYffF^bA(cuYV={l34j{Y~W=NX*NF$!d$R0tIF#q3^TEh>OxcY5m<5e|%XuoV1 zA`>TwU#{W)@6ioFDyn9@VYly%dm98jzrKKuj;{5@94>(V06f7T%j^GrcGsUw!M(3! z4pA`?&i&?%f%l8}VyJIFZK`(?Klk1L2^)SQ7iTqA1q(+_g`wJ z;KoT0%$&+OBGyY+T|MAF2-%Lk03K?ldJSFzE`{guLao@>ufcCtSx=5sMX$Ujb~g$_ z>n@>+6eHj2LgnSrw}7`HaNYx(Iog@SG+WNPg8C9FxJfO7HdZ6*&E1Yir?)XZ1yRPo zw?pgf?d`?JkB!S>GZFv7M~!xl@$A{Ntt{Wq3c1iML~-TNm1oBr@K)c*Q5LK%yVCIk zACQbq=HvDx72w1Ib%i4hP=x;s{oU|u)|~f8HZ`b8R`%_5@lR^`#OE|s{yk^?llk+1 zpXKu;YS~{n-|?+K{(mZ~tE+~0JBS5!Dl&>l@Zcw-SiM}WwNee0V4G?^L0vkOrVg?p+$|c&}e`{kBD2U;Kjcm9OUW#Qf0zefuS-|F)X7HIu$e-yAa=@|(Ha z&4oP{-hMu_-gfL90-W2eADO}PBA#}$EodDg*8jZ-?NfH}?xp2=zeupk)HQN;q9TAa zx?2o&`L7@VTVORGb?avB{x>&%Jg%cOig4cVlu@#rA|hX;K-rA;31Zf5E+uug%a)Vs z*m7fevSSrP!LMSP&WJ)AyJ4F*T}$w4@~_VQ)fNneT+;gA8o{Lgcb+)PyI*V3FiHHs7e33x%9Dl11+hRgXgM)p1Ripz zb<9lE%TJ3382|QJG&BhHAMV*osCxF7_S;yoyT8?xi9gWLyl}poSMhI4KSMPB96N%l z_o9Jq6(5g!YLLr--KUi19hjPZ?F4g8ZnkQT%sd#R0|#hy42r$E^Mm+_Pl;MniZ|^?&cP&2DZ* zwR@-g|G)ce@{+Q)wsvy*-Ddr_dRGtVe=jEuDC7f9|C z{v=G#%jRd$J^kCs?-<a1+S(c#!72jBCmHGK5l9L@bL)5M-FY%idFSWV zL_|a)E(_#PK3*;paW5&zqV}+Ug=$EI>3UK*yKQVzL1JY=<&->kiw!eFjKbB;67kHa9>bZ_zV?%;QC}cSjyjWGm#2@kJDFf z+J(TPY?vy^0&Vy~G0!mN8@JTyCUD=gYTcfn^BX5668rAwk6x!3`<&mNa!6r-&i5vw zh~v~U95|Ecm-{`TZT;|fI9rG?)KOu|dgO~Hah8j1cWP8s)pw}khsst^Zy!#VQyfu( z$~E|cepl?~Y(TJbi&l_@wI3*}i^fs-OLx4^8VpeMvfi03w&dSIp6$MeaMj778vA=g z&yEQd4;7K~syT;d*Y&XY(y=u&`^o*C-hAK9Y(p&b+)u!osOji)qv64_$iZqPCm0Vl za&d{6BS9W4oKEl?m)u!B`7V|^rXq5e={h82(=Bh>-vK<*Q5SQZK=HyF(;wIP8UeS+ zQ_kY^?*=fD5(ST?-KU>u73JkLjQl`z(J_#6?|s=zN4}Hze^`JCXG&!6uBymJ)tHK5 zCI~ozLL(r+3UjdkO%*V`dHqqVdRo@t*sDHRS7uEbl5?5qceSCtimRZe7B}5=e|w4q zI@)^2CCc?em{9FebXb_UOR;X{b*ZmI%J0*@9#}%Axr&JWTNCx~MOh;|JD!Uk$e+M! zI!vzI8=ri>#cpPJAmC2g?xFF4;OSPDUq5rcO6Du(*Y4N^vSp4!F6+IY0UvraSL=$z z^HV=g5L3fHA3ehvCJ|CWP8iHko!_dOIuGI)aMF`h>#=Q1{Y)vIT7KJ?V&pL+&tq8` zsD_QqU%}TtRC=+jfrs~4)MBMDpY07166f}K4GAhd>Fm+@@zf-DHB(*kTzdM__Md4y zR)(GnPs&HiFL44U+kyy0*M>T#jQw4w4k-d}Pi{a~Z~Q$lCmseOD2_&>x+tnA7?_WM zR}rmhUnXyep+BT@;#Eei-LJAD$ZucCT0DXhFyjSZcx5zQP(Q7a(P`Ed|}jf4l@4eWn3I!{$C8IiV`B_ zU0(OZQY9>meR+t^sNCC0)ug>YU%hrzU9`8i7f%HWh^F<`$gM&Brv!f*LaFTH$^Fq2 z9vOL|r1N1lUZ<1FtSOwFten!6ZHJ5(IYvHZ9!XKFMui!UP2S_ZZU#eLvB8~N%3)N; zy+N^{#Rt!jyXfDQPdD-Y;DjHKVlr~xS}n{{1sX)$wnFK(vn!4hLHl9gU}F=%pwCNskoc$Y z()D~|YPIrgxaD?wdi%-(lxnTM;@q?7Qc_Ze{u>jFuqL$&_GjToAPYSTxTQ}WW*G-u zrG@q@Rag{`8&XbNqoX@Eq4Gj-iPghsksiC_J9UzXy|R|ek5{nX;pN4kiqTQvj5FX9 zg5I8BIiy$ZBZzoD-_nMP{f5bBcbhW5kNLvD?NE0sWX0&ADl#r{eZ29oY3omymH+u* zimq<*Ef55kBj`AKGg` zg6;kQCmeBBsT$EYsuB9QtLF$%7`@2cp=^Af4gA*ihAj@ucntADbFUy-XOT0xh{ZPFb> z0iwZ?tBla~IeG?$i_ypwB^cjYS4uX>S0yy=Nnxv(h-Ykf^n0EeFiUdzU=Yj*-|L`c zKPvM4J4tZNALYpgyo@sPkQv^uyVGpvMo1^C@8HZ2LAfmVeUEO9O*IG*n4qp}IsOqz z<+luHzqgn9ug#e`+t8!p`m4|PS33aPp;N!C!(8&tZ2B;gpeo{Uyz?ZJ6;m2Ba#$e2 zykC){-faOGH_!npbD3gzsCrbiE>qjA0n`dQJyN|RAXMjgT*L{FiHb_jhL^&JwwTFz zy3K0WuofECwiPTqS^w$ZV^)9a_F{aU(3)&cW!7Jdid~mBA$E_F18x8gSvN5s!DXrR zD4bL?)#5BU!=ZUA`zBuYx4y^FZR1gMb8&H$ABTkDM{nu+vrQE;zNq!f=!l5!ygC=l z&2>)EI~M9!Pnof(C>RIR@0KNz_H!3hh?v&aRx98vzh5~l4M#ibt6d1#515?@gJ%9gY6AN+99V~(OawLuR*n<2vAEF)e$z9piCZY6nh zvs(bNNgy#PEa`vNRt7ihlHQYa+A%D^9ZqTvmiUq+4d_P|wUTco9=ms838|F-c z3MkH{+bYVl;K@cZ0wiC?3xA>l2Gx@IqCX~Vn{7;{juN&wL1PUP;cKI0L^Qrg4$@n- zUynPPRWc?0fyc)z`d&u@3Eu!|3{FWpDprns4zeC z>#e-K{or>=T?h&|ZYX8Eq435KqHEJz!+LdpOBYxak%0`^G6F`y$O$7FHdPqk=|qZg z2U0`sS@4BOxAukn{`7sVc^?s%I+yv_&4Vu+k1#P^#}<9|DA12D5N?ey@9r7NP`s^_ zs3T*T4`K&2{2Wj%pKw2bQ)r#p#LUbHI{S;26wiQFEBe|+7a`ivKLOfZQClA4UYIi~_1_pN16hQH zB|LlzxkH?$@0aiJPkPcdxVxkL&y%LczT%x_vi7Jzy6G+1Uvin42U;FM0`6j018%66jHV@r<2u-D1&rLHMiQi7G1;I&vZkjf!D} zM#=Pa8h9R@fBr0muY5~OLa#PM@??L7&vU40ueAArcaQ6Ev)5p?w$p}HwhyHzEe+Ml zN+`_8WhR90gzc|HdwVh+g5oW-;wNeg+iSicU`02(szg)^Dw4Ems5&#P7SyNY!h_&g;yp zum%7bg?HGrMj1*FE;)_6vWIB)TnI$^d#2*>v1V8#QcGH@zx~{2-2_M#-K3aHm!xDz z_zJP~fAqPx*Ebbi`LJh{#2jn*z8$1P({`32SnZw8ggB>>t3vN=M#iB7W#s%G!gIMh zwas5Rm)yqKL19DpC$RTdeI3RpD4)n5?(~GB9}}j0S1!)+V7F*$idygO9~?~M_SPFZ zA#bm-wL5Dzf9`0?_Z|moquMloI@}TmLI3S0DDcW!XZAOpEIy!#0zQn2Zy6LD>2&`| z6?N@+^g~#FYSG!z)fHEsLFZ`Tbn}mXxv95J4>do8P1j}P(s{w{Y{7YMK~?ND*p7Pe zkkQ1Xg0HBkXh;Zo`}O+K?MV_0a6%UVWPn;T<6Bu(7OHL75l?ec5qLCgk9HkY@$nC5 z1`uRJkBeE~9~O}CI{m$^KfTxb!8>VPT|4ilZ)B9+f=nJ$5w!2Qun(=M1lXE*EgDq4 z#r2o}OV&_s4EJAJx9?O)d5@uyoOI5`YlUG^$`LYSdBfxaPI0k!N~B)1Hc}#X*(Q9X zo(HqcN3n)K`7IFde#S9tHf+8iVy@i|t6f%?mipMX$p#*nD`EEJ{04!VRw&yek!PGm7 zxVHAGWY_!Fq->zbOa8F^p5Cb)~mCjkh-akx?0&a$*Ea3!4p3???-G?Hv>5>iUR1>Td*k@H7Yr(#jhK-KA)dYA zbGR*QVeeoRhlV%aZ<7f*%rE-iRNzxV)4wJ~^%rqD!mlP*I=<2w^SF!`sEXd`Xw51V zd@Lz|#6=O1w^UjUHy^i0##0F#obgPS&?)fxcavC&Ubc3~;n>?Z02j&b?!3!?kcijM zxUFpelkA&A3ADe)f;5^@9v!s8iBZ=E#F*Ii2n1{7f4D-h`)7V-OUI5XDk{#-Ci3#~ zU@}KRVK;7^CXzV&*Mx1~0k{jg=wgML4)hfj3FW_5<#l)W7BEdQ&&|#0r)Qk-!-u?u zEw$~G?4)J6onfa-f%@7d25-5)>MOZLb=fiaG5(hzNR#j?e+keMw9a8!?wh)o7(ba* z7ut%0Ko%@BEBoNUa3nUqs=Tr?WoDDp$Pa6i(&zP{U|`oNU*(*s3T4ztR?l5R^>X{H z_nJWW<1goIqLmml%`|i1%_7Xs-+r8;p}Mnz9p-up_&;Pk%AtAf9%fa->%SXioO}l< zho5e;GqE#w#pw+63Bqm9e;GD<)b;l|W1*_GjbXgj)ZZP`5*R_uEN-83w`rHGu~ER` z1;yO)8$iqzOr*C3f#IqCOg^)(KXQ%|L-O|%{BDCKO@}WCZ=*evHj_62j~NygmYS+F zTw}NSvk#0hB|3<-v*m=iKu6kdZwq>vbcM`#x*HZWE(Jh{>|;C}lBZ&=L9e@(=t5)l zsx<2PzLlOs?+E;d<=rN1r;S-k}z#lWy(D16q3^Z&2_0DAuhvR4epaml9#EAv0$>%d-HLlLBv}_ICBc=ji<`tiK^%w6Nt^IjmyXg607@d zx&d0DMrW7J=LAA^0pnpKfReAbo%oOW9(#ZJNbb;4o5Fz~zbWI)9DBoXbv^qWN8$Cm$iB(tKmA8(KkO`|Xd_WLzgH`ss9) zWxP0g#ZpLlhyT>nl>h0VcKgYslbo@h))(Bs+pC+66vF_pa0zRK1F8ZClct(co2+uA z6(XIQOiR=GiI+3yjqvXQV94G5{QR;ow5=dEDhw|cyF&>0HU!b)5LhzF)Wc+BDGZx^ zx-38MzU*-v%O4gvn#Y~?>%#Cb{b821TYE4Kl6_|ZvvtDf%=tu=KzFBt0ADudWjG{F z%uzI)B4Ot2AKcpPQJtCv;)z#=*J8y4>0Md+lCuQuH+!i3KSA1Lmx#0tj}X6pf1N^OPg+Yd4}I)I7KSKL|u=+nIHDL$09^Xrhxv|ysMu`zt%&Z zFlH3FKNBrx?00pg%s;>18$(ws z%>myYom%oD`QLh|Rw(H<%r)?&2{7mnr#yja6-`v{V`LlUYB=Aq7?8}+=VemhWQ=+= z!Cs_$h?#Wo*ptyNrQqrO`vgG~y_f04TYr-Xsz`KR1_pc`$;U_T) z2H7m;e^R4ByZmG+Nuih%)FyJ=dPB^^MD|(FIzkC8NS^!$wU#swbUFH*U0+ae9E88r z^SLS*Hac+rO{t(_?dmw^-gj%E*6D4@$gxL%|3JXa-4s|yEoY-ez>Y**p*71BQYcI*hiOANi_7^RBoxbK; zN<3tI*`ynIeT8MdR4kD(ZPipXH90AKH}1OLKSyI@0}!m@Gb0g(AhrQ(ix8(H_iD)k z#TWltF_DuNS;JVhCN@8}#6+T&lU;quuRr6-K-d?97pllBN6Q1z;s|@%bBhowHV%Zv4l<-`n{`ki;ALdQP81H{%0;UF)&xKP%6tGGAHb zPxn_B*XXhJ;s#gxn*9>mI#}gFya1z%-7h}0&-;MW`D#g1HC36xCIehHFMZ#=KRcCj zcB_cs6GBk4&{#%;^#VySk(Rz18wMCH8I7}%-$^}~B!2_dc{+V2%OOh>f_bHw5)eA~ zc04pDy>41ZLbW?DmZr}wG^pyIr0fra=&h3@i%EA=US4+(W#G%DLu&bC&U-?x8nEG| zbuyUt7-awpXqf!VeR3YX(U?$O8P@}-x}aEOqy;1gV0a2VIu7$yCpoeA!yo?U9Oi0T zOifVLZdkOSZWoYCVQFtb(v8@lIStjS0U^C>*gf4rWSM)V`Z~x8#p+qml!Nte0>?vJ5UQ zkCGSc0_RRWV^7e~ZliS!#-;`;{$d~F-~8+MD_AE31H!iZouXgxl>7G6_D?^;W~;{oSY2kc6tfzr6yE;|zeQi*GkUO@VT6%kvd{1b(znarF?q@7t4-1b5${*$Bj5pJkr$g2X%BsyI2$>m4hY9PgC(mZ3OQAKz7Y>#~Mou9wHC)TnXu zS5cX}MtB&PvqoNLq)+=HFUoG=JYKEDQBQSA^OYVB^u)TuK;AX(7ug+r6QUZy(-HX+CXwEt& zncoJ3sH?Q$IN0tb8y+I{ofyR&S$6v4{NsU_Blpir_kluUc6QG4Pn_?jh1F`Od|=7b zLC=3Bv{E)VW?y_pN~3*GV8jqR5VKzDwEBM8PXy|?H!hd29)3be@#nwXNR~t9SN%}2 z00kH4@-aH{Cf*$K(a|yRE_&)Y^P%X~8Qpw~RO(xY_2YQcpgBH%eu7P<%GqK7nGw|| z)M+|^L(@ifMKRj=GMFi+f5iaA1Exkeq;( z58Y>Pxr6@+GFwTp5Cf|00iV?5H)pNze^RSfNAYC!7N^lD-I$SzE7u0OD`2SrlqzyK zm%l0j24j`wjTe5`mzL1=P-4dWiW^SO$H(GYQBgt5>`WbgcX@evduy<+I$37i^C;NC z|LHj738{dWlY)&+c`!5IGt9S$FgmuBN0qT)}CCJUV5E3Ofj99ig$i8+!m!p)!G{#{K)*QL@>Z{}&ei+NRIfca`L z4usNSp@xXr3VCSwhy8I^?>u{yJk>uL9yQoETE1^`e~Le|^Eu5gj!C>##h;JMq6oav z67oT`hXmXHnFuUHh_`APYARbBJM#K*a+jim>PV@m@*`3uSH+fBPXZr{;mKKTFekql zzStiXMKTtl4T}^*)P;~^*xeid*|V!tRT-D%h2^%;qZFM&a6b~%fHc_B^KZ@wgll(H zAGJLH8<);rMi>enWp)W&1xYI%`g&2c)X69(z~kfgMm`cK@gUj;_}wa9 zYQgL+E)xp)r1^{1w|iw`U-O>O&7hn8_`zDlvM_5hVPNo3TwaLMl4P*&{fO;%irS0F z$aP2#(MC+S6nDVkY<;EBd30&rFeP)R5iBW5`p`;VUVUrVvzl%l(8}w4uC6W(>r}+G z^P{7?a}3duk@y{LYSudM+}vzk6bKFr$rFXCKU26i<&lbE6(wJUVj=&8Gh9mZ%14@L zmFgU}&A!*tdI!z@M9?8fR`T>S3=5A^ZFdpn?~5}mTruPS5Bd_S%*=XN|9PkJAMmxD z4^hx2R*mH>!*Rxu5GqaPB4{}K`}Y-;_@k{u<|`9|jnOCa2>;_&)NDHG3~^XgSaD&Y z3|PXb2aueA&&npUPRze&i!WWm5_6_%rzWP#Q{L3)c6N2wTaG!4yZa4Xg|F%Vj3Y(; z$9S{AzWnt12QG(+q_uqD=vO>t@;D}9YUM2MtUK5|seNKX+LM=Ya;8{;Danb{v90)< zz^g%K6a~nj@Dk1)%2k>u$BU<{8D$j_+~bv(Wg`9`6K*Xhg}N=H$N;?u!kYnZH~4}Xn)S6aCN{=6M|PNkl#Eoa6iVRhqCq>d zNI>%Je3~wnbx|?NFEk`XQXCuY)BW}z?%jmvIU+ZVv~gb>R9a(oceJosN+^hm(g2_T z;loqlz2i=aTS|Zhu`oA}Q%cK(M);~5>;?58fK2Bpn!ME8cXYIJSb4FEa%#Jb0j;V9 z$`P@#M~^0ST@@=Azrh4gAv4UJ4VR;_qNvtt%3;MHFD7=}<~;i$m|3Z5^3w?7?p@tA zK&d{hx!*o>E3Z{j{mqsmlk?q*J_oKbgCnm%tMNzcz4rAy1454bjrHGeJi`LML;Hd| zJ0zb!f9@3n_USAn^kUb)e*N+>5AExvhDK?5xy?uSQ4=Ik4aCPs5$x)*viK)hj8`W7 zIbP`qciVaUgE4Jn$E?@Y`6lTzdV<1$u%9S4ntu#D+w^&Or&@FUV}N?bxeAA_ry1MW zg!g!N~0UR!`Cngl`>l|Ez84#wg%lt_}>#xgb|4C2CPo zq-iP@GGu%ujyBLfN_8Wx^bpvKr&WIHZ{w|^6Z{1-e&6YnN=tV!>LjZh7L|g3HF$>3 z;=PB2(>rmO=P#bGvzF_scv-4JNV<@BiX6?R;{_x0O?K(ZUrIJ-YJ<%akGDIfdh@y; zjS~W?65=YLJqVw;r}5EfkD+t9YDE+=@g>7Rft4fix=VvweRAU8T>jU*PbqOxsG8Ig z5#Q^FV5Hy?BnOU#Y%0HB8{S;qAm-#==C7^lo!#nx`<6C#r5+1{5zucH;rdxZ3Qn_; zEMhqjs9#!t-EjQ-m_xMA>$OX#ToYzSMpor7l{DcH^XSR4d+o^tv_qRJg8`iqVr_nG z8wUplTkl7nZQ-$|TTAS6SKXo;j$;0y#=#-FxmwPssi5-1${9dF9{Ydg&5} z9&eBBofqUE`%w6QfSEslOyTL$J^xOjs+H>O>qd>~>msfe7p%RYi-$);>Hrjjtvn5n zm0~fPL+efx36llf*;aj2^q8+L(<3Q_n~QvayaA!7yizvrVwjtKd!}Mo?vU?^(DdOAF`N|%9&QNGhqAP`xE{>W&p8!O;|%`NJS)_PUrTw_3ahrRjCJ%NSX&Q zJO(S=%AB{H@->GF*;DF9_~MkNoyX)a;8jH@$Wgca);eF$vN zc8Q`ll&xWEi@Lh(Wp!)p2}Mo|3Nl&3ovIp({>UHz2Z`59P3m76XHFFo3uuk=&hocPtptRsYyx(*$t#B$;9gCH8(H#A1bF*1oIVZ4$|B7 zbg<>rd-N5|Q_^M}{MIN)dtMu~+~WwBcGt*ab2reG zc`gK~+Q`e>IEM#Qz*6WTISWo_-r+ZPRU8}b5~^qGp%r+eRSJI?jJtcX&IFgf2PRi) zs;N=i_k34OsnpceTx+YY&`jZ|+Sw#$nuP+y0QfW>eihNGG0=RE$$1GFQ?amYTC|A2 zJF^tp6p*@y^{k;^gQ-<_TzaVMi!UztkPO(H8~up1)u!^LXmU}g;U?cO<*o)Lk7e=L zT2VM5cUGP=`&uYJXhI)7`|yfHPEV%jiCU4!b9W)`cDk#QZ~S43Yq!mq{uV9B1Divd z(*$+f@vS|Tnyp@FVyfE~g(nsUCPB6p1UY&k8H#^*Ii_*+ugv=rBqAo}8xlLa#Qr$V z|B5XqcnVKyIZpM~8EfNWTWu67)BO3bWJ`+3;y>Lfh5#g|>V;YV+-(BG_>O9BC2P7k z6Y={ukmfyJzV37`4>tJ}|95VQJ-Ykds;S~k7T>LB4>Wo!qq~k@D?u`T2c1)-Z7HJb zg%M4fHthh@)d;`Saw{NG#}6*)$k_4g&BwvSr{G;>CkASe`%U{CTh2%)=cmhz;0NeF z7iXAg3o^#4T(+lOuTB<>f#x8Sw#^qt7qe{9nRDnHViTkyn`&jprdA7A~RchT7K&(VLNqZd44~y%bBl?+ zVD7Q3epYi|o+H}~6s_aa+ik>b92|#^x-yk_P>ZT2PfL<49xEfY58r+~j;7D;0-w|4 z8pHRxmiP7K*7|x~I{o}40aP846RTAS4zd$jK0Qis%+QPWbL+huudR%t91@kPz@M6! zSa4c<4f>OB!w+?3?j)3z?ZJ4b{%RS?VLyvrlBiGAG?NoSXLot;Y|qm9`MG6zsW1gHGil5p0MT?SUfoy zV`&Wit>PuP=|$Bf6%bXhzkEhQLQYQmONp3L8RdGBUJ4(F<2f0h;vxRCOtsuV4|$iUN&TgkF4T%^#njx|80%{T%!z!TWTJ zYFH%ZId|aVLy4P9=epFPVDRh5@D3ELe%>)NeQrS_z0f;iCR^69OS)qNQqiyWPlt7Y z!Zlays0r6xS_@!!k-ik~&C70QUXqy%O!4ypZkYRL9Eexe{71FGj8@H@~n)+w#hd^ZC>rizaJPuV3fCs=DT=JPWSw1}=lqDPnIW>}SYytTY^8<%Ec zDg6QN&g_v~LgKtDhq1}kKBuPBFSzZ$RrD{-eUp($iP-~=Qz2B1y*>h<4fMac0Bz|fIF1~+XNFGJ49A&QmjHzR7(dp| z#)g(Z26m*9&KvSKz`-b{facel{!k2-t_%tHfLy)BQQOqzLZb(V&TqNd+OxCAey1RA z69ZDq;eAz4PcOZWM?NfQ$xHh1A-csJBLf3*kBTqRFBvNpG5JZW&@B%1cmLVg+R|3> zfTs1FY?8)wbLH^u*WQX-%2igO1+C zJCou&?pe;tD3^Vq-^j%q%DC*j^7@n7rIf5d6ecrs;izV*L9@gT%gd;4W^GDb;-SANeg- zAxmJ1aj?mjL&&I1TmBm$q7gN{o#BKywbfF|>QN)+>2K`c&@P8FW{d9G(VoAS_0qYt zubqEJOw1s(5rn@G${#TY14*at9jb9`IYdc-6kW`+cwSzlrm{+7^mofRKrC?r220|{4@)EfPBJ>y12#lp5l@4*joZ5`Odo(Ko)_^ManK1yTEsl+E3UOXZJM%_~8Z1BrJAZAcE<8o=~<2~`|E1{cC@Jc3Dl7j_Z8Lp4_y1pOoMboEd zA_9#)2i;?Zub*m2SX)o~BZZ%m5P|Xi9uM-A=R_}FfUEL?h=^LXjjycbup6u=DQOv3 zI7#%^Rp=^6oIo7M+kq*NWjugT8rv;4^*LvWb1WI|b+or1HJx%uRB9VKD{GiS^@U0= zbg3XuPkO2sPeG-9;Tv3XZy$g0ih*Slkj#A!Z415^+j{zO5_}Ge=AE_F=DxjIwF}Au zG8aF?(@wVr8SSn_d_8vDN(d&&@k``usDrv=Gn@`4rfB4k_2&Jj{J}6C)X3Pv7kgq3 zS*3Ej${6&N)>c)Tq3=RT9sRMSDdQJk$)u?D_3@d?&A3BBCBU6D&|lvzDs}$p zmQ3d^H=AaaDL?8Upagwm|liCl)uC#q0M()j6yAiMhRT3=66q2VPY2HN6G# zJ6lA8pr96V|H zQ-N4j-w`epxY%dMJH28h4xDM9(ZRg0uLd|8znBM`gJCfmJ0mF-^l()|z@(OFP#G_r z@JN?BK1(^_H9*p8Uj!JL#@KIG_ZF$wkXX$Qug@enf_=-I=|hA!)hnH+4oq| zTDO{}r|x9!gzc#5hB&aIo6m4yE7Sc@Q&mjyl200p_rZ@=AZ-rz6}mD#36>;Dzt?KW zNhore2AIN?my$gXXRk&h%M@-rfM!=FY=`OpumF5{#f0q5JN%`M4ViUC{+H(#8@XUs z>;WxWt+~0|**U*~eYpnr>lM;Cy^11P~jLTa5BOfSZ6UX z;@&j}-r9cGHjz_~II3N7NMKufk`~Sz+TSmpvMqQi4k>Kv`4#5KicMI-Mg1fsV zNN|Thg1ftWaCdE7gKO|c=H`)a-gjor)Ss+QFQB@*>Xvi%zV@QumcUv`*L`i;fc&u! zPdA2F&H=`(tD2fNBbF%4nxVjt*_BBb{IZtIJvpxB#jS!iSGP+k2Ty*^`SwYa|A z>Of!?h>NGR3S@qK=mi8~L`ee9K^je__Ps*$oQd_^Z{i)EUzXC0;-Om=JO(ZX>nuxL|d$d%sBbw;YkC159DVPav%CnR)>g3Hkh^q}Nm5-6yE z^MO6IHG)_M#tRQ;XAkQ|1C^eKVe|w;)=LSy4DgSk8+SKF)*EC3ic>mDN~5VhrGvPb zu3qXzmz5Wc!;Jph^g6)3o}jk^{MR-b^R}9No;T5f#M-y_S5V}79~mqO&HDYO(pn@R zLn}PQZ=(XS($ccY;kA8!_cw|<dLkpJSOfHso)UkH-WDApkEHRlCv0MrV=-+O&SN z+F*-MS5qaIkkglH$Z4(dIH@2QMieh!S(ofJ(?Ayj_N&Ac=hzt;pJbStJb}wDCb{VZ z=?_@yMOlKUxIuu6TB3tXXC(O@kCmjB{D|F4m@j<(tYZ4K(3gE|dlG6TdIBDo2O~v6 zi(IWyLy2x^ttW8_@f@6-ptiNX%m;@M*} z$>6k?^o!H^qq8Ywmumj=^X45bTNM&%*XyC9vkz6`LsMW0>#|Fk@aolT`uP_|N+|pKQx`6 z8Sc{`Qx!Z_cg~i^zot#MUhXCNFQ?l52CUW8N|R%v#TpIJmLtBHnVGS5K8bAtt6*Ry zrlz4u9K+)jjtTT!_a6eJ%0aEp7WnOCha)hY`abP#^WE0quec8D-et)9(4{P?qf3r< z(2GUGjG~3xb6nG%0ss8#H?f7+ETDB9)ZA%a)9QI4qn4z~%cw0XYS=c+Hc(-*lDerf zVOU$%adg#lW7Kxvym>}1{+3us0O6CfsM)gjwj6;%jBBgG_9ZwA)YkU=AeR*% z=$=JsdGW|Na_Bo&u;uy=e)00PJH>uSC}OYuLVfOP%c4^{k+`$z;f-&e7q! zlKdkBtI+W%+kleJN6*cm_GaHkxp#P`__z;JI3IIku`>61rv>qh4uyXteCTd^Ggh z9|imC6Z^L7yQ%;LY`2*afIZrBr*3XXT6EtJXKuTu>reD=y&dW2djNapXug~NUgDH6 z;*Zx;fne25&@Uic8JH{y_NvkRF)ZY!Tqd|Bpjmed$>Yu>U!l*H`ynu#lU9PE{Bf^c zfC_x@?b0Tj{oh%~vn$@w@qM>oldk}NhLMi$T<-gLBhet+z@Gzlg@D(Af6Gz0IbkFL8uIs_iy8GTAOu7)hq6{w*lk|Ep`v{1yFWYl zbuKBtT(G0CnNm3g-GwtjEn0IxiQX3mP^9&u(NX@#yTl!hO_Ou?`>aU*?2lFb21e9i zVu~OiX4?xkRFV3wG$pe6dwYF*(acOFS{U>T`J4|=g170_OX?3M2B{TNeeXs)kp&f3 z=FoT>Oj=L^A2^>WZO+p}_9TuDyOlJe_K$aY`U@d_ zBsL#e`)Pi=9-W>9*`770QxNbwJ9hT?qXc9UFnwGO}q^dL#1Q z--IE<@cg>sXuKxI1oz@EBM3v*hNo@JT7gCWV;eHi`aBARgs8B54lx0BZTI`(*}hp` zZVLaCaSjm$~31RIMm$in|Fw?_y<` zH8ZQX0O}*JrUCaRXy3a2H=yU?0dn(!J%1^e_7IS`L4X`1+Wr9ue~0eQsBR_1a>WtL8eRo{FN4y*?pVlT+?{2Vts536v@wRx+~t-EMbbs zFdwji(@hQo6Lb0TS4)%g@s3IfNrDaYUL*nk*)P8mfEM|KRP#V)X6MhQCKaAoIz`j#;NGllz4TEKIMRIAciK-)T)I*|BI8q7RHW~E>Q;46 z8V9jF9VY5H|G_^T=^azuMrPr!trjB!(F7~g++kzN3&rMTFhW1JdvLq%a8vd}l%Bhs z-(UavNbfw&C%81TVR{0@lRSxH=TwPo&{QG&5o%4CV8V`#S!%E_j}rdu&6~m%7X?_g zLsbf@vuaRtWV^!NrggAhvUs-_jRuB??@M3d4s7O|p8>_9qu6k`+R)=6s3UsEe{)YK zq%1q-9?uu0Jj!%C3|B8}I{M}T$ooapWW-+?eV|KR%x!1S^$^6!qJo0*y;H&Zs-Qv2 zhGi((4ef@Z!L0!_5CmMHk!{|+(F^-!t-G4WVyw=G%Qu^`ksf8Jw>z=O;Z*2HJK{v+ zb8{zMk3`*Ey~+4*D{n-Hrvod)Zpl3;K8cT6kFX6?fhnnGe@5VZmAsEH)$Ow1YN2l3 z2%wl_uf=1Grh-?zA8T4O)_}c+8X~?K_x*eG*_=$njoapfp&^QtI8^lec*p=c#b7=< zs6IyWKb#mu3fLX-Y(u4GA|X{KR{%2On?~qbURhrdNxNo?lE-CS*IdR%qIgN1yqyN3119GNx|fpwsHoolh!wyImf z#V0K4izSzcXoenXb2Ff7?Sm>Rm_5)WUbV0c=0|~?^foZF=y@Do1j7Xr9dy>K-`oQb zciiEafQtHT)U|jbxE?+;rJBj{PLpzVvdP?11!>l8>ZP~)lcNGc@cknKQyZG%pDE42){GNLyh240 z#Q19Ersd|hg_BdwQ~iXVg~bcmc4w@;>8QbLyh9l@gFY_`_M{d>uNKRx0aK}F#d_Jw zOyx&L39L=>)BQNB*Qc2D}{VUF2=z<^j+iyDoYl0P|mQ*Z>Nf{%CX*9AAK z>U!*!JGNpTuk>p*81T8Qu1~jfoN5GGfzP?^>KlHkX8B*^z=p;7xl2;h2l)QgOY~~Y zFYMKosbZyy#@(9&Y0T47yP3UXh`I-4(Q4^%o171Sr6DlYo1U(-8)SZlJ*=y#DbcJl zhuF03Tpa2qlb;SN(-qO#+H7{g{Qx#$&+9SNR6;Mt5=K6kLyab9IcuhI8NU$}ra^_}BLKB+HL0{dqCDFw17H9Mr2cK| zOzL-@<2706vksGdv^-vH{FTVHOl!zwGZF9!^vON_NL+}sB?lC+Ym?{=!x8}d+b_}J z{%PwreE-x0bcKMV42b5?-@rupqp#zf9|B~SfbGez(xs*+_Rj&er#$rQ{U+W3T(EmA zr228ydxY;HX3oi7!pp$HPb*8wu&tz}rL#Wl9!nP?Pg`xTO<~XNH3iaNSd9UU7RzZYrOn3`G8Wjlg!pX;H?XwVq6GKJw5cx-Lk#5>LM z#I8^l*-2Ugf$SFL$P6o~=FQbRowWEAObiT)ed;3&%B!l{-0#|6brMYa`Z_vf^k^H5 zQyf!=g5?Ug$-kx-aEcZf_h*K-529_l|XV$CqxY>w7?C zK1G^<4ex5+>jK&W*DF{gL?k$*?xin>R=mwV@7{5&w!-$>Ccung!y+LC!$!iwAp&k; z_b6(;-;g;cK&@w6B>YuIV)x;5&s;@8e?i7&7V=}~D0bNE7Wvsyu4+{^6+kzwD{jVk z_T@`oUp}Dvl$YC9CSjM7Xn1q;vWL*f+Mw15X!Z_KzPPs8@PQ5YJiF;l$Nq=;bn^OfSFGE6y)r)a*G~oJCBJO8U%?6=Cw?J692+=$ zmjBKz{An(0U?8Gsds0n12{@y3v|SD%XImYaz8=Tqh<_t_z*(d#0AJi{mr0LfFHOrD zYV=xkH2Js4I4||F3lH#2{YRxr)}R>#aGME5eM(f-I8zL`xMXkDkj5086F6*L&;I&T zQ7O@G?nI}*_Itk*l*4sie#C;`iD=gAEMdt(%6GZ3m z_F(M^1rybaet@xiI=-MI12`ZxH=w=tJ|oq(PZClo7FT@ioH(YVqyIlAW4o$tR3n|U zev4=qNq3~X)ri!bx}E~U81IUD#_MT?IoO4&yXx`s%eQ=VZ?LOIXpOrIx?sV zzrFRX8AG02Y4n~}X_V)!5%VCno@+Fh`#A8e*$oT&SiE?iPMP6J77yLl0<5Hahmwr9 zTYBrkCwf}0HV3n8F1<$>oQ3}0+O}S_|Tv@e=2+8AqUo_e%@AGe4qlA;3n@f0^tiKl z0pc&I*+eQ&2q$oS(C=bzq4~%i(o%Mw*@v)}gFLv@%>4v>{P#FlHU*XR|6a&68se(b z9UMGRWG?4_fOE0r1iDo?eY9D1rMnD!E)a8fA*QDhh|R&hL(a*eQuh0D&Jh_v=^i*m=El$G_Y~;Z_~xW4gT}6{)a=e zpb>$T$=D^mSlNA;**3Cns9sH7AqE>+q%)U>HyO`;U;Ot3g7bS!{~s@yPu~ybD;S?z zg-=dk%86?PZ`z03`<(H&wGPutU(-K*AaGnG>#6J(d1m<2N?Cc4nweZN`(84=paX;4 zKkoyUN8Z0MxTpU}Ps^Nn{gOl~C9kYHtB;(cpqQ`&x1UR1SSI9e+W%&H;D6G zS8PNp#Ao{Re*E`rPT-b`Q8AIERumjFmdasgyFtz9zZ3o94gbF$sbavqOtni0z87-PLJH{GzPpGJx->%T$N)iO*FzveBIT zhV7wWYN7vrS8KGaYyl(p&ORkO8;I8af+`+knd0UjBmci1uH&3nJxHb4=nf&%V`-9ptQ%cr0C(L`mdNm{dXH?vee*pmFu zYa^=ge}9=@&?A{N#iP>=P;Z!m^X260q&^4Qii^cIsF6|bitv|8!1op>++#`^15v?$ zm!pzqn0&9Dq0(TCybpWtQ4GxE|GoA9yi!M1VPub{648;48JlQ|KN+C;WJyG`2tMnt zk}0H>AQ{_IahdXjxBve5Dttm{WfDiGr~AjLyqC$|YzxDoF#q$xct!E=0|kEIK5m2| zD@YbrI`ibb`1Q@Z@qDg$$>3s@Og(3;P2!wd7Q)}?iCN>7Zchzze+x zf6r`;_{67GnXmKTNJp5}m$~>oDG3?_!Fi|`fq@U~`}45>-&c4&*$Cr?EI5DEDdPbp z;Lr?{3UxHnY0ouv1;Ecd`=R=GY*VuSNwLbgC7A_aEC8n)tp1%z9C)_e$4~_G&pi! z*?G@rbI<7TgdIG;deJoRa;5}I)eBTA5w@@?a_=y1H#A!ky>|6^9vdo8)bpG_5T;yv zwt&!8^wbH}$9{!l3Ununz_jHH>XFv{bFY@xD-YNU#p6i8P$iubX%XB@$v99hG!;0> zQo+cC*x*-IsGAV{IX%q?4}Y*!BVd`}#_RH#FXjxXm}L(e{;Z!VBMZw29{-GG)w$rd zlp}n%Pt-2NJ9O7^r7l|Vm;P;!81NW3P!sC4Ol6(F(A{dJmB1vM=PG2z{}^lWeqF9b z6bv#7a{?XibR^s;{|uGqT)e!mZ(4;oPqI+2KKY3doUGw_v*A|;`d*TZ4RAjgqI6@T zIHEm=8KeGh8oEaodN-%J1N#cMBz&SA`I3+{lTeK+gzuGA*sx*N(7OmYNEB+IudorV zDBz=_794xCsHwMdlv(>=1I4%_s`ZYuBd z#s_J+HHj@1MHa%+@o!3BwR8-!u3flJFd-c*ud_-ZmgY6MvTqG?Z8iEXv9t(V0`6qK z!HoI2reRDCLNK>5#O!%k;0E-q=;3?)wF#ay z{Pn9qQ?v`TDi%MEPtV?2m6Im%Cw7gK7$HBSg?$BB)CMefzs@P(6-LdY&~#0a>BPJ= zj!zXN#5x7m~`G-1*oQSkAv=ExP2y>CC~iKI6)+#I^B+^q#3qmE%K7EJ@D z4fJAm2?Ym-qPR|j0Z>%#la8Aq%gf)sct5AyA?ot-!wcy^?tfv5ftM>!fMORT6q}Ek z#1Z#V;xU7{qmI$N3+K>X$w49l7X;gXA{|4wZJ_s1W1e>rOYL)k9}>v>9Pq^Ihhd|j zqG~zsppRBkO5t+r@P>i6f>hpwbLoqHDBI4#)+gkrOJQ_cN`L0|$*uLav;XfU>SM`^(ocSVbFtqRV2H)!Nb&v7qC`hVJ4_zMz z{4>%glx~kV*~@-Y=FfN6!Ld}bZzF@?5fH!@7zq&$0c@2K;5w5EOXOykXG$$5ffR&S z*Tcf%PrHbwhx0MG1FmuJHV*&`+g8*M!yd}-ahVJ#EFe)C?O99W6Nlh@$1(Vj+jcB{ z!Mb4s5Fmv<2zIhY;o%d1l(bS{%!>ZGoGV0!DOPkqD+M{FJ0SEUNq+`(Q^EmaVo{P@ z_diWBQ2Qj})iSNsqulQ%s|zNv(^_4Zn_st?da?@qV_ntF#!f{#yvjL3qbK!#$GIS7OIc2j=Ve~?Ls9vA`;aSePf#6XF5GL)PB5G zAl&T%8yfC+T)3!aU@^nm*T+KDrGay45FncM0eN-dWd_kuDNSB}*}Evck>xVk{+~uw zVQeQVMbxlxa4jdkq4KcBXs7=?f;E;mi!Ve$H3PVMBw1 zLi_`3osbODjLKy-^^57J&q0bSMi;_XMQmtn8v1Iyr10m@8Gc^XL{&>QgW!tB_PJ$q zj@@X)74aVj8&lf2IBp=5LEt6vi^?UnDD)yI&qdtiir4nN!zhm+6K5;C7{@e+E{7d| zDP|TjY_25<-VPB)`Lgb2vGuilJq(@=Yy9W4giGrc4{L^ z9|4BAJr^$L;8B8;V2(6&D$9O8tQceRv^m8t8wV^{caol_4(gju{%>kUs`=C^1u8l| zWXzh5e2A6wqydw#e3T9Dr4v%OeG0UE%uIt!>d|t|xO;@F}`jy1L?*J-76w;QS8fGt^ z-%dN9S1#uCEc2U+aK?*wONrx#%V8GhcDztMWfc;UdnWQ$(%<$4ApvH7jh3bPd~Pjp zv=sGASFn;n*)oRBvuI7NXrOgNOI~)G{6>R|jFc{EJqZm^5&07-J!dUmA0K&y&o6^3 zhp7s?)clo`Gpi3Ul3bOk^j#p8HW{xff-RHsOR!{2Hk_v;;C;~i4Do!%lm}HqV$-UO zhZ~?H9+qkrdC$W`ul!r8m*dGh$WuXB1F!|(RTxO1z+4jt)`p9Cjuw{|)+P?;BUid# z6rP_u#$?&-&kR@Y<6vVyuD(Dfe?D{9;(mYY1bc#t&s7{567*TdetQBDe>Qf9h1laf z7@!^@{Sm~1mp9mHWC4zJ6-~FlRZ5@r=W{7WM0T5Js9>;>Yspj?eakGO`^i=FhNC)k zSlY(Ss*oK=%Tip8&D=m1R`Y;Xe2q{TW{1iDYEF9a9&xQvn<3@)O=~gM>#ujJJy%L} zU&d(jlAR~R>sB?ATa00sUwqQYfEPP6M(F3N#l*t9?AlNec~$E9Kup*C5q}fPD&JXy-8P(7Yx);c|J$jlYJj)bwBXgHKZB`s z@LoaUy>pZ&F~ey0Lgu35-$U6b_gP1vrHpkBpAMQx$*phje}zowa1r&g=g}D~`AAWE zygDj*Nl0KId_Ff)4G*R{96=1@`I-IA1?T-;eJ(s=%=-3}ND~z9{<^U~(lSyxdl8c38^?WPXNJ+HIwc zxzf&rc6Fnm+||*Im^5=WjXA8-PG()U>@x}sI zdzvpB0YT$pn1#SJxEjcDT(2tvfe}aQEs=nIeQl{)Ya6TJir-;kikGi3ZjR1C+$t-_ zG;WBb;77;BjI&90-QS03qb%#8;)Eg-+BDh1)dAvI-)i$wa+Ibys<$~3@x)#k=h};a zFo&_vtb4Rc{oiEchCgp)3Jnk;Rub{L1#}Je>6Q;*<0R-)&S2m4Mfl4BEg?h4hYFDp6TkwOH%? zN$4R<;r&Myjr3n+<2;ZoM5)?9#e4y}<*5(-u2{O62VbL@m;^UjctY|D;C^0qyJ8H^ab*#1$;6EoPW#&W z%I*8O0OObUp6{SX9>>iamrYc1L$Y#m-EW*ep+2bB+h2q@?a!>k!OkL4Wbw<$=??#4 ze(x|2fVn^Txtt`m&Q=W<(X~}{szGKIbK5Nes*Qx~?aq1j*B3_A7Uq|dY|{B%Mx9Nk>k- z&B>u<6nj(nBZx)rqt2X43Z3XXs;M44bhP;p6*UW(N<7QHFq~K~0^w4HaRG1AA{gPm zHT>-)sF=D^$vU5w9yC z8}ClKD>0i~mC)M!9QVnclIJ>zx&#jJuW%FO zc?z@auJct-Vp*~s4Qeft_zpC9@f$)3h0uBEb)zLk5kY-d#5E$CrU)0iz;^-f<);jL z+pE=%SE2q2Tq&)Pef`<6tP-n|{l|Ng-cmw9ns%~ar!?wo*iPnh_$>vtpdFPbd}M5A z0Obw+GrEEbbN!<79)&lUkJNⅆ9WPP;Hh3ehgXFN}X-U>xL*<#dO-A8J5)~@axhY zHb)Wup)c%Qunk0X&I3AmYF!422%Yq_qa9@DC}%Pc$4QPTHe&|L@BOcnGE_K4H;x3P z>4+*>IBWRv!@ps^nv7c_$HYgkq=a(KuJv>WT7E_NY<0BEC`zau>+A})f;@Xh%I7jM zc$+CNv^HT#&IxbrvoW5j@jjHyq{h1yl8)O3>;GA&~Iyu2;m9parY~M z$BfX!ZRnBPD!@TW1W8KY{<=pl(W&vRqe_-}9$6JBjq^T|6hR1_M4lALb`KVrFcn2>baS&UiN4*R`ILuFG_oZ)-6Ft4BiY zb*5^}0}*Rh#8FYux=5109m;^T1|Vs?+g6eo3}x^4Hi{hDap( z+nCod(mZQbVYgTSVb}2?*=P#AKVKx6u`cZ#v7d(W`nUTpl?WZ$dYN#f@~eln)!v@h zzRBishlT5LnM{RBr8X!Dr%P|UciF9iVl67a{)kuo$}3Em^nq_46{XU_{xtq)%Wf{; zL|=J?=0PmW93!Ew`ckDx9`0XQ+SGrP{2KZVO-*E}dW?+|j$kiwB+3Yz;MD>R4(cEe z?y3B0*~Pm0j-rG|c3ys$ec{t?<_GieHT-`-0%m?^JwNP$j=qqyGLpnf`T!~xn-+da zhNc{Y4{dXjf%q&6`gW?ArlY78Nc}= zcVanOlkCJD`W|o7?N1388F?mzv})_XAO`J?@CVpy1YiDg1)xl0wYQqid3$b*Q(3b6 z!{|BG>!8sR+2X+LQ@tt5_*Ws7d}(}Pi>V;BDleA^ofP@w5xCu;k8_D3@pALZ4T&X^ zHr0Dk8B4~Wz*ut-mKJRQ%HKkg; zi5+&w404bxArC?PE8euMPQ;4Sj80aWqT!%UrqCr;dNfsoGl8m8x&U8a)q1<<(c;~! zr5#9z75pLW3sL>1RykqrMFAOZtc3>fLJABEP&}NVLsMTfL_LZ453Qq426%F{oS~Y5 z{Dz|wmqCj}feWiIU&2re+}V}(oMZ}oiexD>cHC2aYW;*L&IQv46iTu}CC3nlr<+1p z8n(#Rb+R5xvHt{K`2e#R$n3&U4zYjrFzd4HGsv-2G-n@hcnP3k-ujo=WPhD)7JyvD z-wFHKYMAre@j1fA9NPH*)dHNp?w+nG6;1*LB|i}GXrr-3~Ck@GNJOb^HWof@3&p;>`>8q>r$hpz?LXF zCWg&*b#*6E<1P#j2nv9i|oS4|F;+tQXWO&CJgLGeiPbnbIIa`Hd8{JQPw^k- zR?a%m8ctc#^!4bz6L}Lj0cWs5^iC{!CZxI76yMW*ZH+fqEwUKPkVCV<<)$9{(0|D& z?px)WDT}e`&MvFeV=RsX+ph;yQg&GRq7ltGo1*RzC^g@f zMzYPNo^rfefG}Bpb}((4s(Fm4QPB%#TDl?DgdI!HqJeQtGXsKMN!5D3vD}G?FiuI1 zzl0Y>#Dm2n9?C~f#5SU_lKO==Qe{hJVx5MT9XaWfpEVXN3rGi7xyLqHpcOsY#Mci< zp10)t8*MTrhBh8g^(U9XlS^C)^%O58VOO*XK)0Ov zf<8(2f{^S2z$xDWaT`$#8wClAqUAEc;NPFl5`)=Xh`(J7ThAV3VMy0etG8}MXb%p^ z1pE=W?b`qdLEE3H@nO>QYXg6_Q{MyMk9FP+P{KmD)7qkE4ds8{wNa;jzj&EtF{x{} zl@pYRN-oiIJF!2(X?FvPE_w~-JQ;GHR?tto%v#}U^Hk~p28pa*rbnCX1(*>)n~m4s zoaT9ZS~#uS%nR}13`+kr4*E- zGEntH`dl-gvo&;M)S9Q5fzZqGUAL_!|Dlu)&U`*w(s1MLOiQa$3ul1V5Bn-eU*b?7 zDM8IV?+an;C=sT8s-sDEOaIreT^rFW!Y=$ zGD@UDRx}tI0W&5Nv}8kHa=cHYtB*i0V@vXO3rDD{cSD>_xEDpI;BzM5=f(|{1P1kSK)kvQP|0zXC0tkgnY$q2N}vv2xyzMRb+GA z+0A3>*#f@Lhm*;V1Y~4_0G)RVTsA!^6{PK?m3nq7sHo8!WaPa@VSLc!KOoA}`Dp$W z1UTu5l~4>Gx}QH`n}39k!XqJ3CF}hJ>|GhAziWaZ291L8OB*`NDIBPIwBGJnaM>(X z+uzvdfp(&yjN@SxQ4Ht#Z+sgf4mw@uzYVyV>o>X_oZkl36($48GwIrBGguz6eyZ0VhE>dUSD*TnH;3Lfko?w2&2D2UahcdVUb+kt{at! zyxVy+HE@GQTg5CeMe=aXkn%Wl}kdR1rRy+oWijl1$leofkN ze|*=O2=hufDHy|T^~5MB-kzvh_t#$5noXKS$-04<%hvl+g6N9t^KOcPIgmMZCFqJB zU17UMCR}6X?Cjjsx>OKMMwSU%7hoZ-+rmK!}mF8)%B6E#8bNEsU&!qgmr#&k_buHs!ZEr=EPUe$MHhOUyhBBKkFp#WJ}5!`mJrbFsy3+dJQ4r$0uH1I7ny#ybI#TYAi1M zmnk>~Hip4r1@rzmx-=eoD%V*ScgSwg?$i-!sV`V zOP%IfZ*pucmWuLw&eGg+JxgK7y~^Q_J^zWrUlZwh4_8C#8KKoKfQD&--Z=$9w%#w-b5}p-AjX?(c8YLLaBt&F-uK+ zT+2G$N!*K%4foJbPvhViNY6XUkFB(D20jZ_>n*92@hxV<8+Q%u(8~!i_Bf{uXPa13 zdC-`mZsG-eL4mo24O?xdS8_&RG)8qAVn~kNtM`Pde8LI38 zeK{yxnT&<1YScwgA%*k4a**k&XEejCGu3+!0!n35{87-~&c?@AA7?B3I^O@aQ2SVq zx@4FjV7bF0oCL&VsL|2>79cuYRJf`$5ShBV!DEz(>koVOHml1=xCuARgl>|m&f_=A zdGh-95X|N4PC_*yX?N`4@}@qcYs0~6FAP_vvrp(lEMc-DG(vLhm)@MUb>YT;X6lzW zPDDB#csP{mNVtDRo!_z%df+wL8Db`xcg@9)*|wCuwKC8b&Bo zLtBqZa?|as{0As}{^Wy%&*VmoXbE2hF*HSdO~44~mp0nR946yT=(oh?oxy@iyosw7;LW&&JT$`>7(|dU;FlKZ*W(E-WkrPvG+=`{?_?zF$cngwb6=>a)Q8 zxr309P#^Lt;yb&EY>Tp{5JiDP)0c;+of4i8y9uPE8GU@^v1&z!4{wfz9;nS$KM3b! z78yF4&MS7fVSXjnyOw5~FSSuG;y8$X#d8w!)|Ip(qe94eKr9q1y6mDLFSXKZn-kJ+ zD>L$g9T`@pxVUH99*+X?A=WJW0s?rrt`e`G1?NJT<&dZxY(u1-OeXIe)jl zITBwMfonbNlvKpyG|i~N=-d-Cql?NV^WA+m{>i|pP;y05+~5*V`!nNLJp7V8OL2TZ zC|vkku4UdUPG)TS%nza<#|o9TT|_nG2OU^xbK?K(kp0Tph4ijWC=3#fIwi$6;B7&x zCCB>pnTo6HU!bj%tMbNJB?qxP$@iUFbE{fx`0yKpTK$5&-9k zMJ!c5q(h9`j|wc4125y=J6nv2>{w*KNlCcw->$mWhS_ar;4i|ym{=2FIBtHnyZfp$Shwuyp`vm=D){)t9Z&AVFOe(3Aq^`e zC#=BG(TK^AmIGL%rILgNdr8|J;99<0ml-9eTqVlTwr;F}yK3R&jAoX}Jvf>LI$!KG z?;a2+`3;t2r~)!I$L(-CBtQLQ@f?H{TGtp}@3b7v&_rdCu`==a*r5AVkC)HONfHn;42E5-unrP=NI`wrX5(Frq$0V_I*7F;j93KdAy|kcdq!Z z1D?L~a8PKZ1h|9ZLqkJEDYF8R0_>RsWsczp16B2`TTKMGyz@D_YnyfV(l~ua zD!AM@{Sc8HpHcFTi5$Q6lUdFlEmM z4q_<%@1x!_^Tp_prJBzi@W8*vfbU-_`>z(jQ_a+Jeg=?!3mJQ-QV2nD=;&yG5rT?l z$9(0B^i}Tn*FTE?Oph!Fy{7FHHWNyy^Ni#mz%0cEfHd2B1z*Z357j7oN zQ%n>ZSfh;M@5U{==I)t8ry zpeUR*Y4~DkoIfaoen^}%RJM1J{4P*Ju~XiUoHcY@OJRW`c6+jflRbjyg&8jl>(?M% zBTknhU8)f1t?dr}2hv6T5c~$+J=s$0@}EO*7ZiReHd9A^#(;4b+>|Idh>Kvzf_0L} zR7z97HT=wM5`A6M+$^icwMEygQ*OJ)KH6MwJUk%yaQg?)g(%pRDDp5U)6iV_mmH4o zgMKL-SaS*x%~^!{{a>=P|cyhA2+V@{i$MC`3j_M@vRjXNcIe+{}>Zw<$pDuL3c*u3=+C*|f6Z z;WEi;>Rp)f*v(d9z?y?W5D*p&8 zF^DG}p6Rzf%-w20t*9g`oRHLbjrhA!iPY|}FOy4qA*_}vRA@AUzpvC+JPRIKMQoyd z?W?1ut*L#2xN*{8Up_ugQ9Pmb`~2=PB7$^QcGqqz^KXZw7*s+5zs4-GIWqOAkSObh z;+G=$d_M1pHTeWn$1F~HeOFTVAOEsHY1?Cj2lHpJG7XDK2Ll^F@_^qzW%M&&D`n3) zKEL_U&bWbXvm*^u@d*Xn8;Om1YeWDd=!2!Ob!}PGdKD6Jr~7@D9kRyrEhag*I9~<4 z+z_Cw+b28q>ZeR7(Qk|BsuM%Nq8&a38*<;3I2UY>fxGb(7#c=+&1w(q8L^6n&1)nhZukIWvO_2R&c$;PW_KD6 z9^QI(SF-lGz$eb&=B>HK#W~=QB;v78q!`=@`b2L3=ge~-^8xAVp*-_|_$U7JuX0S% z7N$M986xYQ+ooirn{iRx1hnt8qLjw+@DnZGYEA1wzR9W)@KrFEL*{m(S`xbJ7(n^k z91}hySGluebf1l`Guojh=#Zk5xTH{;f2V&h%ROl%$nx?_ln3?*t0`ZE65>s+YEI*@ z3e1tx)M|uitqr0}r4rs^dwvdCtS?rLMPOpr46|!l>cgnwlKhWDKW#D1? zw?71B!~qtFR7oSo@8&JBej--mG~vbW!q#2?uB^l4uj|0Vkhits^qggx!UomB;}vyWjaG(XjU#waCkLgf zl9u7mW)+DGmd5;kLl{u+tddTV*E|A1o+nU`UTgUSg=5W8cwc2!@CGkW8s-nMU@Uv@ zrzE$cRme?I&eC|y=e!)n=G(hDPTZMubND@)#7CThusIzyyNyS)qV0FjQg=}% z+udr?s}F|B^4lywca~(bJU)GmBt|}5s7pYE-I`G;nvNifKXRPnW3}%x z71Ksx|LiY2yK;SsPIDx#VU5gV`&U{%=4T(B8v7<^>$1W^$9ed=C3|f{(YR{C#N@>d zKw20&bS^F`ia*2ZFJmsUbSdZ^-JRW252)KwssS9%$zP5lZD(0xM#?<ltTV5rrx?O%I|%{RS}R50qO3P8U!SyK{}S37UM&U28Sr9)#sMiJdaZ2X8ffg`+vkhy-ML@!n+w*M}Z z8SJY|h!3^lYJ|y?z5Xqvgzk*KW@-A~-8sV)N{KDv(X@%oOxAPHdBW8IjW-?jg01F8 zbjEm*f(&xV?{ir&P-!C~jp)e4{~T5(@V%gKrD#TjP-c?b;vQn0%|b4v{fqr_ zeb*2&gNXjpn*H6XH%0UMBfz-t1@!Jh7yIIOh@OKT2u_{O2W1zi7iW86xv2*3#~4^b zx9_Oe6Qn8hA5Pb%J61e;VraeC9xI+gKIA?eq*1hp%w79tvbY?_gZI~ku$HejkaNd# z`6zHcTAlp@K>B)CT@CW-rZa57rMdrRz8UNN+6O?B74h2ype(I`gc3jF&t_!#OT!I!lsyf{2rxA>87i4E&{Ee6 z77@$z*uu3~U#_u!^7HoCn&+z&`cgk|w4Bh?neSYh~?rU}M(s&546y zL1=JZEs#JBMMu8e1bOe>FCw-A%)cvn@K0A%L>s_=Pj-NlbXD94%8me)zWMHM0_Cy& zR-F8nmA#{3+z9khhdcJ|IUD&2fqI=#@TX$)Co=@KR!7awEUgD|Z4ZW?Ye#oRknC_W zSwE&R5qLQwA|;{qL3cx_QGy8NWa2!R);x0?O>X*A9~rIr1=vZK-jJiooa5ZcH+xv- z4g_XSl9Td`iT~bCltx@DmureY#d_Deir~<46$+?~a*mU}#O_u5IA)sqh-`Z#4yOs` zj>@Jo#O4NzSmrp*U*P-OL(i(`qD--p5^^?VHuoH~+a^Ln5GRCh)J;tb>Kpzm+ye^e zRhJuI+9QZnS7~{#Aj*4WTHqI2?zSJWdT-5DzICKE@PE=c?fBScJk*gywl+w~tgGRXJ@&m&W{^~rgK}bkQQ{&-a-ND7qY_hs0CYzX% z^1BFywuwDb5qb*%A1XJ0QKc=l;(R-p0D}pj2j%8&v4^z822P}nfLer~K{b%u;iOtFqJEyNg zkb$A&R1*!_Ni;7k>t7W0Y!L#{CzZ2}r%FSdds)?!|JEy{X(}hfk3^CO0bg0NI!4}< zRDHnd!0kej5H&aN160W>+8SM7Lp~v*1NQ|cb}fcY>Coc0Bx+Toi?ZZd&I8AG)eFEX zL@dV{7`ArzbepGz`1geF>B?sezj=l$TA3C0QM|fW< zRy!R50ZZwSoin7)^B;YpRs7c4)r*@{P$D@XHXY7XgUtQR%mjcKrEwy6@t=u%f%r6P z-jkc}!&i=WN*oMlfMU{dw|f)}l8#F~+ys)S(L-il07q$Mwee!z?`j~&BY2s6N0pnW!mIOm{a_n0X56D!v}M3#AijNG~#zo-(bqPg2;7Pa)eYj(p?CK z7hv_sL=exkUzr&Tp= zBH{rpi%_n*MFl&H%rTRmwdVd%aS6SlQ1*J7u7==Jqn{}iFn5_}CSlAWWoe`Sen(9= zoIx83jPCB5^WTWlaXd?DC5Zxswfgu5SfrLe7ad#KlYNmVEFhf`mWL&^p9<6A_-nhk z7;d%U&4RCopN~BbrpPlnD{z)wxq@7vfzb3+3qci}ZlU_uT)%0h#Fq@+>#>5mc9RY+ zyEd>?J#-LW{mqT1h$Sa&&eoQ~(e6-QAePATRkxXVaAy8=UQR&QRN{Cse^%f%i4bT{g>0$xaBg!pM+yLftLNR@-ZK~8D; z#P0f=irw?Co7Mhr7Vxbl9hJii-hAFHJyn`xCsuVTt6kOAwsn&j=|F9?9` zAuG_)eefUq{M;Ni^Q2S^G2N9W?)mK$sX+G0^a%Ut$b?!lwexsI3vkpaM&HyYPMhF9 zgvD7bj%`;qw&f3DGm$#$zzUe(?J>Q=8$Sg7{)SV z;;)d3y|}G0d0#&nyPY3SW*lfD9rO;X7`ZnjcEJ`o;s(1>$AsqR*DH`v?5z6r#@pjO zyazo9jV==^YB5bgVCZ5uSCLP+tPO84_Z2!O>*8w^oLrqy@B3l%wAveN2!#(zVcs_p zggHj)Z)7AS0tUZP@!O-3@Q6%lz+KHZYJJdIbW^kMd^Dy9`?#(|#@$X4W`CrDd;;S7 zOJi{k?}~dCmexfDxVRYUC|T7GA>=8hn4<;6=$|w(3ODn6`s)#u5{KX8kSyevOR%AH zZMyXf1v9gQq++|!sVLQB?aGuQav2Txh#RhAd2-s_2-34TXuJnSnTf!COq539EDsET zOhL<_Vf%tkE64viME}0{?OJz54!OUf+af*?Z1;V6x{>=Dh2q=OM8;D|J(PDd%)$X! z+%6jW-*=vMyDS*^W>@UwS&L(tlS#-8^83k)k<%+UIA9u&i=v%GCC@700A0^(gZTvm z@14Pf9=UdfuU}~{8_S9hmmE7FIJp0_nPz-vN$wc+n-d#nHL zhBw#3B*2!-`k94GgfD4ABm%w2{WA?PpR4wDz`^rp&aDbuk&B&gNy#swC_aIt2 zUM-1LB0}tKW00amRHfbK4+-2%KzU_LoAhga+!Fvo6(&ygG=Li*^8i;-2pYQx%rUj4 z&Tl$=y#eKG>?FLrx+D2#V)t%0RZYd5m)|!PP^-agH|ARZPfl z4CK?n-A(8b{EwzL1^v6vonMFnuxOvCk`4RQAkGpCI!WsYx73j`w3%KrVmJL zAhTn1#M0RGGHyJHlCNE*@?f8TL9<&wCVyCgwiw0B4>hF4>k|`Izz(fdU)CHfyj87> z71PWiT&jsv{ejoJUM}7w4iurs9~$0WL|p*IYSGglIK67P=5Z>lM3R9;0@=fiAGAth znO?`9rulilAlz({Cyqn0zy`kB+Qq()UcbGHb~ePN+a9M`qLdiO*8T45iD<6@l8Zbe z0Y1d{!T-?i6Xz-|?gTmy4u&P3?Jg9eW@C_7 zEX?JT`y%Z$rFr)E>ztF19T9ZScr?G($^Uds8MoNE@PWaDu-ya}&cD3D_&rAjH*5M} z&^Tu?^L6K&>c*$x$O^X`>>G<;b9tCW(5@#!b`hJjE+nF^j=pBNWtE7NBNZgdXzq8k z!3kcFx0;jfjN1^Sx$!91%#1xj`JAvKQszClpKu_Cl9IoocJR-u2pAz1 zl|Z8bva6ENzg~X81A0*Ol|DK*d@x=)QUC?%_M+aJTT$ET5CLmtZ=NO3^n>1aoQOo5 zm3#kSlyhuqV!-89%E|K?*vNc57*Y*G_q>YH%3>zk;Zvn$4+#wwet@GDNX3f1g)cVg z>V)&s2k{K-0x-aX0B2S@x~t1RUDZT*#00 zL0J>fZG7iKsSDPY0t>8MXD=nUNUH=BcLI>&3WKfc1O|Wwx25Pa#8_Ndk#44d5cmeL z=7}%d^SkWHR>1y%ZO=2ryPpBD(lsz$(?6Q{{0|VeAye@`>ph3Ly3_(tT$T|oW`+BG z*EYvN;M)#QFiVz?Sb@O~CwgqA4|&VurIo*^ORp+Mq89FBYeve3AeH0x1coVpc`qUD z`mUid1R2>1od5$<-umH(-!eCQ|1$w~-*q}CF&jRiXn{v5>rV^aKy5Xm-s`(kEAN_t zos?SCz~|VLPi}A7@oh7vr(G$>f)Pthse_QXzHCW8MOQSIQX@1Flq-bTA(51S&;SQl z`(Tmadj6qwGv5+KIX1)!$z~YUHP& znGOy8h={3g7wge88B)NjrM{G`$vsV|O1U!fM4Mms5}fUgtFuF{9b|4!3{B!DPTW?6 z(xOqF{vcaQFyjk}iqzg+Ef}6z0^4jrHJbyn7wA|q#xhM4))a2~{h(IY>UGlo?xfW6 zVm%q?-JJu_FHvXL-7~zVPU|hCiEVMfieWK7x(KWb;L{kz76YTuj+^jk#5k*upvyvM;g_|t)dqjCklB0M1I z$tDH>Bjx}5?HJMlK>LiVMMsB+K*|bld&iTSNXRu@!I1TUVYkFG3}0IiDARef6BR)p zIo%q5r_S5V)iWzgmVftHFj#CvxH$tO=_DuQ>7G}8p}Ia6Ecf)*wFkFWrEuy%H{c;# z*wn4v?}@OBIH_zi%nMqcIkH17tjdmtKf>D+i-*(I77gaQ>oc%(7ZFj`VvoL=hY7#G zsmy@xM}cZfHCbQJjkQJ-L|KWJtj^d}%#i?{rdG4$F&9$;?chny49%lQi*~NXb`zJF zpNzU8o~@1MVX@QU-)gxq5c@3g@y%yF9K z%fo=Ar`J1U^qofd)480$$95h?97;L^ly^+kj1LY6e78x70%q5;bqor_@(N_#IkAGX z(R#^)F`2Wa+X;`2SkU0+(W2(vdNsv`hDN{6OI;K`nN{NMHGrw~g^FrfoXM@m3+de4 zoAj_ahsAc*2KRw!Xh<0S-PY!2&l$pXO-)J3{%mq`VL%WPmLfT}$d5KU$@WnupOsiF*JY2#b$UKl=^*bH$b3Ti$wtu&j2aYnwjnN>EgpQm?amxT%P zq}t;5bbKqc?(;$vPMd*2Y)r;p21IjpP$5JGJ_4>!@kNTn-g##MXI6vt#~zHoRlXJx zyoeI$oRUVdvw;&Yh(@CRItc$Rz{zP+C7D!$<07bD84I;4F@tVrff_Jwbe$2jTu-m~Z)C-I&sndyEmqXS3;*hO^w)bI8Lq6_@JQ)Fa6?)k{Tm?0dL#v1&Z>QIT)NI(qASy|!K!=n#}}gb zUMT3pb)fUwErs9TYQ_+2F|ls=q6;^J0^oKe{&A`!P@$ovp`oRDXA?6lH{z3M?s}Rd zloL^Xcz8&9&(80N1z%emLlwvDCIF&bXMcs-RN;4D9af^Eq9_6G%S;o=vL9UnAI zRObr}cPw&2H+j4J#Ocjc`bHYBzKc$10=gtCmJ&&$yJ1?abc~KT z>B7R&<#F`f;1I`gfy8&3x3HPX@30RJlWy0{;^!?k(7zFE%47=zv9;@qt5zhNTcms@ zGQ~8Mh0qgT(r8HDhZ7Z^Es^Ll*q_EMBn?EG-G;o`aANsTf~eY+9P`=j!RbXNBJ+3# zj8_Bc!0E8ns1{L+7#*w`gp(12zIl&w=hhfmkM-)6 zQ1D0BM8*f~|9)b(Ur`Q;U;n9mYX)?%|K>K>YSBCt>@%qqZz$GmZzr>V;D%g4IZYW9 zvg#c>GtVyr#l;4X2UA;EB9}g< zZdlRPAH05E7H8yoSS`$SExhX8^CI;`k|`FpK=>Tl2)k1KNUIVAx0AM>A_*c!^ec5Nehtt*zMd3pX8Cy zNL_^jvxw%k6l1z3kYBpQby{nVja_=QfQ6fkM~ zXbqL92^L}y-QSwNWb)dA4%x5aU>T_vwtayBRr9oTvf6uK8H+^%dNY(abMU$D)K|zj zM}y(Nl0ut{Gctsys=)c-yDz|A?gbue(yJDn4;8@e6}i>8fMlqodj6})aQ|-T3;{=K zwlF8BYu};EAYDmfPwf{TdGw~3Z)4z54W`~e>q6VpY^~PJh~;Mj9+1btbv75Y*NtEKlNp+ zC`VWt!@=RBE{ta|Uz#8Y(Hgt_Z#@^NF6NLBr>no_y5-9f8MTs+wQ^lGlBx|XK_SMN zMxaV$(x!Im8qWObwpQCNc5r}6xZWLWb4`OdMTmj<#9!R?S7_`z6}sLqMH>yorCy); zej?K?ouCR3Hv^FdP5IDpN3|7LLQhV2G~nrgyb`idC-6XZWd8P4V_Q|EH8%8z_v!_u6dy3>?qHZ1|6Q zHacumasahi8yybl+3DX=r1V_5Wm7#8CoG@^- z6p5hXpBCHIud@l_nIw#XBWS}&VbL^l+fIj6+jIr!Tg$pK{K5p~Aq^H%^&$DA7Lhi< zVN-`N6Bdaayi*L*Y46C`pe=|NN-Le9El7=h6YI@}w?|SWsac1Z@jWAtH|993l2Jz5 zp}9(R4Q4cVL{yiMC;tw4$<7(Lwf>HvR0Cx0G{-^~!HuqJ84S{e&y&B_Zigm6OAa zDx6eDDlg{>JmcBD?fRGbLPHfbwSWDY)xtN++}VyblwX3ksz?b4K-T%00})a|0hz_( z0W_uqL0k`3QN{2w%19Q${!&ZQf;kuS1Mej7=PAwdilZ#Sdc8-#?X6`;BiYWm{=Q<`+j$Wq=43vDh|o*N!k ze!4GdX1OgQW1*}3xI6Hg&LC0J+2(zR+Z0)CnU?(yd9j*#&vhVd;pY*zS#=@$$xrp~ zFa04tnq@RvF+G^hP>I0n^DD1cuPCGP(L!<^+-G0A6~)vHSE}M&&A;C7ySGV!ZJ@P0(CwbIUKI1vuH|u#piWcH}&g8>#Li(*l=9W|mahmh$ z0p58amt9czFp7B?GMf z%F14TB%iPW7znpNmSnHCaZKy79I)Mc1^L}y5>3yfj}<$1zJ#(&i&~HQn zX4D9RbP>nr8+>y;Pk`2Bwq!GrvqE1NQJ4qB_3xEar=p{y&%Sq>5lL>#*>OOk%L3J# zrK3>LfA#b=cIM z6C?IkR-y*VI;xM4XcO!}D2@F|pL*^1*uKiE5i&h3uC3k8M8+{cqWy1OR{+^s#W*0{ z>~ff;nu(S5%%cU!Zq(Pmt+Sk{vR@)w)Z82W`IW00IKn;+Smar5mRD2+5kzH}DKyw9 zIF1AI#BEtL(qRFae=yLwBj94u^?Lwo%ezL`hgsBxxN}gmicx>qFVkQNVS!j&l@-25 z=wp=*mOgu*86&mWr~@nzyESb9o_QW?!Cj6FAahJNuUCs;v+jgOu@Cd_D4E3B{sTM*+lb`WN$C341{m8UlIQ7R@M^jfo+PIrboN_s8yC1Be&%exsd_-TMJ`V^ z;&x;6@4Q@E^V1D&qv7C76G}lF(32Xp=VzuK24BNq!~>ZK>e8Bg!@jYTf~SAlD=OZC z1zmPWN%-8ddc9gwzsf}dwT3tf5!meXG*me^Gjpq-UD6`v?HfUIilMQwm|$Bnaw2VQ zjfg)KZ`R^}dukGi0UxW7kA+A3-kCzZ<12y=Z5tNWSV2L7YGH*Nd!L#e0AUiYZ9sun z-Z{c@ln?a`7TIBZG(Dz_S?l1oj`hGeSva=er2S%n##AkbV{@U6Ei@z1M21C)1dK08tz?mCjm>i`tJVqT}GhyWpHK>m(@3Imv32z>ZQc(`qh!Hnw zz(;iR4VWB$ySqu5-WbO@(ck>eodMJ7vWn|V+3rkF4_nzs??hFz+DSU3k+NN8G%lF9 z_1*RBSARv~mxWz>zAW;86k3@NRA;-jy~{PZ z<~N%wv_BS5M5bdK=}psl6wO_DIkPx_f3nd7xVMm_nqj57gu*5sF;6=bC~5EVhNj7# zqdM-(Y*AbgOoZ<)X-F-AHyR4dO02t*m6|5kp?0i-bIreCfu>TmVtm-?cq`FM%&(a6 z2776JKwf|i8rg4trX#Yqww`)xB`EENV+@$Pn@PAovpz<1Pv8oc00!ZIX#YZ3R5kR8 zK2YSQr|-MnN7tGHRbO3j<&4E>ynV^^w256zP0D`*Ti{}FDEI%Lxz6Uw!5!kGrvqY_ z)1&7rxF}sf`GMBVpMXxQaxYWpy8KjCWqd#Yryv`K4H}p7;NbUrC4zp=RmyUSIjd4= zR~MS8$vA=UTd4-!uxD6tdE-1GWBHV@dQeL_bK=1}4neb$ zKPg_S;%g2_oYZMd6E+SzZ8S1rX){LRIbm7%cjyq7JPismmip^bsL2^mEx4c3*d`&EOh*R&xMty zA%8yqbxmesfdGO!s;)e3WUR#PcS0U#zjU7_M2_Z*FA5@Cu4Yy(33135pWZYAmBh(O zWp=ExBoRfwdA`~93)l$%Kpc+FiM&Q+yaY#;PpXRlF0>cE+Te0g$OO4|;-~WafgBYc z8=E6|v%Q(04}{kQvKgu=?|CQ6XvnA&wcVE);A?=*YmrHouE31{t2Z;CfaVZ+@c={< zG_*U$@xC7S$(%!ri($RRi>>ZWM>?#!*Z=hVKBIhi0A@VLs|_@>eirXDLruqEyxR)B zfV|Y!6ELU#zsQfIwSPiDRy2l<*1EH15{<4gT%iM=f}2^p*Uv*e{KJ$RVNQ(bjWzxW zG7pcf9IV172lv)Sv% zyHg}h=W&4FR)`}fd_X7f?TFbL#9_Y6G0^v|)J1F7W#R?Dds+RZpEqM}@Ia|-)qRWN z|9H-FXaMZT`yw4Z{jH|1AJ>xNw#HE<`-_?|4*~|U7tcM?*f}^DE|SE6KDFfV5Fz(9 z;5*!CbO!d6lAhYK|I=vuefi>~LS^mvQ5MZ(_AOqXZ>Lzoov0s-e#z47`g61UngNJ1 z`7+)+o(I>RHc}}mC1uCv(P%!P#bqyIjMw@09VvEkVexR|#PibwHb36ZLqUp_pjYkD zgC&;nJE0rr4`y(2;cuk;BGzb_88wn@MG28M>95W?ip&3R7C?MsR@!35e4VgJ{%X)KLqHh`{PDpol?(j-+jmjaURAo~4)uiuGsdTxWv1!Uk4+*`U*PHk5rZVRZ2yGjBk+{TWL!y7;n|zm(Pv^b zsnz^MO(pI}DYde#mzy)oYjLf(R!$u`7}o#y`qkO{?xd#6h&Dki39#sZ(R{Z~?uhit z_(=d_?(D-B!*4^tB?7J83humUdaEcf4J@etwV_Rb7L!UR-^xnsGqm2rNR|&5^$<~P zU+#1(rZ}^AuSasocf}2Pw~wt2>`dD$Btv7~qlp zsREQf(~e=T!wd*oQc@bX9kd2$nk7a6gUoJxmNXi=EVp};S^^LuI^fU+nme9%p0*2B zLrxFO#kv0~yto4!tM|iz>^@+i-iVjiLH|t4c3f)t-1L2_6xY`EZ1T%O!(X9D(|)sX z`^D7M)ZDZuS6Jz9EYeT!B7$xvdZaNE_*5h$XsuJHUJK=#*Yh#fwkJeGj7dJsZWxL8 zFw2y_ibTaKHpT7c8iRG^lf`|Kh#;E8Ra_jkAB=3hlE2OoVlT|cNkdEa#o^=x*#+_f zu>u_;-nNGJNSi{U6oN%~woz;w2dInNx=fsFnC>6CXuYPmWN_)`Ru6sCCc6UX7VMk7 zkC-=(ecj==A-e0NF{t&)WEAXe&;6oEwlNh=^T-yO&*MZ~N#3q{#k&nRG}J=V0&!{E z9bgYY*dF@X1Ykhkc2lLUO_VmB{eAXzz%R4ato{SkVobDHV2 z*E?jT+xW(HFj~lmi?yTqCmNxW_V*jcIKg*FBmWzT&eoA;l6F2m&h$oP(5{YW3L2{X z%#V+sOx{*MAn4yv1`4R4md@}Ozp*+mMwTd{>Y++OGomZxHUF0tAQfBqBp*)k7iwo> ztFEatY@Y3uUtD|-Y*vYQP3ptVr3&U}7v@Z9Q#;(($g>ljG=FB2cr%WaFeIvk5$KQW zb^;Q+bHKn3ft|W)Tz~oZ9nVstKGZ^Rau-14u9CRG(xK~dkBj+QI1O0Dd^W`>iXu`w{C#apky|V`eDEJgPIBbB+^dK5{PXXe$M=S>f>V@ z&;?z-Uo0gzo=hBJUu$*b^G+)cUUF=8>AsEl1Jbv)mbKPq>KGd280^Vbq&-!Vk{C3aPohak+@AjN9LEPhy{013h)C9SDrZMcB!Yk- z1dDy^=he2SW_I0P$^Y`oi7>XvVDQgP< z3#zr3-Falcb^sUy2#rs^IyI3Kd88&LZbB|p0Tb=EzmqJ^UL~-@uI{+sRp2u`o#(R~ z(75s%U)(bQ&+(GEK7-dMd@?sTP&4AO~)30~02vJXwR3 zzEQ9LDHeeD9`9FVBqotDCNUk%rKpU1fVL6IhA`>1wu0vHa4dB=)PTjOGA}_N`d`+2 zizaY-zeb++i_bSx+jo5(K~+r~gwv}*VWjgFpr83IgrogXbsPN5X|=HqHL5SJ zu6l!n+4PF_4BQ{(Nq9vmN@`)zaF2+=(QgW%M!t#Fq5iA)0>c(0vF}?F)_`fQZliRB z9gE8Um-X(>qsJHJCYB;A6#Mep)#uy#ugMzu(%&Vux)?FWsg!ITXsEvmr~}o;fCxCJ zv5$wEZ!|OG53Z)Awa>F#-?GL!f-2lZ7AaRxGUqtK;$6C(UT7-oJ0 z?(6N|dSwhSVKS2FIdnYazI|)9+ZB&=xGF5O?fF9?7 z_i6j_VI~W3FVA*ALB#dL4F5`so%_jhei;z43f(+%R}oyp7Xgzy^9GhyKUqX@;uPS( zTXJlf(dMUjXsM`>hd~To?3~q;(|m8c9?%F_6KgnF*IX9b0sWkeckAp@j+2v9)0)=; zu&uSFdg>DCbv)FcKYyewDJnv_ z!UiNOA}<#-OL1Wm?Hc90CM}{%OG^t+b(*8M<77*K*$W$==-ADmOPh$*GG4D8)1`Ka&emUS;>sw-&9wb>y7@-r2X8g790u(?r!GsmWt3j_#7 z2O_>QR?5$8G~Qcec4XJqtoU9(=FV4DU#rAk^=WYy7jVRKWipn_X=u+tDis7}F1H=| zEoyj(YnZVCR0yUYHgYA%g(Mfr#=S&qSaGutrWg}!W8IELf&8q z-l%ek@cnlurmL%?RdK7cW~zM0{OxTpGKEJvnLu-BXbZCilH(2RDIq$I)&C%&V_Qe_nGdwICqrq zrKAe(cLYI;{kEe;cc`&Ge{pQi5A1v2^7qY9`(k4VhCr=cSZ zjR6lc--9Jetx(-XkhGby)fs-5t>*J_9uok=F`Tyot3QnNvKz4ECMdlrrMcE{rF$%*@T=>EsEgnvIU~H0GD^0uUNMGzSvu{af(B2mM7xL| z^&@!h5dXg@;6<_%p8r_#iY4IzQ|J9Xnz5e>Y~}CG+RNEf4Bs-Bs9C+0} zy@7WhrKe?Tsp$sg(z%Ez@HQatZd@o02{9X25fVpIV7v0C&L;eDojciW|nBAp7rgn1RRy-i~7DOK=i$$n46G* zL%teT7Y$jUei+E9Ms(u;OvI+jIk&3-D`sV8l#oP5RIVNB`p}3hZ?(5qARUtLITaC2 z2?e)w)Sm8c#Y>vk^@V`hr%Xs5B5goUw@}Zv0siav&W%& z)DglUzAN;?WnP%mK<++i* zmrcY9-$x;osws3D)inkv#Qb0DX}ggBiOJdE03o zG6j{P+z-%tTOG%D!3$%1FxXy!H!r)o<4$$7oh_3O7*%moQeuP0+z+oGy3yEw3WM)~ zwkwu7MOg`Jc~zC7qGC)$Y{qAPmw8#2@BUu_M#OY#a!Pa}@QPx+RGs-xroI6t5^yDK zdFcxH0+`6_jh?CZR`DTvQOh^T6m4rko|o<%j$RefM^EznfP| zmCrEA)Eofx;LT3<@sUit+rWq2QWm<t0_C*ISAV&0XQI`JP!yZmgUM5)1-crd zYT6kS?mN1}D4q>nT}M5t;-Xe=)ziM>=obHh;|VjnOi`w>z=2$9MKz6FcZ=y>Bfu=5 z$n}OGj~tG^44KQ)T=O&B+(xGON3;|!G;KMsJD}Slab|Pog^zs+LfV?R`=AH)jF)|& zJfcFOtqk`0>-}i{kfAVfQNa~_{p96E>Bz?wOCjiX zm_O6D$X*~7vs!2Q;gI0_>C@1^56o9J&4ndMBDjff2kJc$TWCUK)M0Osv6vU4AOwVB z55cX6&j=4ftATp__Ji>XnS9|MU(E*oJ>~QYTYnAb=*A>@tl^RZ00>d-bt8X;QO##R z0feb9plzCpJe!oo=pa#~uv+>DT~E!QBh=#O2nYh6(3Yv=d(F^enff@#fS4LN-}DIHAABwTR@fe{D9a7fBj}BS*fbv=QU?cPy}Llp z1DQ-7WM^gT!rHLHeA?_L$;BD8>1)>VWSTz4|1{G%Uz`R1UP7#}oe*YPZFrxkP_I-U z4+0uzl&Dn;m8I|JmmGc0?-sU(k|nzr5cC{>weEh}^iPGI8+bpZT#>xs5Bv3I?cD+x zY(-(u4QO;NnefL%VHqk=(WHg7^$;*gD}`+F5S{Mjw6_9JGzZ*~$dZaOYMcgCsW7C?CY!mz0%=s=k39$;Ef#Y&mL@RR8th(9TlOgonRk5L_ z4DbBuB^7eDIIXoG(DSu1^fbr*yrDt?p7L7~4m@`iB%9Lo^j`^6#u;O{7BrESc@Gs0 z(J}c6DHhT=W?{PZ1A$pmX_$p7-%6N4R6da^WHdjEsXxhNVpp=%XxkH`-c(Sd%@y_e z5Y#-NW6@@txZLRp4zqyPM**Lxrq~7@>H{1Rn9sIn1D2FIL2YaE_PpYtk zZ^p)Z>Y%*0WbEN;_JXoWb{DX(i%Sg4z3!`%rRcN1G|Elm^=kKt>;sn;F z5$)-fOFYSlMQdvZzs=Hta%PrzTEH>=>^*ka+e=H)sAO#1RW`?R+I-Bu$n^EA*|b*V zE);Gv0F-mYgm2f|17w5x3*($1r%OV}Gpnm^K&ylh1Bq5TjR3ppw52gdQ}%X9MOF2` zrA5^u)5@JVBT*$Ev7z@?N)$)4#AvKtyH3$TBj=G;m0rvCG|P9aKwTx$=2-x0KtW6X ze6@EvQ!JlWv!&x$r=7;}G7>arSr?63*B1$u?gcIh9G#bsE)yM$7SfHY!8qF3<-M2~ zDGh4h_Cs)W$CVsYpxxilZelawIB7To%%4~bOWp9fn#$i0!!Y?~Hn%!i#PstP7Kkw5 zt>5j&9Z?KIo)hE-d7F=(UX)W$5apWZiE_w3c zjoE^#S?r(Q1MLl?OT8hx)K0=Y%5U$AWnQy_+{(S6=bE{9(tS;L;KsStA zdQ)Y)U0;0Bo`9Y~w#{wewG$GgO?Do!>?~BiKms%Lyeg6#VufUXl(-f;nlHi@3kFb` z7uU4AaclmzwvPWiyy+OQq*g+vN<^ID;pq$xM@v`7$e6e>#QwK~8vu(;lI=VE)(LAZ zcYAGKit9P)v^)&*lH1Px8nVND`mo@8I@+eu8jLN|{p(lyy8xUokOaNPtTI-&B0Z;Y z+3%Z$q@YZ(Pezf=@WDSX`YNGBL^Y+J1T53F=&4LHYEWH+o@2_ zYRA}UYtzcjwR|VH<qnvA;wW&wz&q&HWfw z={|O|HKC3(ct)`tu)KK5AbFMKeL?^C;DK;>XOgyEPD(P;0Q4|C(6>86Zgrvg_ui4H z@uzic`6s@Gn3=|z-n;_Y)v3W#(xlIv+E*^Q@D^P@8}2f$z4#su^8zo=meM#fwFa&% zQYEg2R2G4O@m*Sf+c}M=ukO|q_^0c4}wG|YL^Tx_)A5KWU|Lqpq5u+C}w@W zJL)`NfdpQJGueHo@OqnY@@fN|7xGObsR_gYGC?2y#T804b?EmYpeJ-rMFyB$f3oUJ{F$?o=kNQjjxFnnsb z@j3NYdOib-j846d)Rt;!uSwBYO7Tk!*B=oIxmkH6aeOFh`hSYL>bNMIXpM!EDjgC^ zBi)Tk$I_jW%aQ^TORIo%OLwyfOYG7i-Q6Hv(%lPt7ya(N@1MN0KW5H*YR=4gp5`;w z=JpZqD^&MxIe+tdxYtX1XY^Ld%HRA}Lz?lb$+YLf-g8~ZOa4XV+}rmi&AS%Jk6nXl zC5;d(&#N9Jk z)Ut=-^oBQ;qw?e}pE1P`sgGXI|51yFvY;9%ckA4zigBNc{~BuETw9*End`Nyrek)K zXOOd3vZ2k1O=7B6PN*;i(TVT;{Bgqm+j{^bW$qv%#E_vngN^>?>o?PEVhZih+nx+u zrLXJ^;Hd~iDweCU%Q#oh+em1-?__TT{ODTJL>NqNB14-qD*QXv9H-=Wr@P94#uvtc z4+KK^m3Oj=^^3I9?0TamEIG`h>?Im3nucb`?d<{n=7Tn>yT**Gq^Wt-{Zp%2155K( zs(5=ueQk!80GG8j(ooc8n_Ud$ADeHhCN!%@hX$KOQ8L){IWO?NHb(637!7$PP;V}Z zdR}-TR;%Z|FM!>#;%ZuiZN6-N`=G&XBMP#G0kJGVMBdSTx{!dp0x1hVm zO+mVc5Fmc!z-MZ|E8)4x1blPhB5R;v00=V%lxqf=$r{(Rp-wg?s!IfX>AnjRM?G#oy8uz9pQ4+P!Js+iV` z@FZKD_Asb1mhbQ`pR!|suH8JY)RYu#U}CRW_Qa2uMwY=KB2?u<(8&r}6x-FgvvcL{ z-SFBwZsDP^&bQp6Pt1$5p+dnY;~soMsnA_v59E<;Q6{yki0jI)FX{}+!`ri;vs^ri zLtIcNFfZ$)EqIK{MwWhE9#OdV{5jl_Q9dvh_4(XTC+2O~p14i8(OFTIx;FN$<@k=~ z`71aAXZd=y;9SJ*cseT^K?*;b4K7y#Vxg%p`tt_e)~vdzxzV9icZ`eX)>+PgbMVPi z+*dX0@y{1eBdTvZjPFo*(*S4LP;L1Zo=e&V)hxKAo%Rz^i7?t_@rNB$x9Jpw!^ZV5TYIvqIhT}=1(RP+Xz zf9TL)kqq!?a64Eq_B`+{)%vDhQCFpQ@`q;Az#xqDSzkG5vjtW0xu{>ffBA{$pi;>e-S}pFx*#m`Y z`K>kC4p`$0tK&=G=eOTB3k?@X%nc7jbgsfxYxyWC;DF=9rOQ)2i{>QS)Kj@}G_*G> z%04&Me*23`ZSrO9sxHz*F_AxcOu*)vIwfOEzp@_ro{k_5DI1nzi0>}L`BAtwkp?Fa z8+v_FbS{a>yW`Mq6zUc-@21K9LNue8FEieSVO)|Tmu?(*A#&34{B~&WpkZ&ZX?fFk zr8C3*NTKwF4M3b*5;Tw#DSEwrv){P>++Oh0A)2vl{(2VF8;(E@IC%7tFC3l>)&F@D znn&^r)nM4OhpP4rY7ZOsw7(-N;MltjG~a+)^x2v3l{zG2b;oU1!=oTQJv+o>zz+Fn_PyEj zR_6qE&i&mq;ic77Fv{WJvVjr}LoL=B{%ebarNK$TlKF#PRsDXSFLbO~7YGrxw*j zy}0m{(tQbql$z^Wdpg>Ox|tA&Y705d#$u~miy>AV&*e>N*7n)|btmy1SM;bIKT_m6 z+=+Mx)pTdR&G2EOija zIh$eu)letc3tg0+r(d2nI~_D$s62)?ZA@H!>Y=mC*-W29Ual_EDjS_7@8m<<-qa(H zT`%|Dng9bTMxgZroBZ#S7-=lp`vP8<)B(p5(1wkWx224}a-WY*GTbKjE8S2WHK+ol z*1mF@2nu4d1!XI@m(`RgQCavEgY$#&wVHepvHb6T2^ybi_?Iabryb4m)@`FTub`?uuuz z+_|eXtzJuukMhS&Q!^BSoLls0Xt-%Q>ZZxZ+DO)JtV5h7o%D*oaMcIg35Yl^RC=n< zYr&~2J&-pauZcpI*Qgh(Sm$_k7LpR0nN!z9DKtU`22RJ#FFB2E?E?T+9rRm5p=U4C z^vEeF(u6oECmgb=$c=bgf-9By^#XbNkN4(I(%RP3U_31OHYLYMxexGg-dqxLO@$)c z7|G8KFQ>O?awcL|#|kPzN)+!qgox^hgejDS23&VZ;$kbq`suh!7pQBZBQ0qz-)kv3HQAKGa+23R^h9@e)M@M5jU7f`?u>%v< zOf^!#om74_k6_peud}^AmwQ;Mp7qz+AKkl=sWmWOn z5_ji7U^$Y^RUZhTpk>_R7i(Sj*nBe#_wpkqr>IZ}@F>_)HFoe_$58Y3x_0xQiaxhJ zA)Bt!o{N#OoRMu*h>l1T0B^TfqT0WQC5*kdw+{YMWPent(egX%s->9&pOYq;I?Ip9 zuKFE!L(25(f%#8I0u5G_oARuOY)wIwORw{x8&>;~bQ)5Ol~CF%SrqxMpHj5e)eOuP zc1OZAkwc*Oeg*UpXv!CfH%9aoR3e8stVgwf`JYsEPnBizxG9lw}J{G#~`sp zvT`or$NaZ(kTRD42%kSKu~2Osv7>7S(ydpym#!gM+Ff?LC8@vcx zKf?w6unf^1yl@Cs-&0`ud97L8?k-7KNKh~ht#V@*hk=c@BjIiS=hh;c57p z+fN`>{uRWsn8Zp$w|t|Iko8{eA6Su7NAwGHI4Pta9o-nI<})u|Wi;nt(-8qnxwkAe>Twu;y0g#@mT~=eN!>c9nnh;%bvtO{+ zQjKz24Po$-1~MVTJf}I_^V9VB*+h-J82zSV?4J9t)E~DE)S5+P5JZ#3ZKdXLoA|FOvXvAZ_@ z`O5m_ZJ>gJ!h|ZGYGHyJY~*+e(yiC@U_g&PS&}B1JSJ@9VN0AvvXg&|xCy3OxhCrp ziuM+b`pH1iyCU$O)lKh^`}5(5z$BMm=LJno2*GqiQ)0T@h6hkI$p$su`zJK~+VU!3 zD#$)7(<`g1+L5T(B1>pkQJ@P;6zAUvKOB}t#0{J*mCHcSh)Yh?Ofm~R6qk)+`%J*0 zBaDLpE=v>#9*Ic0PQ7~YXt|hEQ^A-5#QdbFhzRxj!r^>ux^G;ubp;e93j!G`3vwgL zjco3+Z`A4YZvsm=BG$?vh)RPGmhz**gBF4AqPtldO(Oh3C_22%H zPKz(a^`CIA(SWsM)#7yI@#8DyP8r@DB*ULQt3`;UEY2284cYeiLe_IeWKVS|wd*^J zdI7iAeSjn38;HK(T=I3v=Ga=sflP>usEm>)5c|Apa5}LIrYU42$Tt9B6qu8`9Uz+#O~kz`RZD z!-EO!b%u;Pt4W`nz5s{Zt4W6fdWeJl>OjWXO8IQUxxi23*=qnV7(2Uqo9h|n6JN@Ly@5oj$u$*d!)Okan$kO+>g^M3voR zgiH4KonQKB0%7Je37YyN(`)M+m;P=72&nY|J=#Ph$V)X1J`BB-H8#)@#snR zIB}}Jcv12}LTatO|E$grx6Pn}ptU&^+d(Ms?c29{6qfke!?ju=TrRoeb&8(@$y7^x zoED_AI4_lgq`u5VeC3fAv%WyFt!IKT*XfSqKFn>c)0#}VppSrG2B~erbH<;m{!#3& zqvs6Q$ivlEF0qGgdwV=IV%+Kcz<& zA0kAKV_DNgZ%$e=vOzXMG`ga|kOXD)kGrQbM>%(CM+kLZr08f6ME`3%^Ox622C=Ay zLOt2vr@SW$<(l+*CUSChe^7Tw@1I}TC|U1ySP$bm@ak~AV#7hXZBofZ>#V9&2U7L@ z2?%3|wE@h<$^uOHCFSl$TWY>GLJJBP4o zkSlX`j6!=NY1v9{G;cFS?7$Z^l7hV{j0YP26OUBXv?tfqew?zDbflhDYW%R6ypwa^ zRT#2nYAayAA;_3OIq9dKw>Y`iTKSlB+Zn-Xnz!7X{9YHa5L%>ZbBu4fvD)X)D?_GA zQwT@RgfFfJN4G*<Q-_u^|~`hb4_W8WznG=aAGc9YdRBy*Fx%A~Adj$4+B~!@v(&%+0+hzUXI<6t568{j5>|4=L?lTjH zJgp=TrVsG=iDtRn;K#a5upADvA$^cc>g_&19T ztReyc9sWQLZ}x-Zj^BrBhSp&s7s#DSfmtdgMDR29D0z4cY*brp)_bq85_3fj?OhrH zk5OGxuQI-&4ogQuxBA-(<*%-v%V9Z3l%q;hKl{m&qZ&+tw?3VXGk3&ByOoakuplNT z>b-|AU7r1&%b!PsJV|F=M%1jydZ}pwJyw7gl@RMb;xv!n{Xbr7efvA;nMWvA4au=-Zgf z=yXuFasid6?b#4?TgLD5L>A#{l6_sPg4~U2%F^|6oov2zRykd?GLCoDEHIkWHgC@L zY6egW0y`6PGD~*8vbDFRQY}s=hsKJ3pKz>d@FvgefYHf*5IpuZL7yT_i;>ukh9Z*Aq?rfRX6?KP6~ll#O-B=%)(}wwM5dKG-P^uXUU% zastJK^Q=&{6Fa;|s2sjkB%&IvI$z5OXU}Jaf(^R|NPijk`&lj`Xf0R3ZV#>rti=wRS%KB)k92ie@uSLG}xY1-fL61BH`)<3qTQ z)4KW9`A)+Jr_LYjb@29#`clMikEcJ@ArAqkG?VgoJIwIK>wrOPkslGE#ztL5%p&l0 zqdS13)-1P6!?uVy`c{!S6TnHcG!48N)ErCa$nQ0!b}PG>4`DtcwLgw^ zxSiCvR()bQ>tL}ia|Ha}h}Gu~aNMvgN+AH2`6?i0Wp__(lc@HTr8f~%nnj&?5vrbf z>srAT1R^KBN8!K4%u?Tep?XfbFR($FE<(A1xJ+kPN}V}Aw;O*s&aV9;nrtbu{?OrH z9`z2#_~h9aM*dangnUqbj4Fmp_|um~51ORMKX?5bkr}}Bf)$(T{dI)%z7QWkzamU) z4X1$6Vn3as$aJ!kJ-s?9h`m zRAk%M7dvUyutWsz#eI5%?Hu#_7wNZ-EGV-3kmxuEvQsRsx2BJXY_Y$>F{=xzJ`Je^ zF@WmrvmpbHaC|iz$`zUNkCyQmNuYRD;ih{lpvkj)XD9ei?UWi$BSq)&Sh`s;^Y*PJ z>FQ`ne;jItOlrnUt$uwIF%1)-{rUW_|ML6lJ!3i|Zpv$w@>Nh5Cx(&KPVZ&1n=;dO z983@=k#9U#G0yBhtG2qpKls!lD<m-L)3wny9XCvm;JGyPm{)2j;K!>NsL?Rv`FlH^Rex*#V>GsB!VRllaJ<@b}K!}A@ z6GJ70NOYcf6?^uWociwq$_fvqBAhEaCOAw^c*VcY;wngMK8(wDu^XBcn(c||#x5yz ztNiP#hlyDhL(3nIzhJ>k%OHJKYODX6Q7S;sTI^f4`3CHick+Jf$1ipnn#PC{dumpc znttuaZPGGtbc3}_Z{#*yD`A!5dFId zEM>o?=<5cFKXjOc0~_(RF=3EIo`@uP)uN7T6{qswF`iI;aW43zOgmB+>u z=MuI+T0e?cd-EOY<9mxE`;WcUJD59+ayR=*M8;9;d{74?Q5@pb! z;O+IS4K&{W9JE?pO?V)=WuIN*R{nWY-mRE9qGBA9-T{=-q?4u-!x-{; z$mz&TeNeAPAf)zW)12Ua$zO5(QuKQ_PTs_*0nv-oO>yoilx8J5-KRCv$&&$k2I! z*NQvM!aSm*-Ie&CG!NND@=svZk+*F5K{ApMhz20B6HZSXI>=6@I}J;!H0h(0+5ySV zaR$M(QzCJ654w|c9G1m52G-bd&hf`Mz{GMM%Aq!(! zb?2N8(kNBW>%&=Py=pAjE*+62%4a_K;>xoi(KWDv=r}c!6>hbDeMNGLJ;+1w((a!d zp)iKiR6XXs@o`lxye#J*tFDPrB|Lglc!0OnTlgjJ^V@tdUP~(qe+gB%dEN3*mF3A9 z2Qc=U_Ztx9L1{a*^r@eSzpoI#uULmg-!L_;!ssj0-xURY z$$7}DX`-O=O>~5rwkqXRj*B_VaCSyL2G1d<7WiqOT9NSYccFSFDZx0sI8mNXM1qRl zy75JIq1h;yl@16bHSV^z&H#efw*!eml^Wxx;HKDb@-kXJQa@=s&_0FkNs%SYM^2~F zQNoSY{54-8%@cv4RGfT1snK^*WZP4^Y--9i+#ilJOcZLX7Q4BvYEAP>ue6g%3g(+h z!cIcqr#8(J&Xr-Y{tSMT&FgYfVGJ=sY<;m16d?L<%1DiLeV6n>(qXYSxHwjClD4Pn zkcMXW7XPniAvRv0Gfa&0oGQGbI3c#137hRHF>8=mnj=Iz?hi+Bql3u#k#(2RP{Fzg z#EFS3!ws~-s`gfG&^Ct#0$lJpf-T9^;a#q-3`Kf8O)5>Lbv%Ze-+1XEVwZ(fq17D7 z!K)FUaiQfQKZ`)%G<2)i(BtUr#VW_Ig;9}(xi{W0Wu*_*^B@7| z|4aUuJrA8{9qxm0bo)MdU+^?eSd$NFKHo1aw&rPTHZ96tJ#c3_6exb9DZ2z$m!jV2B39 zkicw*F5-V*^`o3pp+s8a;~P+&36M2b(ZLC6qG_V~Cd$|appJ=xlStXYX{bEp@BpUoTV|J=O1#vGK_I1 za1C%b{9eR%khECaLnZSdvWciSsnd>#&u+tZ+9_DH6^(&{Q`U-VSY#S}+|(*2^v$+l zEdKV8yz}#jBP8$fbJPV>G4evbmq#Sf$FQc$T!~eSud%P&F zxH3lzhIiERFXr8qOQuJ^_BpMSS-!%tY)c~w{^;kdNl~nYO?0zBZFY_^?Z99W5^Tuoy&=!tK)gpkC#WB9+3xJ}|LPn~aP}Er z^{`U(9BJmZ+;bGoaA}UO#Stdjl;2YjIu94$nC3Ydw92cB?iwXM6}rj4OaeyvR}LkC zo#wsQoZSjBOVLsw>^Eb-?g&P!;1>#k3@H{qir&0>_92y+YC@}C&u4)@dw@Fn>dk|R zfV&CDfnI6yO78{OiP?d6+Ee3O+F305e{7e*_;q{hUBsn?ZRMSQ{U@&6j7U8_T$-OgO<7B`sKFel)(OnOmSX#0+} z?Nxj*kJV|<*dH_Q%v8|s&g+_}Md1Rw)<7@q^O&-x*5A^6a;C!-jd|dPYN_i){@7-o ztc-k5+nE?DAeNF!UP-TTh8rqM#Y99|zzkFC^``m4sGVA)X71~sf5;z%3}4eMJ^W%- z{ly$+xC@rSNFHT;lR=(nhI;*RIat*nkBCVzWccS#Q$nlIU~?X2R`7Aj$|<2%O1mGZ zjUS&?uro%qQZA;-RU1XB%naz)>`VAErI~z43A)a=U(sFriRm6C9p6P>t((}(V$$YM zm+Jl|O&|qkHe4aPsPDANjZEav!F4$w{TGk8d?W!fqbHD8v#K_Pv^m^T15zEo^PKj1 z9zIaSL(OSe*_0qlmXz$-2~kF)Ou}*2XQQ-2EF57-oZa=Ju#QW zl;j+#v{Z;63Hfb&vu3-ZPpE5yF)`&BgM+#cfqr~n3-D8uU84Q#TxNTVvpVE*t7#3Z z+pwP|t|GW0+@{V$Z(!ha7-(C0Io;1~VwOC-PvhHhTAmQ}SZT;j|zElZMY&{kk{ zWv-sL4DM0Ix(y)y5SlxuRqZn5oA9$<(QN-jb`tzm9t_X8j_bj6)ySjhrkk(6VnJZ%{FE>!PV5&=)8o~dRG zn(-G#`o>u7?+R@!a{`7d^k%9AgHGl0`bJ!2;q6YNAP!oM4uyH>tX8->dE}p&6=SCl zz3z14x!CPI&-0J0R~hCTK)10R8L956w?dl?nFeRj*{*gfb+*^(y6XO>azexH3Heet z*g%iT2#vIUYx;?W3Bjp!CSFST;PPkGhM`y6>JOrMP6RAOvm*hw5F$ z8dp+FL>YYcWX7=PXkJUfKq3Cv)@*x(F)4&4`7d<`Yo7TwoU?#;s*RM2Q&|Thk0tRwN$+xlgjUbF3HctSIu^OvxB9SZDz?~gVcdM*`n7g1Rc>0vhw?%tZ>sVi1``CCE^I!KmEL@Ow}?u*0{!VCs9LPm zD#Xwn@0AaYp$SiAPi74Z5S;LiAXZFV(O5A9P|b_gXx|0XmD`FrcxhHVuRB-xJ~3%* zbV{HmxXLc^Y5v@r;pZqKX$s$_+n8@PV-rB(TZ<53;N`plmGmzDt4F&;0sGXAO&_@O|OuM^Wo%*-$a*_z^ID8~PGU zi}}5KFA5qW4xuP|-2P0`@!Jej{q#GVPJECkwSO3?2yrS7j3rVQW_~7_7rDYon}hY;yP)V0_B4w&X=(`| zi&|8s@MDH&bUV02hx9rxR2PY-mAGPDn46o3R>O&UhOsdQjotnrR^Y<#5W1z^&t>jWf;@?bp2vvz}sd|jlmgKbi233ucmHmXNvPHwCUIqBnJB(dWaDbAm z=Xf)A|Ic`49bcY6EiLA>r_ilP5b7g*TxK$$x(byjc^ZY+hdFGn70ra6x#-@M*p#H0 zYl1pLH9~hyLE|pSgO<;t%A?~VzaEX*;WvJcaz&@~oor=G31r*wcQaVqqdGZs6dP01 z9LN<}sL5)L9}u83pV-Y~HIXJ+x(^M%W~9Q}L|h<4umz$^vjwfz(S4p&8dT<-yC?Ls zchu#$NiUr-hg8_6&|NXCA*byO4&QoEh3SXJ_Wi2mYDed;XKk$(<*YcGb7Q{Yy(YBS z5wOUhS17_n3s}zoRP&4f-6wQ};Vj~Xb7>~^6Ewe$&I;R5jqi9w#CE39-b8E)C8U3Z t_1WU?Z^I#x(vOzzQ1*g;AN?Mo0R)3y+9^JzM*Rg%PD)9#^ux!m{{zN84s-wj literal 78104 zcmX_nb9kIl)BeV2jK+$$Eu z^V~CM=AQcrlb4l1MJ7ZB005{`lA?+L0BkJ)09B0$4cS32F`EMbFaT1bLdtHLXITh( zb91Cax(*EcLr<2Ta6!HOnQ zq|x_@M8Ri9R>qnNM&#Z&>_`WE_A)5eCQKVNt8NWrFb07iKrnPQ$H2np&iA?;9pXsH zg2gSquObw?!rpauMF2oIA_8JllxTL4{Yer3=YM%M`KHmxS51%|n%|0HI#_Wn0<_wcc4|EpbryF>6^A zt}*od)ye`BFuaTY|5UD%B$Cpd@33XQ=7^L_+s{?0e*27+GZ0`@{+^DSFdsBhj;lr7 zmq&asZb_8^$k5vCK%^>&LIL~ylLOHqk_-KhalY|Jl#Y z!`>Rt7^(;p3 ze^VK8th|+@-{QeU?1U#3Uim$`%<4K=|mte$K_j%1uvn!1WD=mJ~k~ z=$nn7I)$d3YYH+7wH+K%@V@RIG!=CXu z7-@P~Bbfd(I?%FhpZ!hK>>`e?l*043i2aZ!;BE%?y;V*%gt~Li&n7DXqQ;d1vaiJUN?)5oqn8q?N4`eL{A%Ukt zM-f*3uWbMf8as8`K-5!oGJ-1fN9-W57Z*n7f4vV7#OlisZy}X@qNTh-IfdefFUE_R z2NA!jh35wvN3Ka0J*c5)T+|oJ#IV0umKecwuI|=pODdxs)-mPW2U*`%3-S;3M^9S3 zo48x0?0N24u6C{N+N#HJOO`z*xEwRDyX+<7)};J}-t>?xPvN~!w^LX50Ix<*?*8rI ziELTRbhO!RPQ}}jhug-$m#S+pV7SjQY1HT{AqdQiQHuW21*NPdvo@RHo%B= zEcaDX5O-7*>VJ?0Aj21opMi&x6z_-0Is8fH_}^Us8E@W}NAnDEkB_#VBu?UiYKig)XKa=d5-@p+jigJo|`@>Qq|&0X;wS95=(h zDq^piI-3IGH}!HYL!_uhYc`Fzd@MM!KIK}(s`4|ISV)A>!J3G4hBIq99%v?mhU+u01-7XRe?ABK0qVZgA40miKJm5DoMquv$S}q73=}M|WHd^`4v+ki(wyD?M4|%>->%t{=_Mvh$*1-4})a8nH*l?EA%*_8~a-j3T(VY zW8W2)ymMSUnT~9W_dKa|v5vJ7XlHzGB6gz&-VWX>9z6#tb{XCN)lOPCbcm^b%tE0E zUQT;!ND~r9QZ8?%MeJqX)Sm%6$`O0$nrpT!j%~a4q;YXrk$~~vl7yg2En`0{dA26J=q!gH_<|((J_TRGr z0CUiDg+hS($Y;MBrQ{df|JzoP_bid|lRLtzQY}Gcwuulwxq{Np>i9f_5};-m)^ zV}5;a<3g}VbZ_&CnK7>w^wjiji&T|0$PQ|R!}oMYn#L~*4zgPaI%eJQyOC%oa!&a4 zVj?|=B(;AOQ+HHStG>t9hZh^A?f1avpCH*?ySJaX&lSS2SuOoMsuoufM|DSW&)FCM zNOFMvGUnSXhreR>gM>2>=j8kx!7G=Am z75VkqTjfe30TUl-&z7kuy9^^0b&xLc90UQ`ps=sSjJWUNpMR23N9@Sx=cPS4IE#sE ziZ?Y>^ujl|Uh$5d=Z_T!4aFTG1U#;JsX0hzq9Nsk9Zr)(#Xq|6k-8j2a5oii8Rw;X4Tzm6&wG$0R5) zs#{Ed9Fno|elU>08R0h)buJ1o0~{ciAJZy~5?N3e`*CO6wj51OA*6!R79MiOH^J+s znqKW=!epbwn|)OGKW<)dkW0|Fwpa7`@H%NKbgf4FoDA$X?BG3G_R5-%c8BHI2ck|NA?1+vI}GCz$mN|b*M#8B9S zR11H^eAQ9_ceaR58G^flX^8Y!^+s$ybxx4e%He25p1^D~65P@DV_h0g2|xk1NOQR5 z;zrrRYNND11|KzGd!0X-5yOh)(dCO$_n_W(GeDku6z_%0=hsrtjGqRGY>-z3CkESN zrm4!j(iBQw6+f%tz{4zzrhj|I9f^yPRz)LLuSi&fRdJ6rlE9dV(e- z5~t;|po9Oi4+Z#xl?zHhT7tYG=V<8fDH_I**krqVEyt9NSIF5`j`Lt@C4uo34-Pr= zzwueNQ`39Rz-l9WX#lLq0uW7w?Sh{>Io;@Gx1S+F3%gxNX-r+2rBD<0SrCml8k(^dNt38 znXd&S#BvOZMVP?9bDwi>VF7}SC!+R*CVk%l0A;b>qCBS3Pa~y6=lD;kglj`kf&qU- zboQ$@{$Uu>OiT+Rtw+*9A1WIggoaF}iD z1EH~V*&cBWC3czDUMT5P=r2z2YF?*v44ujyo*H>WOpOG3>o>iyC_}y~&uZa*Hhp&vU!_g=Nr?@ErWCTSxwB!H) zqM9VDtRyNI5RzkeMbn}qYf-S20En(QnH%N#fZT2&a*8*ayy4kH|91-r znpuVY-Bn=2XaRvd$P+(RqlUrQ=XF~PT`(qUJ{*HoV&T${)Kv*F2Tp~ciaeG1uvy=|2b>`A^M#pG+`&` z1NWbUe6mLa>OQ#d>y z1R^L~&tnK1JPGi1`@dsDvVP@N%CIu-XrBP&#~cwfr=j`?u*wI;F7W@j0Ld#UiVS5A z_9*3~)c9XYr!Cy-PJzdMRe9y|^p>kS9T1wMl`ZmQxyIu@>F-(XnCFPUPlzS;+I~zBW6!bXe<%1!f2T>Q)Cn5+wZlO^t z`Ey08p_F1qh`BqRPFJYajEyZ^Un7}L9vJ0tcI)pxzrK1Z*IaE(o_NDSuyaBSB`u|r zR00JcBZWJp`)-6rUL={17Kr4TIJ!zC+H^Sjsf~>o3mfZjEStmg5i#abQC)rcWi+Kq z>y8iEcFq}^PX`ayHyC`iP3hruxm>B<@V9X9>j;A}F*P-Hk#SKG^}yj|&Z^JVmd|Q4 z8NkfW*AWqLN7OVaM@R9s9F1}p{FywFBbI6cxid>rnLv0+ZG&xn1;svtzXvqj#4vCN zGkyoNk2CjV))`PSS>tElG^?{(oXf|*c@USo; zBjRyxZtuQS@g+6Xdg8B)`h-FhO0l}a1mHZ{3IniDV61R`4i3~dAjl%07pMHrfb$R= z*g|8pqNY8$@W2n|<^A~-T^WyxntI)5QzQUE_oQx- zYBU6mG@Jj;1r`v6f?E&@sR{6IW@jdm%I>~T9(K{lf4jPZH@~1iUrTBXM*X6gNfdh6 z@CChJ44O@g4MA<3+@+P38D7_c_7GLkzoF11JZJA_Kf|`YNS0U_JQ{K9CcS%0fAfJW zFX-TK(^{8;PV6StV#DY8g{IHk=_iT{&a6i9!^v_&j*@~xo$2I7P0ySA(7m-Jb#>c; zd%-l>1{N=0wig2gr1#0Tt?%L^@4d66-pq4hBAix7S1$(wp@!}8jQ_H~aJvDZ1a|{& z1adHWt&>&Y4>15I_yQ&Qsui;fDv^rIJEcs)4K!uNV7YKh?jF#Z+Q!?ggGkhDrHJG- z&LyhD1o`fn)EVjuetKRVc4$B#Yd#74D>|Og9)G2fN0lA9Os@AoHKoq(E442^&dNrQ zp_tj$41H5x7EpU2W>hke==a`=4r&;{k}nRU`w2Awq#%uSkC00>3iyQNq9mW@9o;r&zT~TXLDfU%}3x8j{ z`w0xz$Vlqo*}hm<>hM{#CXynU@7_$(L>u2BulOGiEIfk;;7L^3|4DEybp^c%@fbRW zAgkvA5j5@edSX#=SI*d7qSud6^H*vxpS37&E}KWPzAdjJRll}GU*RgE2 z%jZ)Y&xiFEpBFDiWZ$(lt$0LzZ|`hAy~Bi9Fu45oa5CYah?Kqku*N__+Oqq_nMc>8 z)6H>?&l|xGJB%Qof7u(&Pu1lEyVibOzcma<BD4ABRMSD(kt zCu+uu2^f*@Oe?_7$j5N`YG}Exqs|s@5Rj1U+oO~!%1W8v?%4pmd@ROgcGY}2k0(ftjOArtjJx`cV8<6dcP7tmS6Yxmt$X4E{2|iPq zM;ny>?gH;qJ89Okyxt4*(gosN=YPq+poL zp#C}?jvnU;0irAg0V6$nmJurjNLbNp9NS>zsFBw7-zTe(i#=?JIb~pl)@SE`ya8YMeTin=13gUJutsX(vVx<-7h0U=9{rS~ z+^gvMDW# z_vDbvN3>sCHWxP>PGxQ{yS_126_F(bIzBJz`gFe!R_yQZQxvZqhRbOKZ*VK2%*`IB zI$SqjF1%`6TbDd8L#(*1mWH)ET)cX2H2c^^k`;t&IV6|pc;7BMY(LaLTy?zPmor&j zclbR1S;VCyyDIn?qJvbbP*wA1>O%|jnO_uLKgR_^*~y1!GL%92Vb<7SPa z=+EMTzT?@F ziA?_6mz*B!rPhs2#l#x5XsrpiD#y_HbQl^f?YPv`fB$ADEt&}ZY<)Y6^HO~t)>&HK zTjtNhC53DRMCE4~^f4hn*!j_%f=uwe!^MCe-J+LtvEhWRB8?q zGI7nEQYUd-(^9o&(fp;46QE{VK>QiO(Dhi1AfKTi+i@)p6q9D9vX`+!blL#@gPHX0 zY1O!ojm4kK=5?H^odD`2l*l%1Enlo$O;#cQM&P58$KjuykdU{Ix3|v5Qv(+l0YOzd ztMv*UOM~{R3!nQxX&IRWOBU*)Wz@yZe@}-wWVEde1(@;dpG4A(8_UycX+*vf$Vf{& z9g7XD1dN_8yrH(sLns3LG-O5nWpd7Km`;z%#oIg-w~zoe(v(_?cQ(C z`=s5jufLK*dX-9Kw&yC5X{210N{K8BWU*5M0`|*qEPkF_FmcE| zCVlIGv$}9Gu&o*-+>E&tq;sIK&Q=!sA1wRbyYJOCzDzc(4B|S+6!HG!0j0 zt>$Q7jrqYK*r|ig_jG+e!USA5*$fXNm+Nm*Hr#f{NxfeE+(?E&(!vJ9(0Spw0+a!~ z|GB>6!86PZ72Z9_#=ymt`UkoHfA|I)8QRLPv98>s)uCxqC0O0n!FV-oveSv{`ld-a zXfCva6Ia~w8)zLAEvVA5w4nwhT;l!h2xnx0D8>ydOF2sn1*!;i9Q&zHrDXdPY~b{! z({a_oaE8xB21=3jHCXlE@K@S!zq7^hsm>hVgRk5b7?J3s>ABf*l%s;NK1>@{ExrsD zq)XgpLo0yI11;k$J%VCx;>cu&5kDg>(KsQ(cZqpti-Nqpk`{x>i5dpP@qF9pP03CZ( zbvg715HRN&=o7u491vHC14+M!5N}-lV&!aI|D?g}e9VRUU>45EVs(PoO~K=qJG!k; z!hZM%gNY~8^jAR;o;!KsAI=_>@sk1I+4C$)^h(`Co~i_qde3y)k6@qWboy0Fdt8)i zsqJUU(cpLdi5TaFz4th02y60RUBGNScPb%5iDUxKst` zgwafc^Ol&NoSdDVof?ZnL_|bFQZX%$yN_G64{7nKO(!zBeuIdD2-4E3AWdA8S@uey z6n6RobL4JcP~Y3j!{urV0+iq{tgW=l8Eg|kX^q);WatPX#*i4=pa?TqLRr)blKqV% z!@JDNZ-1X;D+NMHm=|;nVnhBWrWmYbWusSEY=Or0Qcza^z;%q%?p7+~+`!j)S;+(d zQN@VN#D|Xlv62WWeTSFP3M608wHCfr5NhcN8(ooFHh*WGF=MqtjI*E9GHA0Za=Xpa7l%h`(5qVwrSEVL z-lQlgPtMP}vq$B@OEOB;K+AhQZPBFUb{wqy?RfZ7xf_@4IJPtzZAvNPJN=mBjcratcI5Wrd=GyU7Hmazi+s@xGj!m7L z=N;-m@P@fk@%dVtcAho;ko8WE`5cMnn>ZnsdQCTiR6Xn{Vi$IeuzQ|p#Z z{wAas_+lEL*Xz|WbyrG&yfj~Y(v7Z=4*vUpT1~p-CU!sNVBT8%IfutoWHU@~QHAI* zS8Gldddl}#O9ys-qd_>B<#c5k(dZ_@hHF9gqr1n77wZ@1Ch_5OH$==>Wl z9v;%U2U~1b%=)p7cE>I{no&|xlGnpqOFUtp{0s%#d9$W15yY5A`m0V<91^@=sqQd;C{FZ9JNNM3b!Mcpo6)Fbljjwwu=I7)r#5 z!JZBX3ERU_rWnrw%}a98AmdJ(FuU%P_~}o0Os%Sn_9nC$wxF~<_Q!O~>W+&QqL~?u z8y9Ii;@ZMro?X)l3IPb(+-_5P#Gxh zn*7KbVVeK?1$&@nqb%Q<7AeakEr=Fi%E(c4?^)H14yT$dZ`IH===Y7-USlj6< zeM&^lh{@iTlRt$EdMWmcplcF`8&3t0>M; z2b2g$F92}q~VLHiHW?$m91N^eESuFNH65>yR|L$UzP(Ur zKojECALnmOml^=T9V*3$*&Ff=h2J94^JfbF?xFQN^F~7U#uEK?uCt8qgz(4agg$m@ zB-s8w%jk90WDe!@>Wfj^^`o4utkr){s0aqZdIHGmm}u7u?5O=E8TLhP`jr0MstjPl zIgnFfF#Am8f?vt7pm~%APTo8ffz!1@Fl)k4j`NxBan)CmP9X`NeLIaj7RJOT2=!!9A^`!hXElvVJz!hf(~0R!Srxg4)d~^GODEr~6Ls0E5tGWgT6=l0 zctg1wujX!K&-Qz-S5TK;;jXywwsOsdczv3~l{90k6zeI{DyG2Cd?oHvggo9!k@7>; z;J<$d-Hz5$R}4EkrcY5jC&qJ_lsQee{`}a`sAuKoCM<)s*<;ae*Y-I_bsg`d6c8Yt zhjTQU%VaQYv1%9&3<$f1BI{DjS(JpR7Jm^nrV--=3DP4K0p|pJ&1Z+ zLlXb+qr2eLEe<{Z`pg4LBQb8Ki|8M;`$p96oS|hluD=cMtQTN~iPLQyV~|h57P(OU z-7NNt0(|E)#gFEo=?n{TGNV@K?`S8fZNDn=C>r4z>rd-W2f9Se22PE2NeUU3h*-bx z4_g?PSD6eRM9z`#NvE8&Kj~L(pU-cz5jNFn6!=XaKbld|#5Zxd@yx5ZM63^r^Q^Zk z=?F&PEgYEzC=kg=oZxGmhMVjx7%VOtH79|RR4e)i2l0Q0#6}Ri^EXlcPCa}t>#WRV zM$cQe`Z_1A?XnE?)crM~SZq^B@txNEy5h%0hmVe|nPcM?a35Qam(A`EbtbFwIAh9WUJfOK%fV)oUZp1wKJSU&+Coj>{l77g$;(Hw&8N}(bCvYc;Ukj zW&X5bMy%xI*FKZ{n+>KlYiqZ3)m@g0Hnv?)_mnZbPWM)5@c}Gqhb}`j|Uhnf0dNo09+(PIw1r}@c@68g`prQE&`gWBI0Ktcbi89 z9`c>JYD~-Ht8CVdylLBUXTu>{`XskqA?h_)=Ar!&KvKXaZ-2f+W4=>EcU!M|GVu*8 zxBU1eMWV4hA+J5*dpOIk4j!5;_on&Y!QYxzz2{1*QSog=YD+(puKM&Q$C%Gc>5cGW zRF;a|g*_gs*u7nkKrN$B& z#`Zs0;hIfm9q-i@!m_NnS4G8{nc(6}kT6vbzC357#G_N&YVDRTC;Aq?gxbzNb#vDVrTpGX3B>IeR zDIw3=sT&%2@=azu7QK_^fUe<^3zS&r>U0*9RC312FH!fNhATkq`1O-BvMLuGL&2hA z-SuGf_qcPEFQ+@_5_d6D+}iOmFv+Ta#mUsI8Ty|M2t`j()Atfny`4lY0$0CP9fzrzvnT^_cOGR@~_>p>|BySxhI+ zeglVDZ8u)4S~dt=pUg(8&^n2Uh&s;JA6Qs4^2i)FUlGn-hF-KTGIMPfi$sFAqbSU0 zY8^u0VgpDHNnM@t1>%B|u$tXRRLpmKwnuhi5*;-HfTR5jVo zn>5qUv1e!lTU9Sc1EvwWKF-S?mlzL7aYn3PnQZn#N86n)!iTr03&>kRLaRtB<&nk|TDhzWoAs)j|O!svRoxv;;o14ak~emwh`~6L%3rK0DAL2pLZ_+|d?_;inC&D{wqhh3HZ@Y85Y8G z)ctLXs^Ac=N9i3LuP&ZZT=)}TU|OwN@;{J{E zqRz*vtz`f=3T3xgb!Fur()jz=*HJnBis;??NO8$~Ykgotq^g?7e$hW|G}+|oTW~aY ze0``@{}M{NG@#UqFj=(ziRE)g@M zbpTUFf%B$B0u7ZYdUv4oH5T_Q#N<>DFK33wbLH@xnylr;6VA(5@|1^$Vt$S#eHk8m z<|)kHn(;Ps8`7rYF{1e?H_bOpvh;?f0k zGngnRmxtE-i~B1W7SMMM#`7@)@Q|J*)1ShZzlXvX^jy>$$y?1*O06DFQamS#H9ZIJ zegCM!+YEM8D8(aDdYi9|*F4bquP~6mLRb(=D>=yYIhy1n?<%O&wkT$$-3(VTM=!^| z`#XLV`q~F`2|KzF(bCQOtmzIu359QQs1>_qocjnwpxTT803f}tSpfup6+|1cYKdkz zcW@7~iSwQ-Dp;t0$PncjI}zeoP7R6?Dn+s3Aq7AB_o_sgS=2%rOP4`^Lx-ij83#ic z2L*bUDEIVfb2f$P=R}Yp;SHX*lehCa{mr%Dwwkyd(eb+lgOjM3*}wV5kzS8ntH|i2 z;T4hz^6!x1Io`(K;s3aRf3Z44_;>BCK#}y`RYJ>>vb6Adb7Ge@(uq#awOS9!2x9oG zuY-DK(I{n~^_AMZ4c+I}=l){N7v{~FP(P3fax}A_YW0%LndV?zmGTM5FE6rG7o9m{{eV=5=@hxZ= zL|0f{AGyKt;phGKQh?#laeG{nox$PLY&0$k`k%=V85cx9OU4UDOPiQS?50jKV#W-< zb8yzW3XdvRCrL#6=r<9qTeiZv-(;9?u9u!wSId@bk)G`s6Z=%xuosPSb6YW)ZuL22 z-okNddA_cPt0+V3v(kMLZ$&}nSf*iK!LN~04|R4Q6oHb>t@O@KawlS&eG4cK#G?BJs?w)X8Y;wf{)pn18{ z%3l$Dk!fpOsbxVx4^l~=T>0mwo~OF7Kb4NmfNN4Q?tiAbD2UVf<5+i?_ShyRDH-#C zrt#Nbssb1Ut)JO`j^pX1{I5p=ot;{?JP;PPy6d3bes_=v#a)UA0^X>w=rX>oCW}Y{ z$2LgaGe-1#Ys)W*zGt;|n>2;6Avoe20jtMBX7VusmCKdW;}s^4tBpi2IX_djT|~3( z=9h^XPKeX@?l^Q__GWQC*$uyV%aE+Nw!Yn$QAw<8+V&hztesjP9!>Xe&|7cj^|#jT z8XI%)d9WWsW*PkUJ4Oq21ttw;UN30YD~%+)Zf1wFSxAgq<~2H9&v&Oizh`^$wT07- zl`nAWBcXkX#O-2)6yYK7uD=sS?q#l@5p!!z*c-(?m?SZ#cY@ry*4&PGOmtS<1)vH&H54iP=jAQ_|>P%VfBzD zV!3na;$#8-7R|m>YB;P zlKHUtn-la2&mPQr{7&G0Jf)_i9_kN+ca>-ZldY<(oLd`A&b0x}&t@cu*@9d)4v|xM zO)Ofl2Ab~Y1t7RwE(h%~+fI1yoEO!;JPfGmc7AN|2!9F+3d&%Adl{i2)pZVNOr)uDGB??Q__yO`J%j)w*oQq?CvmOQB+g`HqfwAoYZ)(aq7f?!im}Y}dYo$Qjp*$m4njEXAo~kz?1`)nsMAFTEfb{2jITneTZ?M4(E2HQFmk4tgz$&;5R-$A=Ud+T7ec za7EbDDzJf77BT_C!J?qpHzpA4rJ>_azdDN!jZB;`%;s~C`ofo7>)`Bsr|l}YTJIdu zqf+^-zn|UZ(fxQvFC4^vad=wObKP;-gIm^bOhCQjLjLjIVLzg28ya#18ClMKOTGGX z`K7Q>9xbO{2}BtANQvANYi2j@kS3c6A-@MqdJ6MNrb#k*oqDHM#*T#>NL0azyK|v~ zZz}ssMqn26N|cnAga(mY+FxcQhLL$#@w+BC^#(sDzH$HlfpeNc4h~H{aLwXVx8X81 zJYx@c9BbYCL@)aayh0LP>RhVKHbW-7B6$jm!Y&j++ul#DRY;VkUrl1M_#Noy20lW z^Bdh6qYe|liq$vzrc8!<6;*O#Y9oeB)rG;@4KjMLShzjv&lp%aFrVE7N7_${c!dIT z|GyI}5cUN}%j;?X&~Px)a<#b111)+0QvNuyAD)%X(#U3W{(k*iXQWp1Vo=T%hd(^>8NlA0Xd zo||l2c(OfjzMq+e5%DHFJDoJHLRx`>g7!bIPZ?=JKek5D#6sV`ce+Q6D3~;7 z#LCDiRnkjR@>PZ9nnjV*ttex|;4C!`O|>gFv?Z3Yu9AEX6hrD9TepS2 zqs4y#zzUBfo@l{X9^|nLow+D2uT>&cYTB^+5o#s2yHWaEA})U&S!GTEPa(43i*OL% z+)BFu)44h7lpd!_pW(BzRZY3mFNUQ}r~CNZf`e?sza@!2Puj}LP6=OSs2I0sw-McV z9}lb^YvW2}bGU!+h~6Wey#<+3xo7`pju$gD)|h-;rD0rL)NHj;`P;%`g=5n*D3r_T zicCsi*W5u7gp5XYxqdx6GXug|Xl**)S;gXhEV-iciV#N7|2EQ*h)+p@fJ*qYxo8aH z>Bq_E9;va5)t1ACN&YPUHzur6?RKY;;bC6qlfp!mKE6H%&*Pv??-yK%G#w9-L<~?$ z?++xXDtVS0s4!tmAS|42a91!AY!oU9ixv-?#hDNHJ)OpR+lNtgpWqN3y++*Q0QU5IlE8Ua@FyN@R4$x5Up;!L7$Grdb><4`)34Rq*Y&-uMp9E z5JSO?XuS*tcZac*__vO;`mwQ}@#&lv&&zl@v6}DR^RBMzY?P2LvhZf9E~V%2!`i6N z@AxKFm3|w!8?n7w>F)vSlUs? zxr#TxdM!`6PE6ONA&H`4AiPx#>d9|y*@~6>3_qQuXug=(R8^5!)3D#Snp3lF#Drz_ z{PUo`@f!#KJk2UB{b`#%UbMmCDlMPbE*Dxwap8=XymiZJ=0a+^RzWjJLM3cjXK<;U zpd$?uQ^*n|OGSS*jT)9S%XEn|S%zXHCX<$3zmiO}0-0*F$#rU?=fT{yM8~-#;t2&> zru>!%tGWqY!6u`)Uo!>_8q}>+8}<-^Y~~#r##QwjRJe&hgY{jQJ;N3FyXY1p z1>v*pq{NGIawAY=qd23eOYY*ES!^#}OS0XG+4yG^m5>_b!(E-p_6`p%mkyC>3)W)t&ePBZB;tU{nfm|yJLwfWT1v&eP7Dp$lCI>ke2do(!NF_UG zk{)o|(nD=wG(R`y5LIPApez^HkBHGqhXAYSjgCvRWI{G|cn*VBe&N#hCq1bls05Yq z6?Gm++TEWK2=h<|$Gq+|)C@&_2_eK*P?&a=?a8Rj2}mWv?ntLN#hsA*`BPLsO7sP8 z7DGs?MU+#36}E^EqL!lMjrV!~jT%34GGDQ@h=6$JdOX9Je-I=hm#Xm*zWlU1DZpSd zOo9y9Mhv2&mWPP+DvekrdnKr@_)italy%=ElK4!xl5B)$O( z4fn)*pR}Sp_1xna3YOlN8%)1-IpI^l)YR#IK#7UrExg@6*!Fxhd%dvhh@PxFw7fm@ z^!$$aTp$sh4cbS$$qcc#x96X?^lPE{8mvs7nV#HkdTG;kFT(y``OTcQKAF0Bz?`90 zvI2Fy0_ICunK?uuB?`NR&HEzSyhgcf#15IVIiD_&U{00DZH{MfJT;0i5%4)3$!#Y9O%gBrzInEs}$rb?D^-QF+HXjy$e z3%YMDnq$M@Fy7_RwEib<(hjr>R6(P|1iKmbsxYQ+W&Ab+b&@f7Prpc+I@f^NN>_rmP!^qb{@e!*P9)HI^asNLqApYjW zu&$UG`*Mc6C2M>0YvIol8C;W$d0C#To|1oOzwVn}LKt^Ho2n$KfEMVChJ=2|5^G{p z?a{$g5bQ%fa5ni=_?k4(5Lx|Vx;~XhDS3uC3h78A1vts$4Es(lK{C5 zr-Qtbrx8cq8BmOqr^_w;#1$%4^2R~ZPIGZT#4|W7h0Gjx2k$7o&A86o{Vx#UnO8hkVoXJ@V&MX~&rFUrewNeZR-viIBbRtT_e=W>MHIp&{!EkP+8V zAIzql+5a{+lJz5j+q$fgnVp`a9py)UD17*=*QskyB#@Sr=>Xp4-V{U;s2xQD+8ixpSM926bX@T>5!0akXpJ+T4L$$ z?h@(l?(PohUK*BeknXPc`261g`M`(eu$-KGX0DhG?Fha~eR}KQa8krosTs@#`m)#l zzU4b-v;_q^PV1;nxu`gGF0EmBgv7t^nDFSG%#Pz`4aXWTm$Yk-D{2b-gwCU*!{+p= z*@4`(>9F{t%aEA6t-OpNR4tU8w&qih15U$s!B4AV^fPVzzi1 zv#Q@HJQv+bk=)K=t?TbY}k1zg)slgV{dh=~T znFnQV#V{)2QhNoZ3WKzSL_J2~oX+E`n&<6R>^LdgADNcFIWHc#Vw9)^(A$Ab?=p?+ z=fIqg0G~q2oJ;Simn0Eof4RBToPhVb3-V*)GZAyZunNIy09|g)`yRSng~uvXkI1W2(C&PC&@Qw8Ykql-N2rQ$h{(MB z<=TRv5|2w7<#)eUSs}Us=~U@mSnev~4?T;jDJ@HW1DXoMvL<7?xbk7phE9#mkf@~x zfeK+ewVfpBm)&{t*ymzK0rhi*TdNfWl%^l~)aqk(qQ7SV%si8KzoGNx;WC_}5)%t6 zCu+KNTy(l{WuaV?qeG>yukXMr*G5B8@iV!^Mu(?6BN1mv9`c6|AO7VydP(5A8lh)^ zR8|^3u?x^y<1PRFeD7YEOuGVU`$Pmt%f!3i`vq@H`njx1NH3Di8aSSaFUoC7py2sWKpBap~oo{BvhIWReDIxy=kndyxrg{-XEe7G3=%o)ZhtVO4d)$NjofRntU#@jBL~C4Gk(W|7L7tXVb-Ssls~+xp5E2$xRL{Mq79P&?Uw~0u0%7 zUQuWegLX@3o(&RQ$b(WfElp9mP`3Y55g7$@x421dQPF#v%{Q%ICKZO+KyTw431NJG zlGzC{`*9P25g~@(k?U|ma$*uiy{7P3!({zQNm-dd!Yw)mW#>-I_bkXp80f`p%1l^1 z?pU>Yd`A@Lr$-91-V(X9>&!Cd>a+O~y93qhj*(@&KKkZl3@yfD9w_)uVJ@v>5Z%u| z+Nt~Y*&Q(~^P)3y!rZuNnMqfQEX}}&AMIQ=YZ(0cWtj1bXulc^MDr13h?LpIJE}G- zkS>!5o_g6FU~#6?9Ka>0Eh8_+bO1)+IpW86zabp}7kAQHY6YN8Rx zMS06{EU)?nxBdeL*(cF>y5CQRY?=$-V}^uOR5c;VH0`L?2q2|p53+uLkiRV(luU2_3fb1Yn`D`M$neJivK&J=vd`Yml{X5%KBV34buuNB08 zNF$m|C#@2H-!^1b@9N@j5l{uyPzTp09kHVqV%Kv+(OxZN)eFYMcZemKDk>r3Z*QYQ zEvb9@;3FzC~${b#vxV)e4ANg2}fEV}iSFk)RYu4i{O6!dLAkG98 zO;6K}e1c|Bq5nY_4=5~rR#?MyasA9I_|w2!LI^c6@sHf*Y?3spYL-8u)vh87L+Dp~Y zkbPcI1}uymqei$rY=0yEzh^x&TsyFZKI#O%?k^hM0*36DMdRw!r}3GQRM3V0KL&=M@khl zrRhaviw0TG^n(skH5Z8AdOKD7qkGO3L{8!zg=q(uVLCxrdOwn~FB>~YOp2LvCKn|L za!VS_e8Xec2)Ucdbz-kZ#3!j!j9Z8>aju)_qRP`t4QLu=(2z8k_@;pX&WiS(6UC{U zoD5<$R=UgHxVJM5q?jto4Rb0FH!oR%7!!k8Qd$g)2I)WToIaXKVYoELdJ@L#zbm!i z$6rQuW3Mlv6cQG*ipwmQl#Jbt~`RSobTsp+#8bpVO_c@m1Xe{R(;m-tx{tcLaApuStgt+6T zcJmb%dC$&;J+h&S44xSib2RXb83s^XVtvt7VJExl^FD!QO3lpeZQ%^YENgtCSv1*1|sP75apiGxicH-~hl z{ipXQmi&;N9AGLnYXr-wE)}Z#sH)g18uXO?M*UutHTG_kc(^4! z_i;nHuPe3$pfijN@4o2GTz_^zUwI@k2?|G9v)V5qEc}|T4r({>KPev5qs=KrE?dRA zed{mXAnDF#^}aOBv7L`vK0JtSggV=B#=qO|DN^78##?`DGg!-QKTA-U+wmwVDJhS0 zahIvxAlgfHn*OltjEPf0pP>v4(-L z!BndDYO{sRfo3(o;tWY*x|Gc$dS=A?4lPaPu>SnNo1l^p+=SZN+7FifeSJV|&g(Vo z&fs<8{Rp5|-^oNinM5hy>jI&oq$@I6R$WGV=;dk8F$hNz+cGbfMV}JW zRO83}5i{7IEV&D9a@iH{_CE07bT~qj`m2uQ2sdw>5?QD=4Kn6zoTO>`%L|t(UZ`37 zb#S63)6Uhe>4;0Gqwll5#C$#AFng55+1ShZCiaPtv6)S}tyk8N$H4Hf`qr`9Jo8A~ zHZ<(Xi|qH_G>pb!LWA^{eHBbMKgm+EjFOA3_iCJ=B5Z-O0#T{#o{f1$0W?k~xWaE% zxG5io^_@C`_#egJu%nNZ`_k64wX*C*1=7^mWl-%j=kupzo6LU*bCYgwRrvtUrx18) z!t?%=h?ZwOzDmgBV6xUBoH}Qjo0BUt|CL>UpX8-x1XQkZ&8HY2-6Iy$xECuY@bL0z z{SguIJyNWd+8L+kgP|-Wxvnk&h}d)Gk4+;~Pfyd4;>hqwO`LpW>4xo1;?E~*SIOzuH1=)pOSwls}H)#n;%i_M4s;v6lpDR`# z?%eO4C%gpSUjU(DG)&n=c84e-`T{f!= zy`Ha)RpxlCPdvJbI2{kRhm@#9A~@95LC^J6XY`*F!a=S;1E$)TZy0F7C=O9?ETpHV zikV!-2>QosGr^;_#MxlJ!$y2cr|FGRpk#d{EgKFumdYkj1!EB1R&UG^Q3uPZn>Swe zB*WF5hqNdfAudD(WgnyPfb$8ldX3C~71v-Tm*-o^toqubjQ^=(gzaA>WDo8!GT zB79BPAIMtYG=+wB^rji?dJd+5o9zOL+(kI~LoALiAXll7s4so+x6nE!q*QzOSE*FO zIojCrys0z;K?`QQ6i?@9xW|MB=F}kz=N~w&mNiqa~qW;D#yY&I)`Z`<59Y(-y_bR5^GKbxq2p5Qj$wNqna z*r-m(9(qi{kWZTmt3okLO@+(H2*nxJT`*+k^0O~ET4?|1eP@@*uu{>s8Cq>HAk{FS zEgPcb`u_TckQ-w*EP!$@2;aJcVQBTsJ{R-`35r+>L&1L5$J+yqYfe?KgI}R9x-ub+n5VvAEBOH(AHKFb#clZ z^KE;X;leBi%@!}T;!De;+$x2x#ky!AB z$%4m8(qVJKe#b-av5Brq_A8x{cb4<+2SMq59yBk@LKXv$36!EAyq{|V$6=PK3qv}p_{P-Gg=DNtD26KB0C0o6{6JT zu=f@MOG@xhtZx+9kKRmsETuV`%{PjfvAQiQvMb1pm6+yb%nfkrQ$ih3`&#hy$~Zj-)Y)0HkZDD}keKle##_cQFyXX((Ec zmb%Ulmq{ZLG<4h%H)*U`VS}CuvP7{sTN5@sP~ki|Xg`HB(UF1SeYz;cVI3%aZrx^r z6`!q^xlOGfl{7N73q>vxKPP`dIp2+s5Y5uWY9`;LP1q(yTvNGC$J*F|JBZz07;Q1C z=OD}CX~KIjy)H=ZbO^FNTi)grXFm>L%yY4Z+FG=JAb5I0^lbCbdg9mg{9z%LbD$rr zvl``0ifg?_(zOp|Sn);SAyfNDDE*@-rGr139dbS7XEq$%Fv$QOEC-?Pbb*Zf&EMH- zgF?Wj;HB1U<@ReQh1NG79`~+~WRm#o-z@2a))?xMD)hFJvug@@VrhPRvD8rme7H})$Ei~(~1r$@08@OTf> zsYp}@_0(DD*b4E%^<&sSZ`w9;oK!2K;(+x_hxaioOe4K;a(IYnq z80ISNp;671fFJ#Jj}d@QGMzd51QMMFVabsNds+$8O;U*C5>?WAhHj{?U9{4~Vv zE7PdIcsLY~Oe?KybGt`4Mb{wWvuXq7QdnG8c0SQf84WTc!h{omK0TAswfa8Z_pAR) zzV!EG8AFxA7YXmI(wVpj;_+x9Cs#J|;f=Y)4nJ&o>HTufWY-O>o*41+^e+3M3Yhd| zjA>>UmRR_=F<-ntUsh5uL!yz?*QUD6loq73k#?#U*i=*7wQ|iPmMzz@^u@X-lr@gw zQ-1x~5JA*iQw-j+UdBP?)Hg0%Sf+=fv(%az+Q3#bA}~XwLLMJ{KjO*u4|!J+!igS! z%CBbzYuvU39D-vPwl&L)0<6Vwevdhk3>yX3Y;8Tc(97rK+Bc2c6HIeuTjF)J7~VsHk?Jw3zW zdM0~o35It;lXzC&+}N9`14-+Yr33gS3_>Ok46f@|f12q1 zhz?@)N9}AJMN9sSUhLS5nM%e0EuqCZ%Gn&G?2*+jnS>z?_}4nx+m&N>%l_Ma_W@nv zt?x(OA1>2l9P1AuK6iHkf{924)caBo*}@33n4ZaFMlt-j+=hQ6&B=8g4J2eGy{s1= z1D?Z#NOAj;MhgHr`q^R*N0f9`t@8pn7n14zBt942-QC^RwhW|fQ#COKGARJ3y3ur{ zz;el{T%#UzLBq)C?Yy8EV@Wm294gmAmEi(kJp4GaHa0$fn?a05!#F~X$PXlEIk~b4 zh8UNRr=2Lo>}gdO!6O3ACqboWf~1O6w6q63r~>;F9dM*uOQ#)zT4uW5n1H%_Y6)VD zD+*u(=wFBf?b74Sb-o~hp3nlFU6gm2{nLint6vR_)9Hs5^cyYt?Hl`Xawo-b6{f?W z#eohr;p3Od^Gf4aGp4xQp(7XFkyIPsPnd6pWPgm;Fp%mBfK4TA8=i&Ydo=X8{M&OS zgL@>a-4qzk3kVUek08b!z%94>B8b2d5>UQu>otFG>?m5fWrV{zw68F)@^?wy#kHiF zhBgCJSVz|mmuu8(cR!=K9E({bFjyYfdpft)f9tp&fw z&BCC5zl5m9_RPxi>Ao`XZ&3H9%3E@2`tnJ)j_G9&%RyiGQ`@Ia!sOU*!opfw#Q&lb zwKJl~88AXqAiT~KyVTX+P^8MWa+P741+-%nwguRC(oU^^K3vpH)th_o&sTPed0O+p z``dQ;1x^>PMZ8aP7}*9|fBeTT(6x5s#+IDQvNAfs7~#&t8*HWG} zqcVTmii(z>42%u&$qQFZNPB3_V(+ecyXprZn9{S-uK7DcV3q9rno;pSZr=Hw+M`fX z=KX#XIDF%)=)G9mvB@v0QYDcSc~dFx*VrwqLl{ab+J2T$H>jyHtFoX7DTd7sD=RO4 zEG?CCxcRYmTuo(aD?%>RV`5TbG+Q#1PTlox!r&l{@pI`?-M#MV1OKtZ0rb2FRTU{V zlzy4xwadJ+UAB$yWWUAHn@nYBj7P3U=jm07vF#sUi?qP1Wj_Iv*mJR2iK_brZS(`$ zM_sS;iEQC|O~36rrTTk-ws#H)_QdjvphV4B9zDjOFAeB;e$Wgx>`rN`POw`){_jNI z>zP-E1CA`q)|>P5vgH<)Qg(nwkjk>R8YytaDHe6-rQTpsEbcYAD1M?V9}E~sCd%tn z`f^h>#Y}<}L2a>P_o8feGAUCgHinUy_KI;qb0CHOzai`v!!rDc1B@r>H?;*?JYS2e z^P}f9954*Bsndh>a)UISMum+NXc?lzm6%I5NW;@k^SV?_LAiYjQceyZoCe*6%d%B| zF+^8z%|G)|+GU%Y%Y{wuk8zSqRDTdR4UI3fL!nyb25pTpO5%nqJCrd^TN^1DNVS6u zrZ&%rmgphjJBveKmPI_vm{v}#)rEsd8fG*`BNhhxnHC_`B*rO9`BBEWeM}7AizVJ1 zmRfh4Uqg`ETBH~W-2e&~kZh0fdp$WolGAw_3-j{GIZUTmi-#t%d-}suKVf5wnH&Qw zB0_#UaqY=@AbxIbZIYRRG|;!|=U0>1%prhxQ?=#N+!6W$0Fc^D&;iaAfL>q6>;A5{ zmz+FvLLB6J)_i$YE9ddLa_sZam7GO)p{c=TBBVkA~WFL)@JOzz?9s z`=w>SUvm&UnRlQ4Nzk_%AOgkNB~XaTbH*+>P(KQB)5!og@+N$88(gwAKYO38ZSUX&kv;}jV z0;3Ld$wW;uX{Xj%J!4&F5ft`JlrRB_1(K%8r`N`Ngiun3S$AVZMjgw}aTaVpOw07dF=2@%L|B z7`JkC7-DAPW5z>lm_8(v=k}ykib}T4uNOst4UeoM$t8bWx3q+|w4_A*Xg$N*cH^?- zYe(qs(2kdzQia5DlI4WK!J-daRI)hUfdd}33qsIFqUk6JkSk<&TRO#Kx*HoidfNfH zpOxeG>58XNH6by!G5Onw$+rSZ)xBebQ`QW+T&oPtv(7od1Av8Dc3!Fa0oX|wRxe1_Ks9Gy$PRTMw=M6^fnQ3qY-QlDz&o-lWoeX|d<@90tKFF0yr z>^1V(AfN@L$LD77#=e_Dd3(X=+XE*C!Io9k#?CVMNsGtX9uVxZ-siwh@%L{WFuo@4 zkfnynJq|{+bN2f_|G?j>T8xe-Jc}CB^yiaa=KZ*d(0;ZcpoOSXpo)kKU);VFwQe93k>Qj7O+2jcr zl+qx+EC#)xvVLOz&Rn9Cyg95Tgw4KE)E{pZN7x9^0a-&rCkyvik`0YUxhC8kW~H#s0*lZ)|2}=ac`f=Wx5>wa@rHP+qX7$|NUCDuN%j z*0cR}y`TI1_H^RPaPjf6^Nv3!NA@^eo}^x$BYf4UH}eZH>SWUM)~Rid7!2R35zT(d-6hNAQmKxnvya|jTvd0LCYmSD#y#0&0N1HDgT$@2H{zXd?P7LgIYWPx7HkxN608$TKYR1^#%`o(fYtdDsI)Y-_;;Q6Q+|kf}(8Jy{y16`JB?vBqxwauiJDBC;Go3trxJKTS=w z-(^L_K4_amBL)Ks9glqwN^mg?g8e%qDxGOpZgVde3Z-h7QH#C^P<)IVnTM#yb3_>d z4}>+h5pra&ZS1<-oxRNYwu^(2?eOgype@A%#9|nqU^J1}_apqp*I1aNm1>uQ%q6r| z6ezcPTWU?E$7RO-Nu`AGl;+v}xNr_TXgeOoDL5Kq9@}gJBU)V%+mL+$*XknSD(+r{ zcg=6eOi*8KzjRPQ!S>3k8uq4ZhZoH{ZBH~(fr0jG1Z<^S^CwpeO^KN2MpFs2n!aOt zIi8DpHGqoeus5sTH~#j4*Ce`7uhK)N_a47pv^ ze7K86O{_^4@VBW;8fEOb;^qHIXT%aCM%@5F`a!;aQY@H@AGtqS05QWUS?;fpy`9X{ zUIJl8#;hr6K?=iy?pX)}Y0#W0VO#=Xq3@iDP1yK8R*j-u@F6PeO(G{#u|H>6FwziWcv!(w8LYI4xcQxVv`W!`nzX1FG1e#y9d&`&o_|}dG&=19805Woq zI?7k&-MPFA=mcO)c#BAhk*kmxzQa)i(>OT6!KFv*ywpc1lVLrI`jvSW@ZtVWXD7Jj z*=bwCXAt~X5W$`q7iVykUSjUC>Pb9bt^e;l5-$@MRRvRL_z%q`5H=p-7bS?1O$8;2 z0xA1WaY9ZRRH(Q;h51lEKf#H+!bNYfSj&;pN{`y>yQq^1!m`WP)bn@N?>o(OO83L$ zT8?Uw8%PbS6|rK=!oOg(Q%LhI2a)`A&jRDsW%_A0(H%sVpzbUDZ7TWNgc#D2;pb#2mqDfKxZ@BZKG=eejnw%N6s9+aek{%>WxU766@aq zFAzWtdy85W9zQB!aaIexs=o4wvUplFMbkSc-1AiS|+9`AlEth006$ zL*dszV}Qedj_gEddxqwoSC{S?h}t}po%Sk7wte&LQclyy^-~r*uDD*YpU>%?fDrs+ z21z4V9NY)?P-K^yIh;bmM6|-K0UnpQk^>x4aKH8U2C*nxB&AwKLsE7h2zg{jps=;YTI8s4h_k?U(!<0 zslEt4<$u149Uew(q){V|v2X_h%U>JYi&Q`7?sKoS%2)fscRbu}JX=KxyO=Q+4RU)$=OK^7sBC{r zPHn>JkIE>$QwVue;uvU?V%Lco?dCa z4FMFaS|vc=Ypz`Lb)|0OwDXmgn(q4s>`BE4NQ>R$4ooUE%;)ZGz1p_#kD6#u3Ee8p z;B$|}cjt=HADJ5i9z6s^sk7CixqA0tz^9<@G$}5PfUqwnD$0a8SyD>M_D4ZmPEHff ztNX^phCgR_?okq~GAwcaO3QUt8%X#%;{;l*{W3^AWlqew9s>&pzs_D~z9d`ovadvS zOUzm2I#K|*D`s+gety>v81u~($Y>6kfx-FK9oO)0V6tGlX0E^AtfS0KGlN4)Qk@m> z_^m2~oJXnmetVa(&ZNr*VUbVe(Fg`am7Ke}IESEKF)`dzC_VbN{yckktF)E^yHb>5 z^gf+KgzL3*$LE6u_fij+^$7uc2J%^h+X57CC**b#mlh8LoD3o&vQ0~Ba&qc& z+UoMQf2hvg5H7t(z`Zl!70XMock&Z@F`(2{nd)o-x*nmTpv%Y&9%eCS|i0 zA{8)h{z5v;cyg{ZCKbs~|Ncoa^rExO8E!oW?!SMv9vBL@@b&AZZWr78S7wp-{_rZA z%KHrRI+H9`w4=-#=IYyQVrS@mGfxao4SWG<~j=0u_(kh6zyDvu%PvZ=2VhH zR_J*1#_-fz#>MOO=T5q)WXdjark?xfGumdeJNmw^76GrDX9hGg9xr>2F6ff zy^tqhCIX84f1A$)%*A97Y`9O=TLRwyeU#M&!_lbs5!kar+IB|e2++v=w;`u<$I&oH zd9scU14qR8o$}wTJz$&R`OdH54Lf9l1B*iRzmF!?eN$Jf-jL_&DWJ$xhJK}XKCK;Gt@Nb(~hrKtsgNvDTgK_cy-$&p55v<$}-G$Un+J{o3$p z_)%rU;|yDf4=6>ldkK-LP}BeSyuf^~j17+Yzg_^G&l3?&U2gVUU@zeWe}vU9|2v2Y zPG6o4`JBG*PU#bI$?IV>j5?IcDF6E%jE{hyb1kh7ya4H2Ffb>IQ2jt&J`@EF!fmhr8dhSn~70e|oN+c7Sip923K(ih$@ZF=y2Zvj_U#PxMpVAr!v zseedSBFA86NPU5VLNq`&GE`P({cray9as*iU=9OCuFIl4SyhxghKb8lqobOZR+?eG zai;sjw47QN_h#24vDGB0N}KHEE`3GI!y;qH9maphmKE0$6S+wv&NoYMZpn^sN8*vq z5Vyk(Sa5-lViyQefD45j*|xk4^i>7>8P}#mFYH>^OFm2L>u<%x#6U6ys8)q|E$0ig zwb?%ZcZE>5SGkt(LbpCqf(MyPD1lQz#=W`Q&75j3jGHIuok{QaUia~ByZT;}zS80> z_TS$GAlQ7=ZZt7P2}vMjQ!W#fAqW+T&+(Yac>OTEz0==3J3LH7GbT=+%T&!&2U4Os zwwSN*1kU6WY*PxTG?1*5=@upq-?-VGy*)mPe#tM`oD5MA!foAzUZ!Sin+NVY<|yU28N(QArEl$$t;y1A=q)c;_KhUnCbZS!J2Q zV?`8(!ypu;gz^5PVhWlBDAz!rb+N^DN?Aqw8jy5ax9qE)n^DG^&X$}WmPEtAG)b7I zOtrb%51C0w4B2$Q`QDx-%bFH`@UHPoa`}JXNklE`+YbXs;6k~T4fwA|WA9LE?F%8K z^d5Q}|4tTgVn0Y}d*TiCbl2T1QFO@u#JUPI@jkkgu$+-d2FFeK+c(eeKB)+-q|l!9 z&TT}-&WxQFx7sv>H8$YJ8}AIC9ytX&FCj8w@ZN752$bKW0U}>4c@edLQS}S_auh1zad_ zT(78num#>qL|bC}sP*?Ibey$eq}ybsR$~}`P++D`*KNEACvgQ0@RSOQjzsno1yxIt z+vxPjs7u-wWpFCnp}mE1L;$X^$tP!6nD1|Z#;nLcp6+w60rX)a?!NL1LcGe{g>E-j z1g=%-F0n?8NCj*biC`L-m5w|YK&Au&)Ioq^9tOs>U!h~-4|#eQgTJ9NQCwzn)|Yp9 z{1Cd_?pxz%XVvfUvcNHjXFq96n90Gym>dlvHh#ANF$`1WQX`S4lPb#Ty;Z+c^0|zb zbdBZhD9Eq3n5r|4?cFAqwzE?E+~lzowjRE@Itj!RuGAGB9UWC2RaFfQ zr|rKRu~c&6iyy!vfN5qzbo3;1Jy>6n`NgbW1G1L^*t)~5$@bmwIG-#t)02_Fo@O;c zp`4|&s=odXJ3B6NU_g!q5IUEg9#SjmMv5BK0W;ge!oscegrq2w#-)qPOZ)N69&Ow0 zzi7(XVlMJ~#(-7bM2CT0tYKTVTYE>=7Xc2A)8+eGKmkPi^{X(61?tJo=;$cB=NOZe`(XWLxOrYVoS zha`qH%CU4?B^OH7hLTx~Z;qGnH*cCcolMq9nk8Z>C5Fy_Nl5(qrCN&lirf9Vvn9-N z`gCu1tynxLSOs{n5I)%3al*2xd4yiEA^?Wd=IVpz`EM^Cr)Q>x}m++bnVKN@e_ zyCz!IGIjXeH(KjZ2)Xf)KmrP&P=Q}|_T$8WQT1T7@EwZLtFI?kE$+KmL4SB%ZWr;! ziWSX*!pBG-#o1YvCWR)6CIucRD*%Z5-i2_hdD?+}eX>w$Fd*yJE||9_8Tz|cE{!wA zar<^{o`}m1=UqRez;lE)Vf5KJBz^+p67rcHsiQj8yUig&e9PVq>a)-{afM`Al#A`zr%dCmq+Fde~>iggW~znfZ+Y($(LM=4&qz zG|o=PbK&;co{YccQiVd`)Xo1Yu4r=8jz9=M2aoR@9qG0`i<#l^`@HAO_oJHi1kBW#A$u>?Rq zW7|hXj0Nb5zD)d|&#J=@2M1QY&OR#gD_L;>D!L8lSoYw3#ObJf*a(h)_wJoI7x`iN zyOZ(5igtclT1f{QMVqvno27yXT$YH#w?TP)Pth#;cVIy8__7tb*734iI$bD73bv8E zC7=KMoK@xdcyqbOH&L(YJOn;R<2M|PA`^@7XuMd;^j2i`;0+Xa3nF0>{wbDw-WIUw z$?AU7Hkk{OVpyt|ny=7qSz)?{07$}*mo9g4zB=xeBqjqY*`^X;Ioh-omyt>5K8E01 zx9nVw3p{SuM)L8w*6>C6p-kSNR3o7%t+DSr9R-Mph=kD1hUMRfppl^v@cLAu0tQcP zIHx6gjZR0fVpX52t07-j=2rvqu#9VM%G4j#Ph5Os-?tMOfBgaVnm ziJM!0yZtFXk5d{bR}j|6`W(pj&iAKJHL5prBL(i?3rK;?Y{q8iA?Yl{YXl_u%l6|( zL$TJcPrv2*3EGb_$xX`CB~xsxGre(5;?zaqnNT#I1L#rmH4T{>zV@)c2nsHPzE{R+^XM>0=VAI-JMHboYV2*SduMjL}JI&a5wPWlO}&M#x1WG zZItQod}({MZTzBthQ{ykx60OFfbZ%IjxzSckgxoFD4f{lN=8zgbdKTg@Y4EfI0T)x zvp_Y&bdkHkBS37yHm@UKu$#{BA8yxtw<%?Vuc)YKNPpdnE+>=1y5AK{yxi_y=x&4< zyj|1OEh#GrlI%)dD>uIdu9{B!ZZKfT1Q<2fQva;zJc0L6*ddtl!4dH4-bsvnHSQ4r zC~{cCTAqHDj^ulBTA$I;NnmMUX2^tSaCb&r5U1paYUZBvE4t&Z!Hy?;QAmOdsjHq0 zZnjGyoa5pW65Q?w1a!{3_A(~|&vVz$QKw;HVhX-dgfX!=ON8lWhMSW2V>ZN%4-)ca zw%aaJ)6ocM7o1=FMtirv;-*+Bc2@F7syK@Dmr~dzygbiistp*$ zU_YVb-(HMV%BC|TyZay9;G?YbzAkrvK(G8>L!$n1WW{;B?djq}Z~5n3U=2K_e}r-O zr@irEB7M`kFtVz?s;o-zONjht1$7S|xx-))5=ATb~)*@9Y2z<7maI(`VpaIK1Yrxqi6A)Xl{L{BHmJW~|tU zi2(u?va4JUB)zq*?hNz9Q8!SG5QTlyWxuZ2ISZ!g5`sJaeERMPxy1cAG&D2~!05c7 zB9oFYP6NHY9o2XA5PaoAzLOeIZu4sgf>kQ7io+1va8;(_z%jN&@KKzXZk|D44?0fd)$wKIlx&0O(E~Vz2 z0RZy<<_O47G;Kw+wm2N~JD>1rt*6HUx{XtM&DDZ>B;0Z>z@%s&Fm->n?qcEh?wbb& zM8ARY@eRwp)?p4kU`SC{GrFgWl0HU2>aGm?TN@0l4e1VthQ6}>_>Q{*PwTmjvMp#s z*^Y+|z#EdbVpK{$q*R*@c*OA9ZuSC!%XizSJ!F8`Rj6?KUF#YHF~`eHacsat=^_{V zvjz|c@H`hxE)o(G6YpnuHs1e9Te$q^gZDW>UxH_NcS{P!sqv)ATm!c&9c;-~p!>K3 z-l=A)zMG)kdf%1~^c|Z4)zGX!h0t4@mN`j@#~5aoL__87h~(HX{`=l(kilCC&!Ky_ zbiLK(dpUvUw=q5G&dal{&cp(X8W+@(GGpp2MgmWRaA}4!3=0=iSyG_!d#%c-m`(lV zAV8eVfrmtXEwOtG$Sw}zOr;V#{`s_>ctU9qZ|c9YkZl1mp!@NC^kkkyY+oh)udt9S z$EczONHiJa(#_gjrnk=4GfT(ZyDe+veyeEPLF0a5u)2%X5OJ z6*^9q7I;HB{XRK4xz=c0xc4t|&?}d)oS`wJh7@rI@fc`yR8O8~wM3H9-Ll5Dyi zg5J+ycL=#9t{Og*9fqsa6Y>_n<-rw@v}EM`{dxQS_sq;*hWhuM@AlU|d7cgcNm?r# z#&MC$`NTt!o+42E2t5<>*~vlUIB9YV{))J0(+ihtv*s6ig-9>p@33k&Vt9XuS;}QH zvF`N@|G6PeNHE~M+G*`rLPng)1{s_#JoJ!YC4WG<3xODOx4x=4xLv$eGOIO`*yZ8F)*?3(cPLQBuqgu zTXpKSMvbf7T+Gaf#lym={BH9rQmCmx~1>uqP z$iR=flnRZ*;Gvkt_x1+;@Jh^~x0FM38`@`md%6f0?DYZ}ZU@Qx&axE63;E{r0JueC zSV+Hn|BmrwvE5%vnGyt~Q-J4~)i)gZq90eohe`!9H%TARlFK$uGBjrPp`D+H)Peq$By z(*pz=O~>P@_~kbdG?ojb>Ad#ag9?c#9m=XpK*={UG6G-_F)^{tOCq5j>b&jd4`Ec9 zZb)c0)5shnVLzew+xG_joQCaL=crmM7j|P1*E$hDUlZGF6m(q0qJgO z3F+<_q)Qm-?#`hZ`hRgh&+~qK=Yt-H2+ZEI_rBJ()_ML;htViw2}e>I%cOBtQ6-h= z*Gn#OKftK8Be-bm^NIMC3mHg-+yYvyXzT6lWCo}9y`D1fBjdA+*<*Ki_s7kwy!Eo$ zmhxIz4=c>}5BkB(%jodOoZ||Mq@?63my_e*b)&_Ke_I=H4E)Sx=KzFbQPWkC$~5&*{=x_-KXvfuOoP^ z#*1zbvak=Di)vGWl>R>JuFfIB3GPa_dx8+A6cTEMzcEu99@ZXgIfpv%?_bb$O19ix zT=m5*JeB5%2n#_B=Ny+A(ga`=n;J|09(Do4xI>UeV+x`_egVFxJ4CE0kZdUh6&<@s z_ks*&0re&pSChm)~w;W##GdC12NmM(z=2d6LwCCiO}OYJ)o zcd1^~S3OJO+QlUrJ0DT6^dKOC0)(1k>@5bE#n?9Aly_5zWeYCW3KnbEo~7AwCCkfA zE-~QYMgQcK{rJz$qZ!(9fa?}PsmOz$? zQmT;q(drY3p#os<%@rtuhw0ihh#FOp4ZsP=MaODNEK7C2a9{Ag+q)maV`~ZQR$0v0 zuFsvHm`V0X%uTRU6Z`5F0y>(dDep*@8Z`!%q)o-5JwjE0iDv(63> zuY=z3A)>e_+!ixC{3Lpz{BE0zplDsK)NsF$mEQiH?@&= zyBZU5Gu66ANZ~D%afc%DM|fwVzFJji<_DrY&26{$34UOg9da!pePfcCV$lQKl+TNUP;G-RtUndD6~M z)KFdB_{$en2t>fsbGl&w@a}HFbyGJH7n^v}jqr|P*yzw72P{{gRGV}gnB6UEtzw#3 zi6IHU?WjY-$o?F5pU3BkL9%oR>0#!mG19GBH@ zp(Z&yDq4Ew=v=tr>j4gJjFTC>1i8C{E#zikUa8it1U5qKz#pjfMT}Hw%Si=0-Nq@Q zwDIf`4IY)-yVlDW?{9RazII={*-( z%Gd{e2b}#YLOzMpaPpUT!tOZ4#9ME6GR$~)CWj;+p>izZQ0wnT1(2Vb1+Zx`zaS-W z9G(9yb};Zk%Hs#+Sn@(m@y*`E-3{V3E#{fs`W*k&XucXx9$iHkB;O3Kzs16#l)|$Q z*S56OF8o+5JAF1f5^;C2fshH_8nL@G?j)MbHJ?dwny$8CU`_m*Q<_a%tNuy<;>Ohg zdfYjcLMqX|`|Xttij@%u*0b1R1k29y(Yc1`|v=aRd1{p|tR!X77O z!dBJCMR=#}{iVz&ydO`)_ht^vZiILC)9*g3eUW)4=H=tJ@m(6 z0Pq`-pbu@G83hEWgN^i4Z{eGc$}M7$9K-LrT>;p_s}#Ex9wwCpZ%TEm9fyAT3n?zS ztuJ?mz^;#u!CPQ)GmGePw6MtUdCwqXX$C^I;9!hxHjotLTCVKd=(^WrbQ$}mwCR;F z>yoWppfy(g*UgARZ;@N#NpDJ7euvKoV-xyPs!g|-g(^`|x`QvbYMgf`ExN<+u9KEo z3nZkE?x#jxx?&wF`XBjBrfuw6Zdkb?{+nZ4Vut-yCov3_ozK_THq3Z&+F7GYF<0}d)*(TAnbnlC;w4_SfS|a6S zP8;fd97mZM2jvj?U?BT4tF@U4U409^mJquFGC6GFqi=!SJx$TI@UODyUd?EG&u1h9FT?RV+Q8XFtCKA?7pyEDXvXBW8vOef^rwgR#xZ8GsilNgiy~ORP*UYb9 zpOHzx>y_QY77%ipnx*qVRzDDLy$mvR41Iv|@6$J{VC6}lt(}kC#Lp@pu!=dzbfro{ z+a4tC&o$6I1nay-SQkw$A_4)sJ3A_0i7=xOhBEJLmS>@3dxT4{rCGDEvv2<_ETtJL zX3@V6E^4yr7lzpFPC7*66}_R?KWqzu2dRI)nW;4Ux9;uFQC>R*S~aAT3i68b%5dai z_t>fpzx&Z6L;c3k(DCG|FBfhY(gGX#0KRN=xu?}#VeGsT*(&Zp8drA2apxKQ?>0R3 z?&j=jdWL?1Rg2@@yLUoVtqXaZ6&{O2-Q?4iW(mf}N@;welU(GaJ#(PfEQ3~E5$cFK z5@RZ%TkWx?Y2Ynd!S)eSvkZ2VqQ+Hr6SvC3c20&};ms*c{-BfV=CJfI`o)*@gXZ&j zuN7xvUj!P+1|23XKt2?*8Vr3ZDM`C`vD^>u3MJ;YN3p0_8_Mcl42ftla1eRseNa!I zGDTZ2+pUy(pz)@e4*DEvJnr+r1fMpzG}PyV#$uu^=JF)lyLj$M|&9&U7s*x3y3zEW6)_KQ*-g zJr85<_jEyAp3T@j#wKU3Zw46h)=WIDF0Bpj9Y)nhFaxT;R%W9+o$vj?Y^Z)SW(9z< z83MUIhAYjK+s2nI)4k4~xZ5^&h7dg`=j5TLRvmVXuoVO;tK67H=v;Oe=Y^#(O6wW$ z=WMOrLWdtMUcWfT-&e0Rbxs%Zl4T}&_Bcuk?C^aA z%vSJ~g>0)o^-KpsFSoi$N?P)rcewn9k}D2+*n@w2mtJ|Ev#JiDEr&OA>wCwPVe?lb zs`}0ZqfyUFfTehC(Fb)lut?P=>t)m}8xs??fRfZR5)e|@-~022>0r~-d{KQUIX6au z#7rprA}=W52=kr?6e5zZk7e z`-m`!&u%x!<%6rf5M=sPGvlhzv7Iq>7Ep`cuT}LAODR{Pb$SVsduid*4x+{e9-I3awBPMa-k}3isUQxPl_GPnXf9FnnBef0W9zw3?|&WompaYWda* zj9b617A=(Nk9VaBr_0xslkqzrneo1VU#z1^t-@zI*Wm0J5#KU#bJ}+N6#oe;5hFpg z%CCJs`^7&=DI<1yO> z^+M_OS*mM7nt=O5f4|sVMcFg9|GfWdZ0zA&_Pvajj1|fn>RGk{Kk{$NM3He+Zdiwa zhTqxU*nN8Rhhx*6^VP;cBLa*Rg^Y zNx(EFNMuUqwS~0fTdV(CKkcoH`jU=dDsHe>6an2!h`kh~Pv%lQ>;nm1;dA@Q9fU9e z>Dzm)DBUj78OqZ5c0cP%6wlSVQba4u z%D%hJ5TV@hk)yaxd^E*Sv9`;m!gwyc?1B+hrlzkSY^Bfj`RZw66n&pvg6+i^EYvpQLM$ps9_$R z-JR8zbFYLet>Th;n;Tvurs<#pAQ6L!Ay9J^-2y|z!%?+6tN9{gup}IBPPDXK z?Erd7ML4;j{lOnFgygb|;*Nfx{5w2VHkKEVVedBEl|Q^~U244k4#^IDaD#rbJ@(rs z*#(rVK$)DAolP0{$JM0392cI#TdXiWT5N?aW-`&*j=m!yC^?CW+a*&!(9i4nLwzcl zlIrxCbG*tAl`^)2YCmJo{=WVjV!N>dLc<7~h1>LsEVS^WP zI{gA_&hr8954V;26l%WnJ%SZ`(DqaAp9Aq=cVK(%y293SEkS-fWX`$D`^1-T?B6S` zWeP~C3-BbPpHNH!GoEkjW{t0FO0bT)w)QW%c#ts4Y%)yWIH#1Th$|iXKW8M`2zK>I z+M-UH18+uORhQtSmXy=6BcZi56e`pUV`ct(ZZ>Ppg-_1Hpk(@nUt4J-g-W1BuAQcT z6Tli1rzgC{Dk3O${*?&!#`7I$F`3C58Lj-lBIYvxM+w>LGRb^TSgMDHhUeG^B;>#l zUxDVJt_2D9of-I9{Za_>NbDG%%HL?zeI|T+3I4Q^!Tn#U$Tw!BReYYjnN6ESc_o8w7ur4!I^HQq{?g!ZspPSN-|^_{Ck_Oz%jp z^Gr9n7vx`tyh@(E8SyCyG^p2>@OHT#01X}43g13GcCq?1)bvt?Fj?~!kOIUL7?vHV zEOWr)Ca{aMt&QJIn?>$|L6;Tki(9+m1a)MOIy?$?^fr?V8GYgRyK=ykZgkqcYSyy4 zqr1}y{+bq)PX+t9GKnU!VQ;NC`^H1=aBy7c&!DuWdgbNi=Y!3Ncpdu84@*?Ur}fRi zDF(ett$hJI0QCPlRIoN2BU(t(J=UL)77*Uj&ffv@8zI6#R?M5l|KhFi$-TjA{~|Ls zw2ksqMwfltrzL%LP?}05jOo)Oif>q@loiHBNq)o8(L1(v>$g|rq$?0~apYUE>Z3W= z`MGMRgge3WP?`OtieXhvri6?tpC;>sXuZ?jA*=1iJcJJA)+dUx;JTGf8E{dp23z-Gy_>ynLOoqjO=N&h1~ zEC)=W-7nTAKwV7~ZIw{ToMtjQP`dzeK3eT%Aoo^%i;V2svjB?oWV@!Dwl*}9AoPt6 zGn@ny^)?|P3UHD@e~ju`t4$LCFB=m)&UIZm43NqGsjD!i0T(Ru(Uo_E&=C@Oh_7OAKHtXL#Jx21(keWCwDCqFFVgKJA$jvFY0~*0wo=**Te|$h9)8DlB3E42v%*Mx$i8W~t z2@j$htV;TzuOBX^^uaSji$ko}Jg_2^K3YzS43q0cchtkb?3Qm|J&A}P9pyAZHwNOd z{*$lKKOGDKmj>b5X(I#gp={#X{^}7`-BgYoD4A=ZdOz9Xy)dZ95Bj2YdNVH3aX{Kd zvvAh-)E`JEdM3+P*5Qrm_KTiqO%3J_lI~Ltr#rM(c&5buK3fkm4sp_GMzbEX5m!IS zre_L%yYulOQ|+re(r97US-3W_uu13Y+S*_$_fGHc^q22|M#qo_Gz3AJS1`y4rdhcp zcDE-Zg!5KFuTMd*B~{L`qP%!NnZb;AG>KQFKkkj0Vo)}|Yus7@a{*c&e;DHTV6&h;yU`uj90@*w;DDO+$t2H?0TF^aG`l?>FT7l+Y0k3;6T+)>eq#2>4?<_&S>j6HdiOq|o!88s8P-8_} zKLKLH|1lI(BSQkgf`Fl$dBEIGQ~N4!y4&wunLyQNIC>d4J|5qL3E7%YhFg7qeBGUV z2e9PTj#B?{=+;O|TP(n`(QH89wD?zo$&jHXV{yBXA*12*Xx=;IW6AL|Q!_IPq5s1a zBC&24TTx-pLS`HWsrtiS&oOm^3CJgk2=l`71vd~MiYZ7yJ;je%8OoddFe7S zF&$Y@@$$O&?Tr}gH6FyB{qea8z#`{~amuYIu-pj0gAm6%-y{rxjr##Qk;nNqm?-#y zzrwCSj9K*BYWAvy8UhH7u2hRLVvL@lpk%1D0~@T*4~`DJ5A|c;e-oTu;Fo^r6q*5b zLa1;c#SfTW3up&8Zw=?@HDdXg(pog8m$&Z2aS;klec#*c{lAOT3gIBQl z7Qw1>Ikl0#y1tHSc4^IH)9y=G!(W1|--7v5@h7o!p<}k5c&c z>lXlaDLL*MvJwzp%nt9A#5n9-f^`uL_FeC8z1ot52Iv_WqO3h__v=xvyx}lGr0-Ly zta8oLdBKMB&DnbS0IXE@VUAh%9>TZoN3jtfA+-7d0{x8lRn{R2&1eAZFHGd+!{=s! zbQzchPf)vs`bF1H@?((4d#*}3AAm}t(PeAA+-W1^Qr8bbW;JnoE>LF*4LBof8hg+0 zvX4Ux!>=4Ev1x3lI_F+q!Pl;})NAw~Jael8yXu2O2J$ZK30z{>5&ESFU%z=VX);tN zkEeh0*6RvlBXnfz4rXEmMGq-tvZN#Ufr_iu_X-U4A|aNA?`~>102Hh0U%2v>v6j>u zy1m5@*hydruJl<;%4!!`#YX)>0J0f5F|<5^1#=n=W&gik044HCT8K}&YSanJ7qv71 z=`=4s^@to|7Z>F?c>m&@Bgjj@djTdWq&d{+Y7lKjw0=R9c6G7wJ}|Vc?Wz8hxz{9M z(a4FJNOpET-0?YYt2$UdARTcwV05SseM%DvpiCZ>7{PMk{^BFex77FqPk# zd~is?l&iVdzkDMGyr*o;x+43}sk!hI)7;q5{Q)oLlw^Df`-^z~5Xd4)0l;%pYv$OK z$(_;q$~y6vT=&gsk*qG>w_fJzSu%@31&uH8`h~oY2HCa`H4c^Cw%Oi>t=exMh+bbD z{vPX}zsRr{$)ybeKB^1#iIhR!kgi{h?R7;&U z2K_uL`QYpgW(#R`b(1|ZTPk$+(t7S=WA_-)Uc;Wq%zbj(`CpPC*>6rbNg)^XUQIyf zbk}Vh^wm9psyO|{_hgeQcQ+GaMN9;9R$xfd*FCw|5y!4jt%cG{{g0;87sm*O36ZY* z8E)EC)bVI5E8!8{%ph`;|Kzu|6+Yq^B;;-(d-IJ3~?$f*!g z{8^s)2j;ldHWREIG)};Rx2ha+zBk8zGnkKH_KG-gFHm+KK(v&04hi>~B$s<<==4Oi zWrE%DQu1gt9^I3-xH-U_I@_Owt0ZAe9ycngPAF*Y`$Ia%KJq!wD$6-~jw}tJcOZwZJ~%>4hpq=r zPEJB^E@MuTl?vl{40|g=zXR1iiV3o@`JDB5o=Qf_=M`giIDQ`YEB+e9zY?3PkHtD> z##MbiGRg`vUrRGJ3dbu}#eoZ>^E^uNC$e8!o1+)Z9rn;Bw!?x*?kr@T-u6Z2v6tnX;+jECl!J+hZs9(&O4Rwfj-sI(k6r7@lD$E*&|uP(Eiy{o#5vpSL$IbMs+- zavz83$*03JH^ol#sGsARUI75Qw%Ec?yhRLz6@00KH#>PMM$oH0O^Ltp#X39z9~u_= z`rcsd%FY0>kHme6`OjWXDvEk6Z#J>0qWY2k02#)aZE_jgL(Sl$&)5fG0|A8g)_Xg8 zLfkx<+{T+_&q>A`lb4i4wo9$w;yq{e=G+O?(W9wS)6$A^7Om?6pl$j$@pQi; zXVu2@WQYOeCHJw-H0!w5zK-er$_eHfM$!P74n_84p6=WoS}=gfPh$X=8@W)7vBq9@ z>B!cx6F07_485$R*9?&v7K=Cnft_g{N5nu(i{W;;wSh&!3)!F5lD$ghcTIb5#TukV zO470P-g}P$@jyE(s&Nm9*n%gc;+}o%z$!P61-rJ&loVRF{yb%VP%pM+#+kl;9TQHzI)Us{ooY*(ojxF~g0ww1%i4in-j(np$lpPXlezXfI5d@$ z*uy9|BFk*JlHYTJdiv7?nNCMFc=EO?-=zHM~40YvNeJr4#E$$ z%c|*haKea9)Rj912cSY2YjC!LSgpO0HYmPN`~_&%v$L!)ojdnh$*!5 zS*P0b#2B$MqVrLF_zS?wnY;$Kzr-s8et3Zae;qn$iKl}jfU~{orHKJcb6hr=tlpah zlar=wGJmkJZ>ii&!->YSmRUi~9&zZ-WYMh$-LEGSy4sBa<=X7;R?uCVb?8pb&u6(Y z7CF^nWns}M>wc1-Y7(0#xHnw`f}|&aGI7pp_5y%E&Pq#J?6>3tLtSCH0`ImU;%7M^ZFXajQ$Sz0o$NvBkajv|YSWtM- z?l~bAyO?*(T&nRe&?-fU5#v@WryD76IQ zHG9iH0(T>~w8zXNK8d3W#(yl;m;7=M@l{TAhR@oyXuY=@Pf50GAQbj^$@LXkJ)zHS zGo2cuuXWu8fDnlJnAx8hq}Ho70oy>J4ECB}e^Cgm{r!?Rk2|_LYk|~d)FjJn)(c*s zVYLtQG_|%)VNSLhM-D8hU+Jl}xTva)q-47^zn;kkn?f0}n)7*s zvw?~13mH>*T8QuCo$;IIvvUKmu`u`$=?ykCclc)m;CU}#cS-3kCwtsN}U;J#acT)oHQ3g#375Qt<@a42_FhJJB8 zcJ#iy=;qIz3tIiA$@2Fd36FzejLE34RSE2c_lG|axORrlYZbnK54GbQI`&F zz4E*JA<(Liii(!(q-Wi907c0kJ=gtf-jJ!3@d`TVAD@i~a{i-^AZrk0L_fpomy@50s8y!FbG>1xyiz< zba|ED0&+0poVd^z#u>nCIeEMzTb`Ai6WC9H%F%C%9b9U4pb0{4GgI$$#gAxm?j+QV z`n2*Jax0tO5?>x0Gims;8@K}x8GcSTZ7qPoY8M8LEXX)bbiSDE=e}+<+!#zX8%${v z9BFW#SzVl| z=Lutm*sE#6%2sWR{hR*wJ?z?)#nN6TL-KljuDyS)S$51dUA13u0d)AgPJm2YBkJU~ z-;7_Aq__%)-Y(}hC zy?ynjq%Qj3bY8#Ib29c5pnn7outx^k%ISbA|AVZV>opawfl*gjbCXr=zF)oa;Yqco zLA=0Qt|9~BuRZo!GbAMVAzwa8wqIDDJ+m1p7$`5w0*-zWf?G^#trx?2v{{iRRqKdQ zl5@?zokhw=3$`Wt&5p|*Z_vDZBi-svJo9oi7TzC>)7d$m9-0fh^*`DWV zK*1etB^-Igy#S>-G9&5E9k;timnM5+V`^-=3@@%#_&BRsjo^qReV&=PCQ=dF;0|1+ zuJcg1c__xz`~?W-?hcooQ&$P>$OYi4LBhx?t>MTjve$0}EUTo8GF7|rd_> z&UpclH@@6k-|S)3&uc4nE}Af9BQ?ogInLslyj7q`$;-&y06+u3L zCa;0YpEToY9W&eTUbq+v0)T?dj?__>s^4&*r^q}7V+&14!*bH^G)byQcd0k4$Aq-f zlJ1fhy=uQ7W4#QF9j=8-zBLs@;gs3_$UYtUHIsG>B;cqBCVY$56#s=f;t4S!kbKbB zFJ}f%>U~a!2x~EGcrXBjTDk|?{xVBq%HF{lVfDqEZaCO^|%nXRE)j$Jolqxp{k&8DGbUPdJISqh96Vc$j<A7m0+ zX3rruRuePi0xjYBQtaVR{7nc%IP3;~@3bQxnDJb=IevtgM$1V}=CbK{E^ge^WS-e; z{IO7wtMJs{|0(jXs4B>O$c^L)J$V&142m%K)pB&#-N}7JgE>JBJ&n^%{D_o+UQyb& zfgLb#*HzyV&Zrsxkh%0}(({>#s@iRbG4GVZ&MJ$G;sqU+R4ew^$zlEwXCqQ zLT(S1#9JGqF@94(s~}s$wK(s9=XbK%Ghz8OH#@t|vFn~xfa!pxK+$ylD^NjKPF1lS z?J#{*BiJKP8BC@aLV!o1O>$`HC|bXdnqlJ^%$3*mHbr+$VZ6LZn!1b@McaIZMpm-t_k7*cASg z->IwU4}V8;Utcgfd6V7weyZPdEZg{@^+=OZ{9fiP^6N4W4d9IKEM=wIdT`&<6S_sV zOLlf%dptRJvvxBke*Y@0j`)p`-cYe8{LSQfZCs_*L^$uU>Dt;_l%1x7Y%lQt9d()+ zEUi$WJ)4hP@VZk3;&s{?)@YMaMJ;z?SYp%JbaZ9{mSih1O6lKju3)c`W6RI$HeKID zJgIJ-KcRSgP}*`Uf`Y=yBPJ~7_6 zFSy6h=6^#6QjqYVpn9|Q6qK2RzF!HN#g|wPh`h+vj;HD1Trlf5neWuFC<~ksoD=9) z)1XUp0F)zvJD;LS2;)o;|JbK4JW@NrVTG^g3(m2_DY7p98P7!M=(E=&sO2 z2r!OT;O!p{gXXs-dlJ(0Z@MtU&SsR3ZBo(hF7jf7!a)B7FDP)@7np7lbw+)lM?(_! zLu7Ek0D9xz`tU4HIAltpVsBYX3QbUp&t`kob7S6hpnBqsZF0}aYVR(0d=o585sxOB zMjA``C*=4T-u!1nE3WaZ>P3FWD>M`l(pwKIEtio<+AP|&E>_|kO9Zqnebo_pkoj7T z!RuxqA=fgdzra(@eSqpG{B1bcyeh+P;yX)Q3MNTbTcduV!G+Hk#n1O9VTul(vm=#e zaG}Fhno84!;y)y-7t0}Va_GqeYiX^!MgD1Cr~Rd7Q6lQos6yeKpORG3@p<5L_Z>*S zO@&U3-rdenSLkaUx7?g1@7UEz#dG(F;ZA9S7~}fbY%jS!oG7p&iaGgI|2IlC5p>@k z4RSvpY^|L@PRRif(rQ%1Z=Sbxn_~&N3aPsnrZiLJ*g}5NZqLC7x|1orFYLNcdXe;{ zc81$-$`92S-g1!g&@K|>-vFa&9dOp&^-R1;F;G!C+_ow`n9uk;^JEX_aC#?%-0fhV znJcFZQxgxW{pnL+vfx&Y4#4sWFes6xJ9h6CYL>2IdAjdej5R#JzkMJ60ReLz<&r^cfLpYVsx#2W^1{epTXw{H@6mY~`?FL-<4tbEZ zhI~^z^k_H@Xd9N`_eZl9Kco5+@A>!8x@&x(W16ZspMOnczW3!au)U z1nDUh3l$4@i(bCydLD?Ts|J_79|S78s@EW)6v$%|kTqj-!mE2Kxm~nXr9Az40c5Ir zg$c%^?f=^oa1sK@Hi~GKwp#(ObpkA>=GyCKAcD(OoNyqnQs5dPGD@jsGh zv@`ou#BtUP-uW%t!G&gjn;1B<2r<7cFaL@wgmrsQ9~`cFUE|brhmBckolL*+zP{P` z+lV0M`pQF{D{0Aekt8;Z2cSD7^VbJ>KUpsS^=qt^e`toDP#*}Y0u6tpUH7=+-K~!L*9T}xw>ND23*I79 z9(=IZpdoZ(iyiI)oUMyy?EzR6v*k`#po6YrUzfM=O&(p3&-@=4< z?m83F6Fkx68^7*xK_+E-t9NIsZx+HxB$yP^g#ZPX`gKJ``0eG6GGk(u)dEZz0Rv4+ z+Xi&Bh^0fxn_EF;p;8*3#cTmAW|N?tQZ zB?>{uM`!_m|BlnJkvRAM37M(3LX_V8R!*I6HaiEVqNO>v+36X(bo%;ua0H+UHz9+Y zpvTpqi~z*8Q!qP?ZHlhz2j_3T{9ak`poVR*6q|z0L z0eg;oFm*9I3&tm)vNQuGlu0%~yhpjEYJw+F)tcLX6Yp7GIt}vH0IHlTz{KOvv5*4; zTWej&7YIbbIz_*rJ!Wy!`1e`uO`RV#4ETe1@=+p=t z!8FR@8{?1W_ko?8mrTWG49v+wnivnVq$)H`49LJp*QGNb|XUXcwY;#QVFrLatX1%epjDe z7kDMF9+{A!`5`eqCZ-Z7RPFx&n3};`nfa%FVK^?IJ+!pU#Xhn;5{}vteqt33k*uDo zbOlV?hubc@bHhOOAbL=&^vZ}md2*u-2DPk0#b4P#Yq>tEbR&CNP5eOwq6_JdXOXpB z{-%R8CYD^1Tk;pXK1`#aw_#;sy*-^UfJ(MRs{fD&C>N@XrJR?N#x!iBit#E-e@-%G zUlcIYm`6gX)q@|umjU}T9NQ|}xgWG}rZiI%6MqfB(gZIYYHqQYyRm6hpZsSXjo|D| z5NNJl4Gu^yEr z{hf)D7S8@^J^KF5h`e~N%>HW7WDf+)%}cQT%Xf@Rhii zT#yc6=48++c0Ybm=>4g3Iff7*ASIjDBdOYf;-FOR_h4XXk&#wFgr5leG!Dql1K;LI zReuA(+)O4LiTJG}?vHJ$@`PojB5?&Ek5MR@wQ6=#L9asMg`g40zA9+-MMnVn0;qt2 zM3xbd0SATRqlcN_zh^-ksnRM<69f#w0H9!xHL{E-vs7s%vydTN#SOrM(HVU8T*hx)(XiL}k(I{1>?=jIN z_C&!?L0RI1hC0?bx2Dppv&Dba?) z(*UrQt_duH@q~Sol>hsAQpZLQU8+ifO$ABU`_rvc6%18uKx)OFthef3?d3M`95~g~ zcXy7V@BLm#v{J}?YgK%-)Q+A2yE+TsyUT3wsY<;+BuXOWl@F3RYbl>VvcvI%w{X-< zHV=?2NR}Mb+5@27w+0ZjqAYCSBkZcscs!oWP%WbflDCa)fztH8(e90OAiULP_^trP z6qNT@nmG#NNC?xX4!y3CE=qhtHnksl7^9m@HypT2jNgmEe;DPK3VStDXodjoDihP1 zWm7llVMxb#Uw>b%^-Pn=@p^U3&3|`V#{9A}4bwFyuGljkKjt>n@Ad@X1bw>{1E`R@ z2Sl|tBV0`E!{5X$9vk@$<#E3({%A#fcGo8--~UOjU(V6#*aZM9?vF9!mgXEhSe_E% z;&&!&r>e%5(Jy^(3gy6I$y2t-TV1~YKld*`_G0t0(_evqJRkO&Pf`(oIJ+SHpRb^N zd+YO-3ByB$>h0TR{PPD(hjL*^&Hw!x$~Kw3JVL5)$b9xNpaFv#e26Hr5~AvAh>qlK zCdnvDx9!Yy^%NfPYomi#3kuQV{<1Uu&0JOE$2=D2`2V1U0p3pyh5ll9fr5$ZZ@@)s zBE>Ds>IO$Nt^n#Bc!Mj1p5Jg!MH4%wdd50A5Ua9kG2a6Qt z-z3K?GFCgBF{n;NK*Y7f0QUXA=h3_rUG{=1Ve~EK&PX^51~kDgVho%r=x z&If{Jjsix`r?XR)1d6ic$0XdhPrCvCQ>0x)mu4GQj?JB9&J!!j&h&U~r!X*<0HqCQ zzs`9>Q$nAZ$SU{XE2{a+*o1`QOG7$M0{)``(~$%N=znLzvAxz+mNQiT4wy$o z9&Lt6)k3hsBY)`G9_U=I&)7YC#aRdZDLujwLTCR#S#aOGRr*5xbqC_`M-qTibxAI|0Vxzz8S2 zV#@g81?j)HZ)f&m=ocUpsT}l5O{!U$g5f{|h-TMx2)sx)LHDQl@|mHw%K5^#&zmeH zCPU6L6-4hR1pfa(rAfDe39secNk=p;jBG;FfB*@x`kpx3z#B;xnQ{-P?; zVleQ>H3J0D3>dI%LT|6#qcIGfH3GVnEZwDl89ijrh+YST6me3>ctSBC zl*}`;?=JpSBKhdvt`A;$0X;{LA9#a=#IP2TVy2Z*wFkzS8m7{%6?ruYyL#zSF(w32 zLGrjhKV#WAe_U%iYuG;1!PJt<+|{#9(Rpsd_`e@fHk|_5+VHcSvx>C%YL`Z`3sJ(G z-+_u+g_O1Uixs!qPnBQ)3x9ZuaUVqjULx(^1d18O(w16r%-MDHZOFOxHR?yd1aWouU5nQyQW#%-P(Ll+^m+N(fWsz?&pu<_KWdpYU zYipqGPazl|=idgNiP>B>Gjt4e-CbQ>_t?D#`^>j&!nKsb2!JwUWR|r`ny+!zX<7i; zIA1*kZSA38BOqI4qFrQ+AzC3__0%gRq6{)QTL<}GhQwB`i72EhLGH_f!LG|^?3+}+i20(LY zX=#C4cS$}^N<1N9Mcxc#f8Fx|mD2u78OmamNUpLNDxa>dBHbW~wEs9gD^^x&>hc)b zgb}+_q=1Dff)Xg%n+agz>awH-JZgF8-}`ncU$f+`x}8L;s><#$N(!I|8j%0Tid}p%& z5E!BaFA**0OJJ%h^O5Ae2NIY}nl}Iz(qyp6UC{Fs%!3bVnfF)iU>DoTb}dqlNY|Bd zfYOMZenuwrvXPt7Z}Dmi6v5|vPAaAPUO=NAA}A8oH(AQ#ay^;Zj((1qX>>Xt1g@qm zN~gK%i3m~$t?uyDPL8_e#!ry8{po6_i4wBpT5GA+4s%YdP42~C&>PUk9jSNHyw=EO zhhZ*}>;NX=^7`Ie)V^qto<5)6cJ zJ+)|AdItt{DvS?>ckW)iYQ_BqkPA{3EKchK)AcLD*|D~IjSKH=Jknt|1Sf{$XA!_}GoSa8dQPDLZ6lwuNp=tzRCXw);9lc;tB}8}{2C|Vv*23%q9-Yxo`*V?Gz)h>~? zh0sl~n`~z@`!vg^uk$KsM==1E;yn^MjF`E!WxikV%ZN zAMsP<6fMhc^ezCDaaM>@%Cy3u&}1`AAiAr5G2fx=2^zd7t>_3cAqYnAQbIa3A}B1b z>h7MhnLkQJ%4dvB9YM&!dvMB?RaqICuaPU*|Bt4xjEb^r+m?_JknWb2MnXEJLs}XM zX^`$gR0O2Mp}SkUOG>(=ySqDP`1bv-@A>I+xfqA*+WS0@B%sDC57xvxX!76BHE*kU zcRnAkRwDPe{M~wk>NQ8PVbLk!`9_rZBd`wQf*gV5r)p2CQ*{CpiVP)0`sipWrfAvS zR^6Co-%8rTfak2*5)!%SKQ;2`6bWs9UPfLTBj+Y)`2eFF^7n_)C5FCj56^3d=0%2+ z2lT$>!<2d_E6;0356I0l8VMF21m*vvvyED>owo*7l8A1LAp62O;pZ8tv`^~}4M++@vZUxyv`{f!Wje}O~Wl@hq z<`{)mpDQcn4DByPFMstEPqDVAy@iV>^=p0MDRUGRGGJ%}A{PDg1DCq!D1@~8Bb}Fy}A}OwZSsXL-yTPrNQ=e}Q>e7%oUmxxTetRf+>^Fqn zV{MNZukP(B@NP}?gi!}v$M7IX9WORoji#?$UggVZTHmB{IX~`Cv94)cBPB?OIfV{o_U25xj2W5JdP8r!m8+E@mjr?X9VZ9 zoJ;zc{`#*K-ys3>gBiPyB3lAymgdL1i=9a;)uk32I`x65xB`CbS@~lRho1m@6o+2| z!yA=FI<3WBmPyEHDpWvvwo!AY^IMzS;1Vay*@mzsi{#DJMjIMM8jnM<9%p=toIF7ntK{8g^0EF&9NxjTPi(} z$V@>k=<|JDucBDHSS9tc^^RCa|9UZ)n;Srn4Ssic>M~(OWwH?zDyXI|^%#}8p2;Wy zg71&jZd+Q;pSZjgf4$abfUz#FTM(o^3$E2~(^;dCPe z4!zJ%aCUr8dTZ(vH0wukHY#POP&78t&Y^Elg+=yO8;gQfd#ll9>)v*@P)%GrmN?6_ z^Q#iHlq(8ls#mk{eHQu#BdMX`DkPl8mmPL5Y(I;3Vg3G_#|BiZNMj4&fbp-jlmE*8 zaXmI{+zp`ezEDjPS$f1$*4}jUxm!4kJKu^+#hutto{Ry?Vqou1=h=u+H?DVV)qAt^ zY+mIHmFRM!uqUk*Z=tp?*$d*=GC8)>HG(nNX`|Ev*9PE6S&ICD_P}63A0kmIU<3}TKKFED9^-dql@@z$AGY<~xoU61x;dxwn#_+YrAMHjIIL$6CMFLd&xmF#9m z_-y}Kuhezr50fJ!`JS%A14f@FTl~}W^7hV#?mrFdBR@yk^S5BHaD%p~9toXhltGzE zPo3M<4sjO#;+h29uK9K_5fO1{oT zp}u@%zr&ZC#~o_#i#xsE9j99jHOlcBru`PoNZ$?CfZf)p*8gLB3x|IXCB5`zuL?Bo z{f3BC3+u-9EiHa|<`TOwqT9$Y>lDk$=Ec)=_!mFkAB8prX!5VvrS)x4wn_5V!-ed4m zKSy>L0uy6+#^b1{U_ah71;U82hv`mcZZq7SQ z=*-e_(=LxZw;O3FH!9^dYN5$LzugUUxj}^5O>J;AyiT(h5T!0;Bk#E=3BLq zV;6qwb4hZXGogY%ry-pgM$Ea~9&iKy_HfsFPRy=T>eu*UTh2z3MW@s&f~0bfaL8}IgGUxFKEZrZiEi=9r%PM66h2%q03 zFg%y9G>k-jC#QYSqhH|BN;GVM1G@Aar>d@~vUZsF*-0%aSQ@vMn%qis+Yu1(GjP;) zI!=}wRiaKN;|pjz^L9<w_8=f&IH)muALoenHXE8WPyNzpZ}O{usfe(OT8Pf>;z|iJh(4CYA>;5=v4Gq+gN05P;=*H3WU z@DTp`yY%h?si2=&Wc*d3DlT{ZaRtO2sbS$iF)^BdvFbYBpHWZ{XjSVt$QyNT#TT%| znw3c+1>X{$4!$Qw?Ac9Yd0hIkJo;6hVb-yI;E8!AG^0l z^sCHNje_O~QnT?daUTM5V&VkO46C#Co?o@bz&{JP>=A7H)ed;0>&`m#U$I_FTu0oN z>-1qpHx(xlC80u1{-@#j4!RU5w8>&`9au^jcMCkyd_0f|5m4yx7RMwnv&iH=il-(tLIpx z7-n2Qs3O@|r_A*xloQx+1##${lxOQy$S;guG~G^^ey%L#U8&=G8f5mqW;-g#_+-C$ znUJ^x2CA`k>VVYgOi0&|Imy~geuY39{}HN3jZw3Qz!T@u@Vz2x$(w*=$P-+YTo(>! zSPnRWfp>a`he3pQ;M^wOTI<3pAWDE{SwmBU$xrxrH4Em)BrsHJ)LaZwPtDttR&A1- zVos-9yu&O96TSy6S#*CA8=H8LcUpE;0)+nQ&V6l(j-^kkevt+U{Vbon$$@yEH*^_P zNcwj&1x4dC*|&=k@u;+X@dGbUe{ABiSUel)v(TTBD1A}Xt^hpa7I!n1cxt=#>o9O? zrQ0jBpL21m@!0XGNfe-_HW8vkwy(?^Bzig@lxoA@&ww8-){4^5SpT_ol2-Xff$roc zm19t!elSx(w+?Q$l&6G9hq)?-uRD{q|1LxaO8ZhhXPk^Z)p_4%r!L*Ust9F6 zuy1t+;qkHihhX_yJ*xLn@L@HmYY7O&^K=~a4M+V-ql&yq7AYa05UxM=x`&xorj`1y zLH4@$W?u-w$Q-**~8^giJ>8{{_#!eB7t%8l_r&ahSH50X{hyBRm zoeQ-qTax8v*yz12#ay=u78U@q0|0rKwMP>SVyI3&7O9Bas_Ar`$d@Eo zL#bZ#eF}FDS_LWHtFFD3t&)U%*jpryknDd&2Gp@>6RhpTTqh-R%Pm5Ac!b|ozetQE zX_r8RT@FJmOfQH>FgUb}l?ux3t3gtWEEYNJtYEj{p!G(KS+`sSfy?nvB8b6^pqx!> zu5YK{-`oibE7mTP%q4kiinFndlczXU99?}pFvS{{0DnDahKf!7;q;k8=lBswm((0|7;V& zPI<`f#sfxTHU1dnMw5s0yt81I6{n?FF^7(zH7)}(?fIucA$ktw-~h)wWIvsFVN8=qN0LjYRzX}nrex|PyK6OPYv<> zy=($LtjNg7pKX+cgrtN-9tS$N?7fSbLA@1xffRGC0VjHuek?T1=gj=u-=13A_v@!( zFQm%L%MVi{j*qFLXF(|KOZ&>Hk^m>rc~pc`l02WQ*Wk}|?{Rqq!udl^Pk)CP90o@b zvuGiEc{nUSYno>0?o-PUvpghmbbKv+n3K4>yHKA)N0+nfjM?mS^g=h2H3a0ZfS5+a z!@bLmRp99nbJ^q){+%LF4u;zYaBBZe5unap0xVFY$C2%iuD5eD*^TW_ZC}^kch>$* z@Mh=nynY*!k=DHBajp9%NiAI@s{mD0X*WqpE!JR7z}XQ;G8Nancg7~g68@T1wd}v& zbbPpN=$3M=U&HuAX-v!-za5ey3Np&`Uu^1C)-h%bit!)n9r+gTehntP83{OGDm<;y zBcoRwCae7d*_{AQtMqc;h+a~O{QkJ%6Zr>!GeZ(^w6u^{ul~Sq&DjxLw=Wg$BZeP+ z-yki3TQ6d|^e$}=v#DUg)6pslrEKWQdwKH&^8Gef>0KB>&~1(K8zlevh|Elh4Ru1-uC_z8Tan4uTXs`k0$ z;8*+A(Sq6plYg03rJ01pdo3;GkB7`1QI>Ol?D~?=EDXaW9*nI3QBvZmzJz-nD-!Wg7K^WJ_ zlJFv)-L|Y_R;C{>2Q5`0#tU?op6JCI+wm8>!K!o|dUYh_w1(A=pocM|BPhpPP*3)* z#$ka9Zzr5obF-Kt;K9Gqd-b(s*fu--R@1G}25J)+?@}3M29zAgnCAe4#KpzE`nY$i zn81PMe6Y9mTx9*2ipnD{+{bJ4x$IP{-RRLmJuZn76h7e3>ghS(Inu;ur`y-v``4w- z_G(~M3?_9xe#CB2OGXzv#>~e~IBP~J?3TSTm77Z+at6wWb~CQZ7KsKkR>(=Fz$1;N zaA*B-@z$xqi7*v9#+K6t!Q5fjuR-G(j#Y?@jjeK-ayorG<8*rgK)YWe)NEtT=HVva z|1jhLTj@XwC-tfAT+RZy$H5`mSL?4oqJAaRvgwj2$UxemY0xPx0R`E>Hz~Huwp)EK zr}&Egv<824MVe2Xd!`zS2XCQp=3h|eR8vwR)La@99!4QX8XFxABU)~C4gS*Td2_tZ z(C%?GG1VKD7$6HB|MTwWt(UJ-ZxR$*=l;Oo__{BE)9|hPxlCv*!@KDMr^*lGW|*U- zvFfiH3xJe8ns6=Iy#(ZN-$HKTU{6MZhJTIpvQ_-KM4#h!tV*9#2XP##)F+b}j23S(N2&Q!}wbUG>}Z$m?hjrTfbTJPuu^2;)652go_BUM#N{8eH9 z!vYlli<<>;u=QtjyX}5+F2C(!zGtYUz%@MC@vcVOh?XKFiS`c>AyU%P&2IbmJ3fz7 z)L%E*!e6bVK2LDqpqSI{QaEh31K^X<@AOirn#ge@s~5jgEO^h_qr}zE8#FaYG`i!hw`duh?JN>Q7%|0HBp@{V1e z1XflY!SBIA824)rH{V3-$w-)XitVt6_9SS|5{-i~sqQC-8u^rMtti1&e|2+{5Y9v^ z2Hz|K#lLaYRwEYIY=`YSWkP0D$IGo)B9Vaf1<|$+Dbq(GBj4p@CBUvb9LF@u83v6c z6Kwj<(p(BBK@+A@5`54w>~dqv^?l_Jg8JUX^aJCXNsAZynMtJvVN{EE&m##BZ1kM& zNuPjmH~6Ie*@eE_#k5K$^}UeSX+pJfS}UU@4)Pl9&BGhSO(1Yv{Z~xa&aj`G^`a){ z5U<=L9&JyGR&jhC4lgsPfvJJQ!I3$;uGu-0!pFt&8^Ua}{`#$n}nb%<$PsU$Nl9{O?vr+Ig z;ipkhg#|jdJl@r#OJ9Lh6ylo-omw&k3G$?LAftGBgouuQ_UpC6vOAA@vj=6R8y`f- zI4nD}ozv?vaZQkckAarEQiRQ$GtWPG{YcmiGYV}5unGTRDQ0d{U#o_9-G|P5<~#XP zyV19bDu7UJhdOPmj0)w2BG2*C3)XiIFZKjTsHvSQ9>s!Q5P_Q=9TOAt1!lg9BHNEs z5f|`hv;DLd?+ZlO;lfQl&(-RR?+w5GB^kY{EL2@UK;V5UX9QHv^lV0N!Q}ViFySV+ z8^-^6cML%Q_3DxWLSxP{0!Bbwx^fkkxjD6v6I6X55cc4wf^IHhRlCKfb}Ry_3I3~R z)1R*{EIdsC16YIe#{Se`LWLpxserM;vg!yr;rl;Jb)#GX zf8L3CJ$3!JvIGB-%AASQQj5pN{1mG$XfsyjmGXyh@7f>^c<$veq88NsKIj$*~gr`HHWQ2s?8&H-F+@r%w#y=NU zqX06Ud#;p`V7R^hw@IapH`T+qm-^#D*p)rM1w=`C^rZOR2^Q+}=ftdvEXIZJJBL>2 z9sEe%KpQLmY@xjRSrH{LhTSTxM1YG+xE#$Jd#RRq2jg*>R_LBM!MZ^)a0gj zKT{V`6f*KjPF8lY#Thmqaj?UgYA((qi8+tOP_JZSm)LKpJx$8ICZ1s>_6jkL5_UNW z55ge)dRxij>Fmxziu0Z)bMv4=pwg?ElG2}*?ePA}?#a02^Yx)kc04$JzC(0mk8x|* ze0_X$iZp(~^$bP5AG>)iS|PB3T2`x(bja;0f_iw&mkr$BR0+a=fB4b!)?^Z0g1^oW zvwiwRLs`%S&eMQ?yKyKx`LH-=!qViP9mwhgm#XwrNNu&;yRo>FXPL;mfI#_&UeEFQ?(X6#Q;>bhWUHp;OfyfB zaQztpr?DZMRHnth+X;Em4ujDu!;Xq)-HqD$J=c3l)XA2-_07$*@cl}^i|JtiLf1L1 z1X?4cy{{i&*`o*UIi~Bw<;IVbC|)92Sy?XBr)8)$Pdp<@?eKfo#aVP2jNOsxa?P14 zWjPFDPUG2!n`K}$5&!8+e_&m1RG)9MqK1fw7)taV%CS_eO+M)2=JpQ+uGXE*`hebv zMk`{j!NPYuuXmZ=z9lLA;eU3{kjr$QuqPvM8qZ2d_L_)^Yol^Y-$GN@C8-YkRKFYE zGS*hzKHXiVEc~!H$+Tw_8j&I(aUIr5l;AAb%we}RTzId3?vlXnWJF1WU0}{(kP=d8 z=THwm8_vrP1+`c7zrK^(8R+JT#{5Jzqxodco_i4p6WiBqbkTOr-~sRv4~gO5l84Fs zoqutcohg9^<)(tP+-H{CUA9|i<|RJw|EOtcM;!+k>)(XE!-?%e9i= zR|gM!Cge|szfGwSPA~iMO;5%Gl0vT92w9;z<%T__@zb|MH#eN)1vkF_KJD$EC+Typ zpSpW{@wiN0Qi`|_TrM_VSTB${IyzSH**@y$%nw{lE2-V^HsuKMjq?3?jK39)CMQyf% zCQ{-Hv@w_Y6tOAmQ14vTw@aOwW_lzyBHn?J_4SRAS3@gw+C6&?KBaY%IUdo5pUQ?4 zbhTQnP#Op=Xy8udqIKb*RhxBYx=_!6hXgyS z#iu*dG)Z-kYn?(IG893sdnJ4U$nr|Fxnd5ZCVVcyT&4>-{DiPe%i#jFDUBOeunuQH z!-+$(dBQ9+H;!uQc)PJqV9dT}!SYStbfOP!Jq^MOxET>{TJ?g9?#aoE zC1Zb`GQ$E-Av0u2tyn&jw>XS@da`qqMfBBXabMhXZ704r{SBaM%@tjErtxPw=v-<4 zU%9Z=wV#QJ$-(R6%T~*5sCd>WXM9kUp35iqo+USMY6rexc+Mir{&;gPo;jijQX<;? z(yPvi_E8j?v)C9A*tU{R2{C59))%wej1ka=F|AXqYU-#afN>4r{zaM z*XrL&=2_2+!r>dF`4Y5;}(RS3bX?1@cCt0~?aD>pq2Mj}QwnS}H zi^h8BD__)06gKj6+Nr>i?{Z@N`lh_ayELKAK8a})`eE=5Zyrix1$t08=7A;7X8cG>{|gmO!3d8n1KZ)J zf{`P!C%=vbr>lt*>+8d_A5l>!w`VIMLAaEJTc7SVQhKZ?xov)BRAS4xXsk!mB3&OY z!R`uRt*sXa1PXWOgu4>3FnGq~Fzz$d>yv|tJXi;Uw6X7zJ8XU@rJ}E4#~C!eiVBl} zsmI)m>REO8w##5%>BiU^(Ql{a_9qF;UUUD7ca`rQS{keCVe4TCPOW#55Ya-bC;@*z zzg8BZSG7q@EA$7kZ=g(pIwqga>X)b$9&R-lCW3`9{E60CCsNWK2 zq0aGmY3FFk*WC*H590|EjvM+N{-lsNErW+o~n zh_B>x*3sWsoBqK=)B?a`flQpLv&*laNZoHvf45u2V~FlNU_Yk5loE2^2fDh8D$4=9 zGSa)L!T`9SFG$yczvkrRL>+(K+`NN871EvC9U9j=nkLW;bbv?mH5J-Q_XV*opr&VX z%1-w`Ue$POHS61D1*G-{T;xezmzv_4Vn-x~z1 z#}%a;@1IsvhrG3wWZhed7zW)P_YMICB?Mm28k=et%R$_-thBUcCD8yl^;en@ihtH( z+ByL?eR8AtF(w^?<5fSJS5Z=T>6`ZCugc3iB>WXhpk)BLu^9RAcWVl!e}Klt#&$SA zHAu_>Pl=8`-Bmx_8cZmi?hO}8wRft6G|)5BhcNgxJ8%3LD`@_bfe3m2D>U=b598Se z!61(!kP*1_e!Kku0jIk8QWGCN>#D`|h8cMlQ4&lBCS4uQ$bb|jpL~m3$uv`80{o3U zFbp&_5CCR#r0+iPi*;vht&^Ku{+dnJ`^5MJ$gpFb+&1oOzr->y08iD)3Aec{uG}jy z=mLIZ&jjL>(Hk#Yk-)B7{)Jln4OT$A1p9QG!vnYhMIoumo1ej?a4$z}t5q?V`7wuV z$}ZlU?~krKGJDI+1FVN}={wgpTaFq%th=Wt6kK!3n2kR)&v!;E01udnlUtza7_@|J zB|>za!?oeeCcaVmwod!5CuWE|?ZoWCO zY={pj)36L86??5+jJ)ir&g6M4;|db`?)Nt~&I5v~E z@9^=hD$7CbVy&bRQYeQJBNrF*20ALr^9n#F<<-BtHR;aDrW<}z@;bdbV~AALF#7`? zPT8gA-jONNDU)pk|2Ce57*Atha#GVd?C$oy)_UBf>&MkVw$Q^UDfsSO%0DiC`yUpt z)2~@%wxvkO`UA>gAXHP+9)NyU6fA|BOJv zeK3y=w@qQu>Z+HSWXtJz{X)@RWRm=XQrLq(;D-V|M(fk6HL$& zL~K6%{m8jusi>2#8O(~xjm)uMUgT{L>%Sf&zPrB%8iB+T=}$CJ%lPx#eth{<_k(Gp z77N_|7hsS~YxF)~gjKhd=~h5$i>Hcb3bm_!h9C2H;{Se&WPm2O`?z=$+@QHAFNu=9 zwP$q>4UL#Ha>EV;% zBVDD+tzbv3e4s^qU|zM;(A%Z`8K?KPirNms!e40&dH?yn_PuPO72ZB_EB~z$tB!S* zao?)#RG~oFiI?)nWow|v(x)kY_h6xQIXp75N~eZ&htxah5<^i|oR7yPnukg`U~7QT zbXnwHzetMVKv#gqegB{T>8jXR`vw&vLWvIVyfqiWBc>(iL{pbHrB zgxnARl^e-WPqwz;^G79==jPkG-&FcRmRw_wq?EnY z5EiLB+g4US0iW#PQA#*PuB_x!IC^&gXL`WddY-9>pSb^hi<-s6L)x;RxEY+(nM&BF zo9Ec_AwC_uS(^G6+|Nj;|d7Qd&2BFzV00#}gBlF6RE(LC(nhMf{)^c|b7^OWcL|2-1_JZUH zQLEM4CySvTuCDlQ)|>v51@5d`g@0$B>!p+0YVM|;nfkPq07YCd_VG-iUf8qUiM)j- z@#%D)f)$*_Xq7tyr>kPkdMbpjVIJF!3{@9KMg}OSuJMGzjiV@_O&Cd{KBN-u(nqa6 zGL3y(AA;aK+44M`!BQAZp{0xT?;B<0K14^_@KE|zT`8N}SLSy|YvMq3-5aRg& z00n5C5YD90v9U95m+I|w%II)@;_6wsfH3(1aQu;J46Lj?#f{!~-dJBR>H(tL8YKK8 zyf4-YnI**-;@R(u3wMDTHbV&CP8`hK#A%o4GBPlr1I)1?;XV)v1wnWl;4Y-1MwU=@ zkJQufK92Hom(fjd8UJE2r~@(;+B~U%WWgiA;URmZ|J^^l#7|d`5tso$s|!tm=!P;2@)RXBZ&s`ui0muv>;8e{gLR=G06{0U(f+Vi~{nCLyIH ztybIku9a1%@I$ed_507C-8wfwD9sWv!fLVL9akn^VihC%S?>pJssBLp5Ds4(MpAPL+TT>TMg4)h?##ZFTma~%{jOp(hpcJurl5^Z` zmVyEgkA#GRgk*Kcpo5H@60yvnp6vJtM7|Pp7&2=WMq7k|St!-nk>G|?y-qnPI(lq% zHM-+V2OgZkB?%&vW?b)GO$4|n8q+$*1w}P<4-x2A;qHQ&deZ-`z!h(u`tJ|>zIWBuqKlin^oMV%`O@!nVuthJ^0TIHVKV3V+DgG}ri~?yiKD(nG zTG~AkJn*M$Z_%SFX>bn~URz8H{cO9xRTQ~PTL{_}C*^L(i_Tx#BP_C>npu6lK^4V%Ei8(~=pd^NLr+i7c$@7F|FZlRs zjI;e7NC%(A7Q21@Dz)r$K&hVqDJw83RSJgI`ZNbTbsjCm2skb`xU(M4*OsiU)pw^R z8jv;=3~)WWrrg}UtKR9L(`fjtZboeEv{++Mf6Rb{h>`N+Rrc9fU^nPSUUM`zGymf2 z^WYJm84Prqy?p^TP&q}f-KmnBn4Dlv!|1cpk-%>#lhU%m2n;I7ytbG6_mru(-~Sed za0!WFVuL}m7nC{Oy6-7G?Db{*VpiwjVvfeu{`?tain*($2}yR)GcXan)_U&}@Vwga z`WOZ<(umzLUXwPz@FqYgLHTHulL(yU*R-@FP;+<(@9d`3J6rk zBl0{iS}lvPGBdGZ=CUkAG-4JWx7N2UerHZsNKi{O5gG!DKzJzh2Qh95Z@oJ?4VNM_ ztysTe=hx3qR;1sYEIlul4Hc~2D_)ryNxzr`fd$X8b}28_#P1eY+L2?>=?0m%_G?Sj z$JN~t#Bp(-_-N7R^0zOpdD71+TjJeV(i%P zI_VUx+$a9T z@Nu)o-aY&dj$D-#KBO0p_;Gj)o-N9IJc9{O>31Isl1TVmcpZMSXa>-rt=CxR-Dw%7 zwKWMnUd*{0d2F68+G~Z)DUiLk$^3|3f4W^p{>h=B`vOeX#YU|@!*8~&wqFR<#1pgX z+yD7~a^BE|&$KB&n(sh`NYK8S|#HMH>3m=Wl62TFd1-GW3NyU$O<*&tbggBc3a21b(Xxzna|1 zF{#C3d$B@2hUc}PB|&BE?>~`7And6-$;lx_0W>WZBuyu_n?ek?*N`cXRvKMXtozE- z7WQO*Ca`>KkYU1WR@(-%?dxd!6+>A|APa8W&{ZP_P;d zqH(y|rlzOAvrM#0bg=LidX$LO(g;XCGBPm$aetX=+I=h)P^L1f+-Eth+@J%_*~h?z zSm}6AdkK4%sF^R{oqtHrgi4hWvSy%i4RcAcqt zP5;*HiAy2cEmg7q#^QYRHFoK1r{GB)QJRcW{gL6}Fb`OcwTV0>WJ#Pi(6&yK?TV^( zf^`KLlu}Rx2p2~EC%ab4+7GHB**NfXAS6f>eJD^d;Q!g^^H63?-SX59$e`HE6?Cg} zbNK*6O#d~fupa0z`9i&eNuvv)to`HG0Aumz$H79@wxRXeaXM6ND(l5t_mI(7L7>;# zT*u4n_E@wTj4TAi#8$(I)C2???GPA{A}3e>oBc<4^`G7`4-xseh$q)Tt?Bvj_jljB zqU0CqDCg^re%UTIdydXJ=Y-~`iuhM~S8qr2A5NE{rXlIwJ-Ae& zV&Yu5jontR=oA~0Rz;nmJA>{ZTvtMJ@@D&1v*Q5VkbI2i)OGA%w;a@ETLHz; zxK@8xKZ032SmvDxHpOh;4e(Cf0e{1%r3yDK6D(>yAiL+Y(@Dxuro*IsQcJ#Uy&^WW z&yb6xV#HMbG4b$x+z7-ry8ql8iN6$^$csflsD>mr`;Lw{EY$ZTjiLxmWab!}h>QF1 zw2WV1Wl|5Gf6drY`neWS0oQeJ{WXs`C@#El)~{JpUH|2FZ@!lN=Q2eiar?N7bSY3>1~N0U3> z{)3}PE={#d-xrCi4|XL6Uv}e%MMMTb5=DM>WYY3EF7m#5W#2bK3I1cRva>-lkTNhD zydZk5^|!)wGJkl!_5e*?OMkZh>4a6c0u9AuYhbHbdzq071R*~5_YuB*w)sCSAmeDU zabsdD3^xXMZS5m2?pkLjE#OwOE0&twD|E_G5UM?v$ae*)%}IFWiFDXM(x5cBbk z{IWOs;D0aoPtwW2e)cxO!s~T)c$IqG+3Wydp#B0*QD-%&x}b~hPj>bz_Ow>hPN|-s zgbp)x{s%fd>8G@Fa|%uWD00OdCi9OP(9Hodh&_E0VP>W(<)$XEf0(U^im8BoWeLyu92F$BJ@})QT}d?sw{?BWd3D zBe#s{2W$1{WO;%BuGU`$0#1IyHd0TVD5o2^VfY%5?aLy%zQ~um;y7lb070e|dVOAk z+Xak0!^`DAT@VJp-vWPDq-7st?A)kF*nU-b+d19t6YU33r`{pPsm9sE4jt`s+mKj5 z&UJfTs+8cEC#A?;XQxSJD?66HqS^g8R%diMDAn)3C%#(k?MmcQO@jzy*Qud}} zyVIxn>8vyE-c~(Q2)ZPeVGCH1`oG!>9{DbRkCuL5&m#5u8sHkuPL3)DD-jA!5!+xo$-^I22QAsMUc=Fs={6aBBm#*dUqjD)j zO5xpcm}Sk~z_NzwkoN_RbOxUt7Af`edp0JQYy6JXdYiM5(uS z4wOf*n@7Qzuglzb@P|-0jdv$rrCY8$!(x8BG{VJtuAhqsJSq9yZQT=}2jb!s^~PiVk|lZv_TixGkyTm2|Q!KatEloQQm)o4_n3C zg;!EvHI&vGFLuB-@Kst${!_!b`mup*7SOg=lBVE&TgG+I@Gd(gxr9wd&QFO-4w_XAdoX zOVV1Gqx)L$m;Tp##4-W!`35)uLpnMrXg4Zz{X`f4&FL<*R#*HubWyZ@ar!0Xh{yjlr{{PCgo{P(hyqefio$ZU@k(CzlYr+p!98t7g)?K0W{kOJ9I66tW}l zeNT`G{2t(|>Xg1`Vq~bXURJbB4W9u*KEN`Acxp_%sBc!+8*kkAH}%AaH5bKz&Tw)g z=?a5rh<$fwCf~s~Et2Kt>T(bN8wz$b;j$AebOK4#?3NjLUedCCfNda_p`F9%nw2Ye zvOpD+Y~rcpBi+%M*(D}5j~mW`UH~a3Ilsj~)MVi3f#zM4S3f5(xWr;Al(6L(=Bqq# zv#>OJRxIDAVPCpGT(fKDs|3nzp^-+Jo8_iWju>~0s==(VHo=eA{DvYOFy*_y8Tus_ z0cRAPvjBtVk&I6Lo`>`-A);k}ggKpFkOxdX`#p03BYrna-Nf60N#nN@sp`Eh_*<5} zaR+x5D4pjCoW`o~wFe+*Z{_{;dh~sE{-8UX0d3N+?>{gLWMkfDi2L&IH9J`ir{Asj zD4B|Y*6*FsC^EjU7h}fJ)T^#jWgtiyt91k8hO=N12gqVK)E*G!zHMszO4Su&?){B$ zSKOg(q$B1A?ZW=0b8pIE5v0b-`8__OZp z1C*MHDT)k6=*MgQ@ghHhP8O?RMTJlBqK9pd#Jr04bY3Z@@nRIy`**+9l3GJ6qdo}sz30G!!wL8OxZd3l<^Wop;$?=a~e{Mqo)JUt3k53LlLN9b5>mKv; zcnWAnmV7;zFq;Lh`u4W_%B_)FwQIZ&@=4zmEVa7IIdWzWf+$9q?dDjcD-qFDIo$^Z z6_q?sc2Fh1Rs5 zB>B??JYyuS`WgJQHyW6;uu1uMfrZ#*Lp}P8GUk8yr9KQghGkR;{ zeO+rg12PWwj6G<^)a5XNuU{a*;qYyiYA5iJjpcy>$K&Pnx?r&BuG+m1U13gVqa=3J-^chueGZM`>ui{oLbV1x8>$c|)S5 z{tbP7HdnO?BUWPJQ}8(-n8M({1D@K{>Du_C3b$%M0FA`yseueNP(@>Ka5ct|$*nPhnNcOe2UyIy~ z&RriF0Ph*3;APs9=;Br%-ESy_i$cuLYotZb(@_rpT{xwHPv3|G>>i>4#8X#tV^tL7 z zSENx63BMF%d)`o4>F3Y9>i;*b!{g~etTKZCD-G7i689gdOsc!!7&KxVN&ckbXGkF# zeHa;Y0ekN6jrdZ#Vbv+eedjbr`m*At*?(6TRp>%mc{KR91u`GZNpIUv zTq(dmHT7|(_X;2dkLXN%Eo)OxvT(az{Cy+&?MU0zpl%~Oa|tKNr1%pa1Mou`*2t>^ zZG{sxwYa#r3H1uoWgPUJ{tmwM@Ra(r^KUQiYa?pJI_|kKp%fFxHgF0&`CR)|+r(`H zh7G}q8+HY6^>s@G%fIbG*8w5IVm42%=-||%{#)033t$WPz1m;Oc}NL*L5O(#ZeX$i z-2UTE67pGnX8&ePw95^zHrg_@N{o$;xwqQ#^F)O~Y4l@|D_Gxgk3O_RC}F;>&XXW= zWx7bS?;PW!mlnunA?AcPTQ1~FYs|?6jzx0WO^1WUIp9*goxg0M*oOOFac7yB-MAY9 zMM56r%T-Ca26-irP&|eynRf>E3z20?2^>1->?{9-bL+FW+wQ>bKLf9Y9-4^ynkYCi};gbtHPat0zAxTmYJ)c1%oKfw@<@WMw_1Ua1t;ygJvcURf z`TuvWpTniliq`1{fVl$neG{X#l;;Z=NvMTAEP%mn!%Kk9?qA_lor-X0QNXds-eXsK zYx7-t4=++IiUhe`bO&3AQ`^!-MtHv(-&lY7%Om0cb{!e?r!QY5z&7hgE`*IkOic{5 zdgq&=z={zXg2uG(brDgogs@qwpb!dFFRysO0@8}dfJi@VW=6#ktK2r4D;*q}Cp`## zO{me5*j`^>Uywjdd1s*0!_++~2t7SL1)netwsY9%Y86UZWKsc%%OAD_a-uvJZLhMw zg{K<;`V#f=dWdw82>``CI@O=bqNAhndXxxQIcQ3hz(p1o;PwPZnrDcj6!*EmWXUGa z%9C2Cb1qO%s|a(`?rnPVVf?Tdm#w zoy0!_+^V2`Nj09wYwSHXHnu`FAXCQ;oMb33@&08RcQpZAvq%FgyD2I>Jd;kMDeAKB z9ms}I53V)|i+8^c`TKdY$aixrGHoH$xj{ zU!cy(QoCCFM>*MI%AJxdfZK|Ri7hX;aT>#kFSr;LuZaD7#!`6rif1x&2&hfwG5j9= z{94XId`6luKmT^;f3{idlS#?_Zx3J--aTS6BYr{5p>?PuS`i zhm&!-y5?WK8`IV+vl1!hdHnx+x(;Zz-uG|s5hL~rLByUls)X1x_NFQ}B}Q9R?GmFz zY>Lp@n;KpAh-#}9wc090YqZ*`YD@dS^!xjtbI-Xs=bqgAzVEX?@AE!Sx$TXgfE#Yx z6_X7dk^KGhSIo6*K=$Vg)};v`Q%^`zQUVI3nyw#|LR?bE;u_ZK$l&Y+Z2BGTwg|B5H6;DE zj-YEAk>wd4J?E=FB<$YK?DLQ`GWb+n6AXtH^Z}Vf*#m)_39h3O2igRF}LuvQB=403b$l^OuQBfeh`1B$u=dx9K$wF`D z{_%=oWRYWznuTS++_;JIY$~?4^;K?P;5#n3YTbI<`=LNp5&)#Nm`DT~6R)@a3fSC@ zuZ%;EvT@?$bv{n8i<^STirPH%9>UIEolZTyzPXnxW@jU|EwVi!v(lZsqoaBy(*%tX z1u`c+0l*i?c>sv4wD|=n608iIu*@*Y7(7sZ3!tAsIfmS00+0s2r`0X-Z>Z@kUYk5r zES9tHkuJT#f;^nTk7#|m-N>7zeAThxm%eSzNINw50RSCvvf0HUGn?t{)%CqAZ$8<# zdm)0{TPvTm-Tev_bvmOC0)(V}pK}nVQaqxlUF=b)f}gZ2)Z1PBlXxMdoaae<$mn_f zCr5u|eGO~!DsyeyR}AuO?126I^WGQU?tJ_6p=|=Qv-ANJ+y{>SHrHUg_3UrN@8|=- zF%=UXPEOW8&iq}pFOTZJ`X#z$nTGl%NTH#j0jOSEdm-bw#{%BzJx82b3-SB6tPQ76 zlN&k0kitv-LTC4_a?G@&eDYHbJ;-$7D*@5r?rYN_px8pci;Q$~H9jhmYwtdNc~x0< z{^5CzZ&wfSQB%dr(2|?JZNYuhg+az?746xpUx5wU(|x}GW}2;=4I}V4_OA5P;YN;t z-mGgJ93DY)#Sv)ZzfqOE+?vF$tRG5-X(hoj)7>lQ)4FB-{%&I}8WSS@iI@?GwM|C)|qxu7FI@qT|*xKXl9HN|s=9+PCTG3CAuEc(4b3=o! zLA9+D#ZLE`en0qYiNSO>yTH$3&$tfP!_y_|S-&dLS;B?)iGPnMl<|9aw}*jJyhejN z234#T8(#n?r5npV&*~p?6gONqo9;C5gqK+0fAa}~zkl31VP{{X|Br)N=F&x75^=Kv zGl`VHGr6SsfW(_)%l+OP^BtA_bIlSnec$k*BfRI>6?IsJ&hw}rWiBl6{QGYbY#0V) zGP#deiwx5M^2zeTntBOkVM=Fi-T26Q1TzVzxT^M$RP{>h;bXoFFGiA89iKc8;Zye0 z?&&bMusDf08PJIE9lL-P#~yz>2MM@!HaqVdo$xC#wxZjb4eI$md0cZ(%OY|Wtd+3E zFprhL6}_XtZgzIN-0@aAENQ&p{mtKAW&ryT6nx)M1ATt{*!d11VdzXu^LSt zBMV)gBd3gYm1En5z2)f1GI{ym=Ms0{f`Cq}mFm~xfTS=WRqwsWtuPzbe;SPCk2R=+ zvZOPM2#(%{P@#7;NcFmjfmQ_sYe^d{cJb)w=-%a5)zK|Wq{de=4&AW4Va*U8LGZQx z{h^cJf8X5jdR*JV9EYl>ur@j)TKenB#+7RCrVmXV^s0O9^t7`{zVGfPUR-5@42weh z>;M=$6D<4b)9s1Zuf+h*`IFG6YnTla3o{FIb5nD3)0skntzR_eUwu^AGnpe!NX*hHV%cj3@bnTZnT+z0!Zzb`?2 zr-oZ*&vFK260%a4gi-vvA_!g6wAr?9mRq6eY>6y;La=BG9ENYsA^G5pLp;Hwcf5nb zd5gMsFi%&4+{QP``(O1nPbuZeuiEuLkL%?c$S z!E4N8&!~|b%oK>slbOgCNuLiBHoKgh81PI@|4cxpXm~bmkcPrV2*RNbf>pKcFE7w& z7Jwi7)Vr~v- z);8yPnc8dxoBvTVHDm+KmDbtUF?_@qu#rzFVj**RU03Y ziPxm-*GOzBD? zzns6(D8~)eI&63^7xWYkvSamp2!%#H>>7M1?%TSY9z~f$9d~LAU)~a~?ip(rm=hdiJ^;vm_Aq}W>idsT1kz18{O?wJsGLYbf!yD@dD&SF zpr2i6*INlYA#R`?_aeE#WyqOPXNtD5zGuiONtqW+29R=>%+%y;%@Qe}umys8C#pqo zAz?M3DAp=khrAuC5428fp%VU6`zG(D4bb)wF9=MZAoUSZ$op2vBXti6Lf?kKEll{w zN^M&GOeA5j(;4M9n=-(gup2G$L|sBM#Y;u^YPq>;wVOB;bSr6H;goNtAcGt!Uz=8a zL~sugQ!t$E;5#6FmM-bNKvqI+)$TUc%_QTex+>GKj+$t@PGIk75L?R4Fg~In@pbIZ zyghMVkIAQGk%w}Rmqa>3(Yq0(4(;4TIh+}6TD6ZCiKkOtYj?Wv^uQQo8u>ia<;{R* z3m-5Ljke|8Uiv?qfTV~JR;|ytsGNF?HLq7r%=jslcnw_=rOxR9!%ediZcduT=;jLK zlmWd1Dun$Q5h0uNKmu2fY{%2hu1oh8ixeBicL|O3dxYMl!b*+ zDXv>Q^XevP)<}$NP6CB$LpGCPsC%FGvXeC$JKZO9uUz>DA8-=J?^BOpVFm+}^B2&y zpoP}$?r3;&Z`b7Wr>Oo9hYK+kD}hlydvBj^^D_Wr6JunI7Cw z(KJ7HuLZZMi_s8S~Gw#TKA)1UPM|{R0%&`4y6kPQQbYsg%&w5O^2&YXNaA z72rGf4f565)m?wF_LuPiS41Fx^_$R&NRATm8pHBwU6Aomf6IJq-)IhAHhMFv-GO0 z#x^jHiUNAu!y$dXxWc$tFL0Igg~0OVYcJiLC3G4p>_Wu9qD=N+BZk&)SfUNUedP(_ z7XF1W4b^7vlj6cM8BYJZEkZc;tjTSu0zb^NIW>d8iW#0RJ}*k*kI%6Z$S9dk zlwbF4YylcSm{J8wt?Ohb1Q1ZFsY(2zHdyQUejjjuk4O}RMhzIe+BhiIgcW z?~7Q22+8_Fqq!E`nS1OGupBvQn|juw%Hgn;E+6o(KyVVDiv9D&I_xFvd6LoEYOC7; zs$+Z0q0TV(KK+~IkK#Qw>LKYOjchym`zy+)KY_hD3hY>=w>^jomAWGH5IFimZzV%~ zp2N4#okmCC>Uee@f9uq}U!;G9UL!=a1sSLzm`fU<{6c%CPdE!3}4U4#{gYGREA z1b8;Y{{4JXb3tV2rIE{4X^MGLxNZUY)>>x*9>2Tu3t%JbBY!1Q;2SFp)Y^(-yO z04ztX@R?W)t(iYp>Td)Q_gZxOb7)+XNDO0l&>2&yM8 zf#S2E8#Ty0O}u}LpuGd2%W<+Ok=InEQ}$OWW;m$^jX{kqxR;7eK4sfEQs+nt@DtHU z&0akm>Vlf*v#bPmD#R{@=Y)mb6JA0eCjtb$SyceSukH#bE$|xmLY2CgLa8XUI2yd@ zhftzP3zX^6cw=U2aat@uDPuJ<7!~z^s5&PSbe|mWzkJ~1#BK3R$rSUR@`HP%1Gdr4 zK*a_-ES??=xDWd-Ajh~l^=Dtn@Tz!B!0PY&7Y-+75bqUhH{SP$?*dkYjU=XbvhtOmT!ME)tT@B2Ic4<$ zWs;BLvE1aR(yX3^I>2lQF!VE6Is4Kj`7mb*nH4v^r_9g#%Ju{qC<`gh;zS5A{t?$p zjOH}TIOT4#OX{=@!)jvcEJzTo(aTwpcYeI$E>N~qJm?MnPK>fwb(Phr%q8MapOs^V|4Z~hdMlAFQo)k`GV+_S=$i5lk&!<8JRhV4E> zf@TB|bf)ij)&%ZhO$`X8vcA`%3Vz>Kc3&KleLo2=$YSsoQx_pa0OtpCZu&SnJYZ-e z2L`4a$)288Bi>B3Fn%i)dQ3A_jc$^3~laLAplWY~Q{f=n36G=OpPU&?@~#haMukq}Yl8 zbQbvhmXGy@a6il9g|TgTJA|x*AP()*u4zRt`acO3)}OB#e{I{(EZ!uyPv-W<>0x{eE(>c6gxpk$m64(n9c3FHS z6O%K90X+J`t*y}??nqg>7$-*YDMt_*O>ER%bL^LwbFQ`~SvMPZ=a&&fLPBoosXb4J z2b_}?uZip9C{_D*JEf|M6VXXdLx>v&c>w&rR+~Qan~hKcGXJoSNU`CcZ%`X5tQ+~NYGZYydnF2He|Qlgc)^`2n*f1me2^(TK45EeFRmw5~n{h zsv>lAd_iXqh|<~D%`CmcOSV6%;O8r@hc))|xc%fiUq3Wy z(4#939a@SU_)t|Hzw}I$uNw3NlB)-*)796m({>!{sTqiASG1LS@p9f=C!@iVT-^Z7 z9fdPPyGG`2!+*Fh8y|$-sdQ~w%9h)a;RPf2ExCzCY|sgj24TrHZkq4o02?q^OMHZ4 zk<|b*3*+5Ib~B&iP>A5eN_kih7rOvV)L3nl&Z)_F?&i8aU|q?NQI$^M3pk;pMj!+z z7g|jiyBH5ThEKz$q5-#l7IwXU1vA#v97N(F3kjBij31u~XHxx_&vc5gAi0^1n(b%O zA+`~tLd&ewomewO%62SnYC-8QLzqyVCu-5db=as-Oh|W;t5V@v(gqI3hSZ9_9)hRMJYjl-HP$$Xl@SBhW!N|DgTgDuWtp0kbJ_KGQwFK-P>Bc^xYDiUrfpyy$zQRz zpb3DPJKTpevAW@}b?GdPw^eEPX!ixqRU>!4dFEq=^pbfI1yZ#jm?BAjtiF>Sy=LP~ z18R{P@jyh;-k5%aP>o*&?*(Kmp%5E&kZ-O8;|~TvQlQGB^?a!Yerwl-x@k+-5f>_G z2sA0c4giVvEbe}c@4#O$sUk^(q~e%U@91K+x!88SdX|qJQ`udgTIiU(GhLGLMJgVj zpGP2hUt=|z3-C*GFQN;QO?m-W&yF+hJb(H68tyh4zc0b@P8^9Mpfam6FZ-&Dyh-_` z$2<*g6BYd(WLxB4%}HH)>UL7^yr}5hgyMHbo!fV1u2^LZaB*mHQ0~I9EmH!))@u&gyeBgb%aoW2E7G8{?ORHDGb~ae+?gFC*)Dca*4UYMj^t z*F8%(`fi|E`X4SAeCrvtw|afo!WbUq9mcw{x6OSkM!7>h@y$V|Eh2VFGM-sHmDtD; z)u7)0R#QZ_;UZ&@le>;1W;qcR3YrR*>D%207?{TdO81TX?bTO60Z3U;*;;$yehAbo zJw4O{L2s!cT3Y3Fc?x7+J;wo5x*4K-nYtvS^{!y@op?=>6TgeXt9mEM&_y7eT=MdH zMDmHNf9Y@C$>1T;b&fdh9xSNjwhVUQRW1TlqyMdy%*?!R+rh=1S-A28vKVge z5YC`ezGzg;MUVGTEoIu{?H#{JbdRx+ONUw=CE8w-AY2 z8Mvq(Ul!ZX`>PWb04G);fsq`ORiF+ecmPdr^7WY%X|so*EnTw5}Sl8q`{hUZ83LQf|*uyf7M z){$>HAl2`SMDpAIvbr_3 z-RwS;fFqfy1!ry-(P_DTij-CKdH>j6o=Tj|o$y|wy8ss`u2S%O{sPYKrcH@U(Z?id z-RjX-5uM^TNaQAT+N7|VBCxxY*iRX1+{&w<=1$qJ|Ag4M;OE(?U_8;#+9ItwQ(xH> zED0z?^DezwB4E%lOj%{Mx&0STx~pPhUz(Ck(%JLsYF)?Tp|atRSP;1awEF`AoJEE{ zMx7OancSgm1`@2@l+TkD^XK5ER4ex%DOyALMW%(OW!d6+GV4~9sEiN4a+k2SM#(0b3-GvXRVkBpK!nzw)7 zLjJsw;a+|Iolg6h-;VtZ^{gV>p#h~V$yu@*{gVw zSg<9}fB1Jg*bX#H^0ZdFTqr9U*l(3%_PsLMYU@?}F* zO?Kp**X1z-Q@+L)ZKzOM#8k*fRMBI`6*WhJWN)4QlNY0s=PjK>>fh*uS-ramDt=)5 zZI7bI$Ge5<7NXUkIaGqns&L}2h-Fz&PyEnAH8wNwA*_9DP9QwwhvUe0;Gz3Sbnc22 zJ_|c&)nSzWfz@Wr;S4zdrcKZoq^izRK^xP+c)>%y#S_+8LOS410{%D^A~IF+1*L-# z5-2k`Fc?{l1OT2Y-gaw_OVk^=K)l|mO{qPp)TKzX#OPpW?Y<~+FvxOII@HP8$oj;2 z=mI6F{0)O9USeZAdoSX3msJLk)mw|BNs&U5hw9zYt{KS0Keua*SaqFJh+M)7!W(}` zP1`IR3-}cf-5ox=aj^1*8b>IHTp3~^hwTUV!Wj+?EWqDMq}`fd+b(Jf}e9j?DuMcqMw#^;D(X!D7Qv*v&``L+yYX2|7y1Sr^wfX$m+TCc)fH#7ik=o81 zm^S`gAeLp8Zv;6KkLkw6nixh0pB`c>f_^*PDh6L3V*#U`xA%~VH?$oBgdSYK*A&9E ztxtN!^oWsVL&Mbpn6zK?G+g3C-C0DR@j4$gA-QdLmYR?H3|4{F~btmQhq zK6eR>*sH&ODU3^b^|ofI-$dJ5r0ZP!K$3o22o%QK!_9ODMt-cqwZ-&rT%bD2{8Y{J zr%${eLsS%AvA^Ne0J2%n1dDEMP#2eK8l$_LTGmw7H00G@;s;%ts$}v#@-Z1?KZHgqoy>2*k#b?!jlGJW#Ao{<^XFZGvCt{*(0rFOV$vG1Ay z$;dm9Ul`6h`X0;pRxXyCbE0kG7NhOMsqgCqeN!95@=mq7_=1_++XJ#!8FgLsPohbm z@iEHOUp@p%aIv-<)idcjgZqbIO~WTy2?B^C-d6j}09V1Q68=4pw^?&%00>u*vW%QM zObPVAg~^pM3h%Tor;jAFR@&xSVA@MiLfPg2r;iS4^HY7rJOCi)!J?I3ioG^60H@ub zI>;R0A$J8l0*RBIwUl7!=vGWyP~IlCv|g2t@v598skAlg)1TL$D-Xs#i$jFR&%hm12RvHWi$dpK3B{!$7t9WYAOis- zt-H&SRjM|(4$^;OcYA~wbMrmCg1F}l3%GmHz>7ZCm7#oLBH?0N)qwZb(j$_B?@TXY zf-9w)H zKV6*_#v2t+H4&9NbbHkpx4QaoYS7;lY)PF%o;1Glr$E|+66A1MDL&}#T|m1To*_I? z>CPPr?XZvqUk1RKuLAL3{gvWW|3~sfKIdEdtU4PDeyXzwt;VY5?QmX{XMF+h1Hv!% z$$5x9v{!VRj6J!PRI$^N8SRZi#~&=DNVvYzVyAW6z{z&kY-_7$mgrxj5%RoGT_d63 zw?c3r@eTnSezYQLb*z>2D#Y$HoDPV_sm83b0UUFY51+M3HmY(btfW413q*VKpUKLh zzcfeNtQRIwoP4UI_w7_@lvmg@OJ=hysp9?PQTfOwX zqL(7(s-4_hvsa_hKRd>!(v6;Xp%F?nhEZn0fye4nEA#owU%lfKS|&oStvU~>MdaaB zx=udKCNBs&hq-29zB}@dIjRjYe_0--mSN~wG~$LRLPu$AG6h+E8aQ@e6*3xAgdMIAb9yKpPhu^e)Xn@Hyyt@ibeGY)KZz_ouM0D|mQWpg zN-jdJag=Ej>EsJ6gI76=7j@K5qK&SDje}tRyuUv)uE*5BjBjA&aS{Y}2zWcFl&K^E z*X)!Z1z5l9SwCS;E(XA7?=*<;hNo0m`ak3D?Pte;=ucH+G=4e`1giYVGGwIjeOeix zqHl}^KRaZA##qh9_)XuG<**s3Y{6nC$$k^@CGB7^bf_u7FXUE(gyikgd{5^}+;UsS zq_|$8a4eT;>5w!$jQT+AV@kc5eq%8xQnF`9cog>dLOJh&`9~n3oZm-X2;hJ4G z*7mO%kz${R>6exT+-Jnh2oSq$?NDJpq|39Rq`yEb$-jTgjTXNF?XqZ20RT!eA{`;( U-~rM3w1bw3fhDq4&nxNw0Nqx*00000 From f7e2f8cb5e78ead27dc5419e819dcff478b2e5a6 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:33:21 -0500 Subject: [PATCH 28/72] Add custom history JSONL import --- crates/ctx-cli/src/docs.rs | 9 + crates/ctx-cli/src/main.rs | 299 +++- crates/ctx-cli/tests/cli.rs | 116 ++ crates/ctx-history-capture/src/lib.rs | 1361 ++++++++++++++++- crates/ctx-history-core/src/history_jsonl.rs | 213 +++ crates/ctx-history-core/src/lib.rs | 3 + crates/ctx-history-store/src/lib.rs | 34 +- docs/cli-reference.md | 10 +- docs/custom-history-import-format.md | 228 +++ docs/providers.md | 11 + docs/storage.md | 13 +- .../fixtures/custom-history-jsonl/basic.jsonl | 8 + .../malformed-partial.jsonl | 5 + 13 files changed, 2256 insertions(+), 54 deletions(-) create mode 100644 crates/ctx-history-core/src/history_jsonl.rs create mode 100644 docs/custom-history-import-format.md create mode 100644 tests/fixtures/custom-history-jsonl/basic.jsonl create mode 100644 tests/fixtures/custom-history-jsonl/malformed-partial.jsonl diff --git a/crates/ctx-cli/src/docs.rs b/crates/ctx-cli/src/docs.rs index c40bdc7e0..74b0637b4 100644 --- a/crates/ctx-cli/src/docs.rs +++ b/crates/ctx-cli/src/docs.rs @@ -214,6 +214,15 @@ const TOPICS: &[DocTopic] = &[ source_path: "docs/providers.md", body: include_str!("../../../docs/providers.md"), }, + DocTopic { + id: "custom-history-import-format", + title: "Custom History Import Format", + audience: "integrator-agent", + summary: "ctx-history-jsonl-v1 records, transport, identity, cursors, and import rules.", + tags: &["providers", "imports", "jsonl", "custom"], + source_path: "docs/custom-history-import-format.md", + body: include_str!("../../../docs/custom-history-import-format.md"), + }, DocTopic { id: "provider-support", title: "Provider Support", diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index fc30932ae..c10a804c2 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -30,17 +30,18 @@ use ctx_history_capture::{ import_antigravity_cli_history, import_astrbot_sqlite, import_claude_projects_jsonl_tree, import_codex_history_jsonl, import_codex_session_jsonl, import_codex_session_jsonl_tail, import_codex_session_paths, import_codex_session_tree, import_copilot_cli_session_events, - import_cursor_native_history, import_factory_ai_droid_sessions, import_gemini_cli_history, - import_hermes_sqlite, import_nanoclaw_project, import_openclaw_history, import_opencode_sqlite, + import_cursor_native_history, import_custom_history_jsonl_v1, + import_factory_ai_droid_sessions, import_gemini_cli_history, import_hermes_sqlite, + import_nanoclaw_project, import_openclaw_history, import_opencode_sqlite, import_pi_session_jsonl, provider_source_for_path, provider_source_spec, stable_capture_uuid, - AntigravityCliImportOptions, AstrBotSqliteImportOptions, CatalogSummary, - ClaudeProjectsImportOptions, CodexEventImportMode, CodexHistoryImportOptions, + validate_custom_history_jsonl_v1, AntigravityCliImportOptions, AstrBotSqliteImportOptions, + CatalogSummary, ClaudeProjectsImportOptions, CodexEventImportMode, CodexHistoryImportOptions, CodexSessionCatalogOptions, CodexSessionImportOptions, CodexSessionImportProgress, CodexSessionImportProgressCallback, CodexToolOutputMode, CopilotCliImportOptions, - CursorNativeImportOptions, FactoryAiDroidImportOptions, GeminiCliImportOptions, - HermesSqliteImportOptions, NanoClawImportOptions, OpenClawImportOptions, - OpenCodeSqliteImportOptions, PiSessionImportOptions, ProviderImportSummary, - ProviderImportSupport, ProviderSource, ProviderSourceStatus, + CursorNativeImportOptions, CustomHistoryJsonlV1ImportOptions, FactoryAiDroidImportOptions, + GeminiCliImportOptions, HermesSqliteImportOptions, NanoClawImportOptions, + OpenClawImportOptions, OpenCodeSqliteImportOptions, PiSessionImportOptions, + ProviderImportSummary, ProviderImportSupport, ProviderSource, ProviderSourceStatus, }; use ctx_history_core::{ database_path, default_data_root, utc_now, CaptureProvider, ContextCitation, @@ -123,10 +124,17 @@ struct DoctorArgs { #[derive(Debug, Args)] struct ImportArgs { #[arg(long, value_enum)] - provider: Option, + provider: Option, #[arg(long)] path: Option, - #[arg(long, conflicts_with_all = ["provider", "path"])] + #[arg( + long, + value_enum, + requires = "path", + conflicts_with_all = ["provider", "all"] + )] + format: Option, + #[arg(long, conflicts_with_all = ["provider", "path", "format"])] all: bool, #[arg(long)] resume: bool, @@ -498,6 +506,36 @@ impl OutputFormat { } } +#[derive(Debug, Clone, Copy, ValueEnum)] +enum NativeProviderArg { + Codex, + Pi, + #[value(alias = "claude-code")] + Claude, + #[value(name = "opencode", alias = "open-code")] + OpenCode, + #[value(alias = "antigravity-cli")] + Antigravity, + #[value(alias = "gemini-cli")] + Gemini, + Cursor, + #[value(alias = "copilot", alias = "copilot_cli")] + CopilotCli, + #[value( + alias = "factoryai-droid", + alias = "factory-droid", + alias = "factory_ai_droid" + )] + FactoryAiDroid, + #[value(name = "openclaw", alias = "open-claw", alias = "open_claw")] + OpenClaw, + Hermes, + #[value(name = "nanoclaw", alias = "nano-claw", alias = "nano_claw")] + NanoClaw, + #[value(name = "astrbot", alias = "astr-bot", alias = "astr_bot")] + AstrBot, +} + #[derive(Debug, Clone, Copy, ValueEnum)] enum ProviderArg { Codex, @@ -526,6 +564,21 @@ enum ProviderArg { NanoClaw, #[value(name = "astrbot", alias = "astr-bot", alias = "astr_bot")] AstrBot, + Custom, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum ImportFormatArg { + #[value(name = "ctx-history-jsonl-v1", alias = "custom-history-jsonl-v1")] + CtxHistoryJsonlV1, +} + +impl ImportFormatArg { + fn as_str(self) -> &'static str { + match self { + Self::CtxHistoryJsonlV1 => "ctx-history-jsonl-v1", + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] @@ -536,6 +589,26 @@ enum ProgressArg { None, } +impl NativeProviderArg { + fn capture_provider(self) -> CaptureProvider { + match self { + Self::Codex => CaptureProvider::Codex, + Self::Pi => CaptureProvider::Pi, + Self::Claude => CaptureProvider::Claude, + Self::OpenCode => CaptureProvider::OpenCode, + Self::Antigravity => CaptureProvider::Antigravity, + Self::Gemini => CaptureProvider::Gemini, + Self::Cursor => CaptureProvider::Cursor, + Self::CopilotCli => CaptureProvider::CopilotCli, + Self::FactoryAiDroid => CaptureProvider::FactoryAiDroid, + Self::OpenClaw => CaptureProvider::OpenClaw, + Self::Hermes => CaptureProvider::Hermes, + Self::NanoClaw => CaptureProvider::NanoClaw, + Self::AstrBot => CaptureProvider::AstrBot, + } + } +} + impl ProviderArg { fn capture_provider(self) -> CaptureProvider { match self { @@ -552,6 +625,7 @@ impl ProviderArg { Self::Hermes => CaptureProvider::Hermes, Self::NanoClaw => CaptureProvider::NanoClaw, Self::AstrBot => CaptureProvider::AstrBot, + Self::Custom => CaptureProvider::Custom, } } @@ -570,6 +644,7 @@ impl ProviderArg { Self::Hermes => "hermes", Self::NanoClaw => "nanoclaw", Self::AstrBot => "astrbot", + Self::Custom => "custom", } } } @@ -1222,7 +1297,9 @@ fn command_analytics_properties(command: &CommandRoot) -> AnalyticsProperties { analytics::insert_str( &mut properties, "source_mode", - if args.path.is_some() { + if args.format.is_some() { + "explicit_format" + } else if args.path.is_some() { "explicit_path" } else if args.all { "all_discovered" @@ -1393,6 +1470,7 @@ fn run_setup( let import_args = ImportArgs { provider: None, path: None, + format: None, all: true, resume: false, json: args.json, @@ -1771,6 +1849,17 @@ fn run_import_internal( let mut totals = ImportTotals::default(); let mut imported_sources = Vec::new(); + if let Some(format) = args.format { + return run_explicit_format_import( + args, + format, + db_path, + store, + analytics_properties, + options, + ); + } + let requests = import_requests(args)?; if requests.is_empty() { if options.allow_empty_sources { @@ -2105,6 +2194,148 @@ fn run_import_internal( }) } +fn run_explicit_format_import( + args: &ImportArgs, + format: ImportFormatArg, + db_path: PathBuf, + mut store: Store, + analytics_properties: &mut AnalyticsProperties, + options: ImportRunOptions, +) -> Result { + let path = args + .path + .as_ref() + .context("--format requires an explicit --path")?; + let stats = + source_stats(path).with_context(|| format!("scan import source {}", path.display()))?; + analytics::insert_count_bucket(analytics_properties, "sources_seen_bucket", 1); + analytics::insert_bytes_bucket(analytics_properties, "source_bytes_bucket", stats.bytes); + + let progress = ProgressReporter::new( + options.progress, + options.json, + options.operation, + stats.bytes, + ); + progress.message( + "discovering", + format!( + "found 1 {} source, {}", + format.as_str(), + format_bytes(stats.bytes) + ), + ); + if let Some(warning) = low_disk_space_warning(&db_path, stats.bytes) { + progress.warning(warning); + } + if (stats.files >= LARGE_IMPORT_SOURCE_FILES_WARNING + || stats.bytes >= LARGE_IMPORT_SOURCE_BYTES_WARNING) + && stats.files > 0 + { + let warning = format!( + "large import: {} source file(s), {}; initial indexing may use sustained CPU and disk", + stats.files, + format_bytes(stats.bytes) + ); + progress.warning(warning); + } + + let validation = match format { + ImportFormatArg::CtxHistoryJsonlV1 => { + validate_custom_history_jsonl_v1(path).map_err(anyhow::Error::from)? + } + }; + if validation.failed > 0 { + return Err(explicit_format_import_failure(format, &validation)); + } + + let record = import_record_for_custom_history(path, format); + let record_id = record.id; + store.upsert_record(&record)?; + progress.message("indexing", format!("importing {}", format.as_str())); + let summary = match format { + ImportFormatArg::CtxHistoryJsonlV1 => import_custom_history_jsonl_v1( + path, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(path.clone()), + history_record_id: Some(record_id), + allow_partial_failures: false, + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .map_err(anyhow::Error::from)?, + }; + if summary.failed > 0 { + return Err(explicit_format_import_failure(format, &summary)); + } + + let mut totals = ImportTotals::default(); + totals.add(&summary, &stats); + if totals.imported_sessions > 0 || totals.imported_events > 0 || totals.imported_edges > 0 { + progress.message("finalizing", "optimizing search index"); + Store::open(&db_path)?.optimize_search_index()?; + } + progress.message("finalizing", "checkpointing search database"); + Store::open(&db_path)?.checkpoint_wal_truncate_if_larger_than(WAL_TRUNCATE_MIN_BYTES)?; + if options.print_human { + progress.finish_line(); + } + progress.done( + "finalizing", + format!("indexed 1 {} source file", format.as_str()), + stats.bytes, + ); + analytics::insert_count_bucket( + analytics_properties, + "source_files_bucket", + stats.files as u64, + ); + analytics::insert_count_bucket(analytics_properties, "failed_sources_bucket", 0); + analytics::insert_count_bucket( + analytics_properties, + "sessions_imported_bucket", + totals.imported_sessions as u64, + ); + analytics::insert_count_bucket( + analytics_properties, + "events_imported_bucket", + totals.imported_events as u64, + ); + analytics::insert_count_bucket( + analytics_properties, + "edges_imported_bucket", + totals.imported_edges as u64, + ); + analytics::insert_count_bucket( + analytics_properties, + "skipped_bucket", + totals.skipped as u64, + ); + analytics::insert_count_bucket(analytics_properties, "failed_bucket", totals.failed as u64); + Ok(ImportReport { + resume: args.resume, + totals, + sources: vec![custom_format_import_json(format, path, &stats, &summary)], + }) +} + +fn explicit_format_import_failure( + format: ImportFormatArg, + summary: &ProviderImportSummary, +) -> anyhow::Error { + let detail = summary + .failures + .first() + .map(|failure| format!("line {}: {}", failure.line, failure.error)) + .unwrap_or_else(|| "unknown validation failure".to_owned()); + anyhow!( + "{} import failed with {} failure(s); first failure: {detail}", + format.as_str(), + summary.failed + ) +} + fn print_import_report(report: &ImportReport, json_output: bool) -> Result<()> { if json_output { print_json(import_report_json(report)) @@ -2229,6 +2460,29 @@ fn source_import_json( }) } +fn custom_format_import_json( + format: ImportFormatArg, + path: &Path, + stats: &SourceStats, + summary: &ProviderImportSummary, +) -> Value { + json!({ + "status": "imported", + "provider": CaptureProvider::Custom.as_str(), + "path": path, + "format": format.as_str(), + "source_format": format.as_str(), + "source_files": stats.files, + "source_bytes": stats.bytes, + "imported_sessions": summary.imported_sessions, + "imported_events": summary.imported_events, + "imported_edges": summary.imported_edges, + "skipped": summary.skipped, + "failed": summary.failed, + "failures": provider_failures_json(summary), + }) +} + fn provider_failures_json(summary: &ProviderImportSummary) -> Vec { summary .failures @@ -4186,7 +4440,7 @@ fn import_requests(args: &ImportArgs) -> Result> { if let Some(path) = &args.path { let provider = args .provider - .unwrap_or(ProviderArg::Codex) + .unwrap_or(NativeProviderArg::Codex) .capture_provider(); let source = explicit_path_source(provider, path.clone()); if !source @@ -5103,6 +5357,27 @@ fn import_record_for_source(source: &SourceInfo) -> HistoryRecord { record } +fn import_record_for_custom_history(path: &Path, format: ImportFormatArg) -> HistoryRecord { + let key = format!("custom-history:{}:{}", format.as_str(), path.display()); + let mut record = HistoryRecord::new( + "custom agent history".to_owned(), + format!( + "Indexed custom agent history from {} ({})", + path.display(), + format.as_str() + ), + vec![ + "agent-history".into(), + "custom".into(), + format.as_str().into(), + ], + "agent_history", + path.parent().map(|path| path.display().to_string()), + ); + record.id = stable_capture_uuid(&key, "record"); + record +} + fn discovered_sources() -> Vec { home_dir() .as_deref() diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 9f20e19b0..48960a65d 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -32,6 +32,10 @@ fn provider_history_fixture(name: &str) -> String { materialized_fixture("provider-history", name) } +fn custom_history_fixture(name: &str) -> String { + materialized_fixture("custom-history-jsonl", name) +} + fn redaction_fixture(name: &str) -> String { materialized_fixture("redaction", name) } @@ -41,6 +45,9 @@ fn materialized_fixture(category: &str, name: &str) -> String { "provider-history" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../tests/fixtures/provider-history") .join(name), + "custom-history-jsonl" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/fixtures/custom-history-jsonl") + .join(name), "provider" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../tests/fixtures/provider") .join(name), @@ -742,6 +749,114 @@ fn import_progress_json_goes_to_stderr_without_polluting_stdout() { assert!(stderr.contains(r#""operation":"import""#), "{stderr}"); } +#[test] +fn import_custom_history_jsonl_format_is_searchable_and_idempotent() { + let temp = tempdir(); + let fixture = custom_history_fixture("basic.jsonl"); + + let first = json_output(ctx(&temp).args([ + "import", + "--format", + "ctx-history-jsonl-v1", + "--path", + &fixture, + "--json", + "--progress", + "none", + ])); + assert_eq!(first["totals"]["imported_sessions"], 2); + assert_eq!(first["totals"]["imported_events"], 2); + assert_eq!(first["totals"]["imported_edges"], 2); + assert_eq!(first["sources"][0]["provider"], "custom"); + assert_eq!(first["sources"][0]["format"], "ctx-history-jsonl-v1"); + + let search = json_output(ctx(&temp).args([ + "search", + "parser test", + "--provider", + "custom", + "--refresh", + "off", + "--json", + ])); + assert!( + !search["results"].as_array().unwrap().is_empty(), + "custom import was not searchable: {search:#}" + ); + + let second = json_output(ctx(&temp).args([ + "import", + "--format", + "ctx-history-jsonl-v1", + "--path", + &fixture, + "--json", + "--progress", + "none", + ])); + assert_eq!(second["totals"]["imported_sessions"], 0); + assert_eq!(second["totals"]["imported_events"], 0); + assert_eq!(second["totals"]["imported_edges"], 0); + assert_eq!(second["totals"]["skipped"], 6); +} + +#[test] +fn import_custom_history_jsonl_format_rejects_malformed_atomically() { + let temp = tempdir(); + let fixture = custom_history_fixture("malformed-partial.jsonl"); + + let stderr = failure_stderr(ctx(&temp).args([ + "import", + "--format", + "ctx-history-jsonl-v1", + "--path", + &fixture, + "--progress", + "none", + ])); + assert!( + stderr.contains("ctx-history-jsonl-v1 import failed"), + "{stderr}" + ); + + let status = json_output(ctx(&temp).args(["status", "--json"])); + assert_eq!(status["indexed_items"], 0); + let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); + assert_eq!( + sqlite_count(&conn, "SELECT COUNT(*) FROM history_records"), + 0 + ); + assert_eq!( + sqlite_count(&conn, "SELECT COUNT(*) FROM ctx_history_search"), + 0 + ); + assert_eq!( + sqlite_count(&conn, "SELECT COUNT(*) FROM capture_sources"), + 0 + ); + assert_eq!(sqlite_count(&conn, "SELECT COUNT(*) FROM sessions"), 0); + assert_eq!(sqlite_count(&conn, "SELECT COUNT(*) FROM events"), 0); +} + +#[test] +fn import_custom_history_format_is_not_a_native_provider_importer() { + let temp = tempdir(); + let stderr = failure_stderr(ctx(&temp).args(["import", "--provider", "custom"])); + assert!(stderr.contains("invalid value 'custom'"), "{stderr}"); + + let fixture = custom_history_fixture("basic.jsonl"); + let stderr = failure_stderr(ctx(&temp).args([ + "import", + "--format", + "ctx-history-jsonl-v1", + "--path", + &fixture, + "--all", + ])); + assert!(stderr.contains("--format"), "{stderr}"); + assert!(stderr.contains("--all"), "{stderr}"); +} + #[test] fn import_all_discovers_and_imports_providers_together() { let temp = tempdir(); @@ -1112,6 +1227,7 @@ fn public_subcommand_help_is_golden_enough_for_session_retrieval() { "--provider ", "[possible values: codex, pi, claude, opencode, antigravity, gemini, cursor, copilot-cli, factory-ai-droid, openclaw, hermes, nanoclaw, astrbot]", "--path ", + "--format ", "--resume", "--json", ], diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 638f26889..6bceead7e 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -13,13 +13,16 @@ use std::{ use chrono::{DateTime, Utc}; use ctx_history_core::{ inbox_dir as core_inbox_dir, new_id, utc_now, AgentType, CaptureEnvelope, CaptureProvider, - CaptureSource, CaptureSourceDescriptor, CaptureSourceKind, Confidence, EntityTimestamps, Event, - EventRole, EventType, Fidelity, FileChangeKind, FileTouched, HistoryRecord, - ProviderCaptureEnvelope, ProviderCursorCheckpoint, ProviderCursorRange, ProviderEventEnvelope, - ProviderRawRetention, ProviderRedactionBoundary, ProviderSessionEnvelope, - ProviderSourceEnvelope, ProviderSourceTrust, RedactionState, Run, RunStatus, RunType, Session, - SessionEdge, SessionEdgeType, SessionHistoryArchive, SessionStatus, SyncCursor, SyncMetadata, - SyncState, Visibility, PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, + CaptureSource, CaptureSourceDescriptor, CaptureSourceKind, Confidence, + CtxHistoryJsonlEdgeRecord, CtxHistoryJsonlEventRecord, CtxHistoryJsonlFileTouchRecord, + CtxHistoryJsonlRecord, CtxHistoryJsonlSessionRecord, CtxHistoryJsonlSourceRecord, + EntityTimestamps, Event, EventRole, EventType, Fidelity, FileChangeKind, FileTouched, + HistoryRecord, ProviderCaptureEnvelope, ProviderCursorCheckpoint, ProviderCursorRange, + ProviderEventEnvelope, ProviderRawRetention, ProviderRedactionBoundary, + ProviderSessionEnvelope, ProviderSourceEnvelope, ProviderSourceTrust, RedactionState, Run, + RunStatus, RunType, Session, SessionEdge, SessionEdgeType, SessionHistoryArchive, + SessionStatus, SyncCursor, SyncMetadata, SyncState, Visibility, + CTX_HISTORY_JSONL_V1_SCHEMA_VERSION, PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, }; use ctx_history_store::{CatalogSession, Store, StoreError}; use rusqlite::{Connection, OpenFlags, OptionalExtension}; @@ -204,6 +207,27 @@ impl Default for ProviderFixtureImportOptions { } } +#[derive(Debug, Clone)] +pub struct CustomHistoryJsonlV1ImportOptions { + pub machine_id: String, + pub source_path: Option, + pub imported_at: DateTime, + pub history_record_id: Option, + pub allow_partial_failures: bool, +} + +impl Default for CustomHistoryJsonlV1ImportOptions { + fn default() -> Self { + Self { + machine_id: default_machine_id(), + source_path: None, + imported_at: Utc::now(), + history_record_id: None, + allow_partial_failures: false, + } + } +} + #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ProviderImportSummary { pub imported: usize, @@ -1842,6 +1866,64 @@ pub fn import_provider_fixture_jsonl( ) } +pub fn import_custom_history_jsonl_v1( + path: impl AsRef, + store: &mut Store, + options: CustomHistoryJsonlV1ImportOptions, +) -> Result { + let path = path.as_ref(); + let source_path = options + .source_path + .clone() + .unwrap_or_else(|| path.to_path_buf()); + let normalization = normalize_custom_history_jsonl_v1( + path, + &ProviderAdapterContext { + machine_id: options.machine_id, + source_path: Some(source_path), + imported_at: options.imported_at, + tool_output_mode: CodexToolOutputMode::Full, + event_mode: CodexEventImportMode::Rich, + include_notices: true, + }, + )?; + if normalization.provider.summary.failed > 0 && !options.allow_partial_failures { + return Ok(normalization.provider.summary); + } + + let mut summary = import_normalized_provider_captures( + store, + normalization.provider, + NormalizedProviderImportOptions { + history_record_id: options.history_record_id, + allow_partial_failures: options.allow_partial_failures, + persist_cursors: true, + wrap_transaction: true, + fast_event_inserts: true, + }, + )?; + import_custom_history_edges( + store, + &normalization.edges, + options.history_record_id, + options.allow_partial_failures, + &mut summary, + )?; + Ok(summary) +} + +pub fn validate_custom_history_jsonl_v1(path: impl AsRef) -> Result { + let path = path.as_ref(); + let normalization = normalize_custom_history_jsonl_v1( + path, + &ProviderAdapterContext { + source_path: Some(path.to_path_buf()), + ..ProviderAdapterContext::default() + }, + )?; + Ok(normalization.provider.summary) +} + pub fn import_codex_history_jsonl( path: impl AsRef, store: &mut Store, @@ -3514,25 +3596,900 @@ pub fn import_normalized_provider_captures( import_provider_capture_lines(store, options, summary, captures, files_touched) } -const CODEX_SESSION_SOURCE_FORMAT: &str = "codex_session_jsonl"; -const CLAUDE_PROJECTS_SOURCE_FORMAT: &str = "claude_projects_jsonl_tree"; -const OPENCODE_SQLITE_SOURCE_FORMAT: &str = "opencode_sqlite"; -const OPENCLAW_SOURCE_FORMAT: &str = "openclaw_session_jsonl_tree"; -const HERMES_SQLITE_SOURCE_FORMAT: &str = "hermes_state_sqlite"; -const NANOCLAW_SOURCE_FORMAT: &str = "nanoclaw_project"; -const ASTRBOT_SQLITE_SOURCE_FORMAT: &str = "astrbot_data_v4_sqlite"; -const ANTIGRAVITY_CLI_SOURCE_FORMAT: &str = "antigravity_cli_transcript_jsonl_tree"; -const GEMINI_CLI_SOURCE_FORMAT: &str = "gemini_cli_chat_recording_jsonl"; -const CURSOR_AGENT_TRANSCRIPT_SOURCE_FORMAT: &str = "cursor_agent_transcript_jsonl"; -const FACTORY_DROID_SOURCE_FORMAT: &str = "factory_ai_droid_sessions_jsonl"; -const COPILOT_CLI_SOURCE_FORMAT: &str = "copilot_cli_session_events_jsonl"; -const CODEX_MAX_TEXT_CHARS: usize = 16_000; -const CODEX_MAX_METADATA_TEXT_CHARS: usize = 4_000; -const CODEX_MAX_OUTPUT_PREVIEW_CHARS: usize = 4_000; -const PROVIDER_MAX_TEXT_CHARS: usize = 16_000; -const PROVIDER_MAX_PREVIEW_CHARS: usize = 4_000; -const CODEX_FAST_IMPORT_TRANSACTION_FILES: usize = 512; -const CODEX_FAST_IMPORT_PASSIVE_CHECKPOINT_MIN_BYTES: u64 = 2 * 1024 * 1024 * 1024; +const CODEX_SESSION_SOURCE_FORMAT: &str = "codex_session_jsonl"; +const CLAUDE_PROJECTS_SOURCE_FORMAT: &str = "claude_projects_jsonl_tree"; +const OPENCODE_SQLITE_SOURCE_FORMAT: &str = "opencode_sqlite"; +const OPENCLAW_SOURCE_FORMAT: &str = "openclaw_session_jsonl_tree"; +const HERMES_SQLITE_SOURCE_FORMAT: &str = "hermes_state_sqlite"; +const NANOCLAW_SOURCE_FORMAT: &str = "nanoclaw_project"; +const ASTRBOT_SQLITE_SOURCE_FORMAT: &str = "astrbot_data_v4_sqlite"; +const ANTIGRAVITY_CLI_SOURCE_FORMAT: &str = "antigravity_cli_transcript_jsonl_tree"; +const GEMINI_CLI_SOURCE_FORMAT: &str = "gemini_cli_chat_recording_jsonl"; +const CURSOR_AGENT_TRANSCRIPT_SOURCE_FORMAT: &str = "cursor_agent_transcript_jsonl"; +const FACTORY_DROID_SOURCE_FORMAT: &str = "factory_ai_droid_sessions_jsonl"; +const COPILOT_CLI_SOURCE_FORMAT: &str = "copilot_cli_session_events_jsonl"; +const CODEX_MAX_TEXT_CHARS: usize = 16_000; +const CODEX_MAX_METADATA_TEXT_CHARS: usize = 4_000; +const CODEX_MAX_OUTPUT_PREVIEW_CHARS: usize = 4_000; +const PROVIDER_MAX_TEXT_CHARS: usize = 16_000; +const PROVIDER_MAX_PREVIEW_CHARS: usize = 4_000; +const CODEX_FAST_IMPORT_TRANSACTION_FILES: usize = 512; +const CODEX_FAST_IMPORT_PASSIVE_CHECKPOINT_MIN_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +#[derive(Debug, Clone, Default)] +struct CustomHistoryJsonlV1NormalizationResult { + provider: ProviderNormalizationResult, + edges: Vec<(usize, CustomHistoryJsonlV1EdgeImport)>, +} + +#[derive(Debug, Clone)] +struct CustomHistoryJsonlV1EdgeImport { + provider_key: String, + source_id: String, + from_provider_session_id: String, + to_provider_session_id: String, + edge_id: Option, + edge_type: SessionEdgeType, + confidence: Confidence, + occurred_at: DateTime, + fidelity: Fidelity, + metadata: Value, +} + +fn normalize_custom_history_jsonl_v1( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + ensure_regular_provider_transcript_file(path)?; + let file = File::open(path)?; + let reader = BufReader::new(file); + let mut summary = ProviderImportSummary::default(); + let mut records = Vec::new(); + + for (index, line) in reader.lines().enumerate() { + let line_number = index + 1; + let line = line?; + if line.trim().is_empty() { + continue; + } + match serde_json::from_str::(&line) { + Ok(record) => records.push((line_number, record)), + Err(err) => push_provider_import_failure(&mut summary, line_number, err.to_string()), + } + } + + if summary.failed > 0 { + return Ok(custom_history_failed_normalization(summary)); + } + + let mut manifest_line = None; + let mut sources = BTreeMap::::new(); + let mut sessions = BTreeMap::<(String, String), (usize, CtxHistoryJsonlSessionRecord)>::new(); + let mut events = Vec::<(usize, CtxHistoryJsonlEventRecord)>::new(); + let mut event_keys = BTreeSet::<(String, String, u64)>::new(); + let mut file_touches = Vec::<(usize, CtxHistoryJsonlFileTouchRecord)>::new(); + let mut touch_keys = BTreeSet::<(String, String, u64)>::new(); + let mut edges = Vec::<(usize, CtxHistoryJsonlEdgeRecord)>::new(); + let mut edge_keys = BTreeSet::<(String, String, String, String)>::new(); + + for (line_number, record) in records { + match record { + CtxHistoryJsonlRecord::Manifest(manifest) => { + if manifest.schema_version != CTX_HISTORY_JSONL_V1_SCHEMA_VERSION { + push_provider_import_failure( + &mut summary, + line_number, + format!( + "unsupported custom history schema version `{}`", + manifest.schema_version + ), + ); + } + if manifest_line.replace(line_number).is_some() { + push_provider_import_failure( + &mut summary, + line_number, + "duplicate manifest record".to_owned(), + ); + } + } + CtxHistoryJsonlRecord::Source(source) => { + validate_custom_source_record(&mut summary, line_number, &source); + if sources + .insert(source.source_id.clone(), (line_number, source)) + .is_some() + { + push_provider_import_failure( + &mut summary, + line_number, + "duplicate source_id".to_owned(), + ); + } + } + CtxHistoryJsonlRecord::Session(session) => { + validate_custom_history_identifier( + &mut summary, + line_number, + "source_id", + &session.source_id, + ); + validate_custom_history_identifier( + &mut summary, + line_number, + "session_id", + &session.session_id, + ); + let key = (session.source_id.clone(), session.session_id.clone()); + if sessions.insert(key, (line_number, session)).is_some() { + push_provider_import_failure( + &mut summary, + line_number, + "duplicate session record".to_owned(), + ); + } + } + CtxHistoryJsonlRecord::Event(event) => { + validate_custom_history_identifier( + &mut summary, + line_number, + "source_id", + &event.source_id, + ); + validate_custom_history_identifier( + &mut summary, + line_number, + "session_id", + &event.session_id, + ); + let key = ( + event.source_id.clone(), + event.session_id.clone(), + event.event_index, + ); + if !event_keys.insert(key) { + push_provider_import_failure( + &mut summary, + line_number, + "duplicate event_index for session".to_owned(), + ); + } + events.push((line_number, event)); + } + CtxHistoryJsonlRecord::FileTouch(file_touch) => { + validate_custom_history_identifier( + &mut summary, + line_number, + "source_id", + &file_touch.source_id, + ); + validate_custom_history_identifier( + &mut summary, + line_number, + "session_id", + &file_touch.session_id, + ); + if file_touch.path.trim().is_empty() { + push_provider_import_failure( + &mut summary, + line_number, + "file_touch path must not be empty".to_owned(), + ); + } + let key = ( + file_touch.source_id.clone(), + file_touch.session_id.clone(), + file_touch.touch_index, + ); + if !touch_keys.insert(key) { + push_provider_import_failure( + &mut summary, + line_number, + "duplicate touch_index for session".to_owned(), + ); + } + file_touches.push((line_number, file_touch)); + } + CtxHistoryJsonlRecord::Edge(edge) => { + validate_custom_history_identifier( + &mut summary, + line_number, + "source_id", + &edge.source_id, + ); + validate_custom_history_identifier( + &mut summary, + line_number, + "from_session_id", + &edge.from_session_id, + ); + validate_custom_history_identifier( + &mut summary, + line_number, + "to_session_id", + &edge.to_session_id, + ); + let edge_key = edge.edge_id.clone().unwrap_or_else(|| { + format!( + "{}:{}:{}", + edge.from_session_id, + edge.to_session_id, + edge.edge_type.as_str() + ) + }); + let key = ( + edge.source_id.clone(), + edge.from_session_id.clone(), + edge.to_session_id.clone(), + edge_key, + ); + if !edge_keys.insert(key) { + push_provider_import_failure( + &mut summary, + line_number, + "duplicate edge record".to_owned(), + ); + } + edges.push((line_number, edge)); + } + } + } + + validate_custom_history_references( + &mut summary, + manifest_line, + &sources, + &sessions, + &events, + &event_keys, + &file_touches, + &edges, + ); + if summary.failed > 0 { + return Ok(custom_history_failed_normalization(summary)); + } + + let mut result = ProviderNormalizationResult { + summary, + ..ProviderNormalizationResult::default() + }; + for (line_number, session) in sessions.values() { + let source = &sources + .get(&session.source_id) + .expect("session source already validated") + .1; + result.captures.push(( + *line_number, + custom_history_session_capture(source, session, None, context), + )); + } + for (line_number, event) in events { + let (_, session) = sessions + .get(&(event.source_id.clone(), event.session_id.clone())) + .expect("event session already validated"); + let source = &sources + .get(&event.source_id) + .expect("event source already validated") + .1; + let envelope = custom_history_event_envelope(source, &event); + result.captures.push(( + line_number, + custom_history_session_capture(source, session, Some(envelope), context), + )); + } + for (line_number, file_touch) in file_touches { + let source = &sources + .get(&file_touch.source_id) + .expect("file_touch source already validated") + .1; + result.files_touched.push(( + line_number, + custom_history_file_touch_envelope(source, &file_touch), + )); + } + + let mut custom_edges = Vec::new(); + for (line_number, edge) in edges { + let source = &sources + .get(&edge.source_id) + .expect("edge source already validated") + .1; + custom_edges.push(( + line_number, + custom_history_edge_import(source, &edge, context.imported_at), + )); + } + + Ok(CustomHistoryJsonlV1NormalizationResult { + provider: result, + edges: custom_edges, + }) +} + +fn custom_history_failed_normalization( + summary: ProviderImportSummary, +) -> CustomHistoryJsonlV1NormalizationResult { + CustomHistoryJsonlV1NormalizationResult { + provider: ProviderNormalizationResult { + summary, + ..ProviderNormalizationResult::default() + }, + edges: Vec::new(), + } +} + +fn push_provider_import_failure(summary: &mut ProviderImportSummary, line: usize, error: String) { + summary.failed += 1; + summary.failures.push(ProviderImportFailure { line, error }); +} + +fn validate_custom_source_record( + summary: &mut ProviderImportSummary, + line_number: usize, + source: &CtxHistoryJsonlSourceRecord, +) { + validate_custom_history_identifier(summary, line_number, "source_id", &source.source_id); + validate_custom_history_identifier( + summary, + line_number, + "source_format", + &source.source_format, + ); + let valid = !source.provider_key.is_empty() + && source.provider_key.len() <= 128 + && source.provider_key.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'.' | b'_' | b'-') + }) + && source + .provider_key + .bytes() + .next() + .is_some_and(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()); + if !valid { + push_provider_import_failure( + summary, + line_number, + "provider_key must be 1 to 128 bytes, start with a lowercase ASCII letter or digit, and use only lowercase ASCII letters, digits, '.', '_', or '-'".to_owned(), + ); + } +} + +fn validate_custom_history_identifier( + summary: &mut ProviderImportSummary, + line_number: usize, + field: &str, + value: &str, +) { + let error = if value.trim().is_empty() { + Some(format!("{field} must not be empty")) + } else if value.len() > 512 { + Some(format!("{field} must be at most 512 bytes")) + } else if value.chars().any(char::is_control) { + Some(format!("{field} must not contain control characters")) + } else { + None + }; + if let Some(error) = error { + push_provider_import_failure(summary, line_number, error); + } +} + +fn validate_custom_history_references( + summary: &mut ProviderImportSummary, + manifest_line: Option, + sources: &BTreeMap, + sessions: &BTreeMap<(String, String), (usize, CtxHistoryJsonlSessionRecord)>, + events: &[(usize, CtxHistoryJsonlEventRecord)], + event_keys: &BTreeSet<(String, String, u64)>, + file_touches: &[(usize, CtxHistoryJsonlFileTouchRecord)], + edges: &[(usize, CtxHistoryJsonlEdgeRecord)], +) { + if manifest_line.is_none() { + push_provider_import_failure( + summary, + 0, + "missing manifest record for ctx-history-jsonl-v1".to_owned(), + ); + } + + for (line_number, session) in sessions.values() { + if !sources.contains_key(&session.source_id) { + push_provider_import_failure( + summary, + *line_number, + format!( + "session references unknown source_id `{}`", + session.source_id + ), + ); + } + if let Some(parent) = &session.parent_session_id { + let key = (session.source_id.clone(), parent.clone()); + if !sessions.contains_key(&key) { + push_provider_import_failure( + summary, + *line_number, + format!("session references unknown parent_session_id `{parent}`"), + ); + } + } + if let Some(root) = &session.root_session_id { + let key = (session.source_id.clone(), root.clone()); + if root != &session.session_id && !sessions.contains_key(&key) { + push_provider_import_failure( + summary, + *line_number, + format!("session references unknown root_session_id `{root}`"), + ); + } + } + } + + for (line_number, event) in events { + if !sessions.contains_key(&(event.source_id.clone(), event.session_id.clone())) { + push_provider_import_failure( + summary, + *line_number, + format!( + "event references unknown session `{}` in source `{}`", + event.session_id, event.source_id + ), + ); + } + } + + for (line_number, file_touch) in file_touches { + if !sessions.contains_key(&(file_touch.source_id.clone(), file_touch.session_id.clone())) { + push_provider_import_failure( + summary, + *line_number, + format!( + "file_touch references unknown session `{}` in source `{}`", + file_touch.session_id, file_touch.source_id + ), + ); + } + if let Some(event_index) = file_touch.event_index { + let key = ( + file_touch.source_id.clone(), + file_touch.session_id.clone(), + event_index, + ); + if !event_keys.contains(&key) { + push_provider_import_failure( + summary, + *line_number, + format!("file_touch references unknown event_index `{event_index}`"), + ); + } + } + } + + for (line_number, edge) in edges { + let from_key = (edge.source_id.clone(), edge.from_session_id.clone()); + let to_key = (edge.source_id.clone(), edge.to_session_id.clone()); + if !sessions.contains_key(&from_key) { + push_provider_import_failure( + summary, + *line_number, + format!( + "edge references unknown from_session_id `{}`", + edge.from_session_id + ), + ); + } + if !sessions.contains_key(&to_key) { + push_provider_import_failure( + summary, + *line_number, + format!( + "edge references unknown to_session_id `{}`", + edge.to_session_id + ), + ); + } + if edge.edge_type == SessionEdgeType::ParentChild { + let Some((_, child)) = sessions.get(&to_key) else { + continue; + }; + if let Some(parent) = &child.parent_session_id { + if parent != &edge.from_session_id { + push_provider_import_failure( + summary, + *line_number, + format!( + "parent_child edge from_session_id `{}` conflicts with session parent_session_id `{parent}`", + edge.from_session_id + ), + ); + } + } + } + } +} + +fn custom_history_session_capture( + source: &CtxHistoryJsonlSourceRecord, + session: &CtxHistoryJsonlSessionRecord, + event: Option, + context: &ProviderAdapterContext, +) -> ProviderCaptureEnvelope { + let provider_session_id = custom_history_internal_session_id( + &source.provider_key, + &source.source_id, + &session.session_id, + ); + let event_cursor = event.as_ref().and_then(|event| { + event.cursor.as_ref().map(|cursor| ProviderCursorRange { + before: None, + after: Some(ProviderCursorCheckpoint { + stream: custom_history_cursor_stream(source), + cursor: cursor.clone(), + observed_at: event.occurred_at, + }), + }) + }); + let source_cursor = source + .cursor + .as_ref() + .map(|cursor| custom_history_normalized_cursor_range(source, cursor)) + .or(event_cursor); + ProviderCaptureEnvelope { + schema_version: PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, + provider: CaptureProvider::Custom, + source: ProviderSourceEnvelope { + source_format: source.source_format.clone(), + machine_id: source + .machine_id + .clone() + .unwrap_or_else(|| context.machine_id.clone()), + observed_at: source.observed_at.unwrap_or(context.imported_at), + raw_source_path: source.raw_source_path.clone().or_else(|| { + context + .source_path + .as_ref() + .map(|path| path.display().to_string()) + }), + raw_retention: source.raw_retention, + redaction_boundary: source.redaction_boundary, + trust: match source.trust { + ProviderSourceTrust::Unknown => ProviderSourceTrust::ProviderExport, + other => other, + }, + fidelity: source.fidelity, + cursor: source_cursor, + idempotency_key: Some(format!( + "ctx-history-jsonl-v1:{}:{}", + source.provider_key, source.source_id + )), + metadata: custom_history_metadata( + source.metadata.clone(), + json!({ + "provider_key": source.provider_key, + "source_id": source.source_id, + "source_format": source.source_format, + "raw_uri": source.raw_uri, + "raw_source_path": source.raw_source_path, + "fingerprint": source.fingerprint, + "importer_version": source.importer_version, + "cursor": source.cursor, + }), + ), + }, + session: ProviderSessionEnvelope { + provider_session_id, + parent_provider_session_id: session.parent_session_id.as_ref().map(|parent| { + custom_history_internal_session_id(&source.provider_key, &source.source_id, parent) + }), + root_provider_session_id: session.root_session_id.as_ref().map(|root| { + custom_history_internal_session_id(&source.provider_key, &source.source_id, root) + }), + external_agent_id: session.external_agent_id.clone(), + agent_type: session.agent_type, + role_hint: session.role_hint.clone(), + is_primary: session.is_primary, + status: session.status, + started_at: session.started_at, + ended_at: session.ended_at, + cwd: session.cwd.clone(), + fidelity: session.fidelity, + idempotency_key: session.idempotency_key.clone().or_else(|| { + Some(format!( + "ctx-history-jsonl-v1:{}:{}:{}", + source.provider_key, source.source_id, session.session_id + )) + }), + artifacts: session.artifacts.clone(), + metadata: custom_history_metadata( + session.metadata.clone(), + json!({ + "provider_key": source.provider_key, + "source_id": source.source_id, + "session_id": session.session_id, + "native_session_id": session.native_session_id, + "parent_session_id": session.parent_session_id, + "root_session_id": session.root_session_id, + }), + ), + }, + event, + } +} + +fn custom_history_event_envelope( + source: &CtxHistoryJsonlSourceRecord, + event: &CtxHistoryJsonlEventRecord, +) -> ProviderEventEnvelope { + let payload = if let Some(preview) = &event.preview { + json!({ "text": preview }) + } else { + event.payload.clone() + }; + let raw_payload = event + .preview + .as_ref() + .map(|_| event.payload.clone()) + .filter(|payload| payload != &json!({})); + ProviderEventEnvelope { + provider_event_index: event.event_index, + provider_event_hash: event.event_hash.clone(), + cursor: event.native_cursor.clone(), + event_type: event.event_type, + role: event.role, + occurred_at: event.occurred_at, + fidelity: event.fidelity, + redaction_state: event.redaction_state, + idempotency_key: event.idempotency_key.clone(), + artifacts: event.artifacts.clone(), + payload, + metadata: custom_history_metadata( + event.metadata.clone(), + json!({ + "provider_key": source.provider_key, + "source_id": event.source_id, + "session_id": event.session_id, + "event_id": event.event_id, + "native_cursor": event.native_cursor, + "preview": event.preview, + "raw_payload": raw_payload, + }), + ), + } +} + +fn custom_history_file_touch_envelope( + source: &CtxHistoryJsonlSourceRecord, + file_touch: &CtxHistoryJsonlFileTouchRecord, +) -> ProviderFileTouchedEnvelope { + ProviderFileTouchedEnvelope { + provider: CaptureProvider::Custom, + provider_session_id: custom_history_internal_session_id( + &source.provider_key, + &source.source_id, + &file_touch.session_id, + ), + provider_touch_index: file_touch.touch_index, + provider_event_index: file_touch.event_index, + path: file_touch.path.clone(), + change_kind: file_touch.change_kind, + old_path: file_touch.old_path.clone(), + line_count_delta: file_touch.line_count_delta, + confidence: file_touch.confidence, + occurred_at: file_touch.occurred_at, + source_format: source.source_format.clone(), + metadata: custom_history_metadata( + file_touch.metadata.clone(), + json!({ + "provider_key": source.provider_key, + "source_id": file_touch.source_id, + "session_id": file_touch.session_id, + }), + ), + } +} + +fn custom_history_edge_import( + source: &CtxHistoryJsonlSourceRecord, + edge: &CtxHistoryJsonlEdgeRecord, + imported_at: DateTime, +) -> CustomHistoryJsonlV1EdgeImport { + CustomHistoryJsonlV1EdgeImport { + provider_key: source.provider_key.clone(), + source_id: source.source_id.clone(), + from_provider_session_id: custom_history_internal_session_id( + &source.provider_key, + &source.source_id, + &edge.from_session_id, + ), + to_provider_session_id: custom_history_internal_session_id( + &source.provider_key, + &source.source_id, + &edge.to_session_id, + ), + edge_id: edge.edge_id.clone(), + edge_type: edge.edge_type, + confidence: edge.confidence, + occurred_at: edge.occurred_at.unwrap_or(imported_at), + fidelity: edge.fidelity, + metadata: custom_history_metadata( + edge.metadata.clone(), + json!({ + "provider_key": source.provider_key, + "source_id": edge.source_id, + "from_session_id": edge.from_session_id, + "to_session_id": edge.to_session_id, + "edge_id": edge.edge_id, + }), + ), + } +} + +fn custom_history_internal_session_id( + provider_key: &str, + source_id: &str, + session_id: &str, +) -> String { + let key = custom_history_key(json!({ + "schema": CTX_HISTORY_JSONL_V1_SCHEMA_VERSION, + "kind": "session", + "provider_key": provider_key, + "source_id": source_id, + "session_id": session_id, + })); + let id = stable_capture_uuid(&key, "custom-provider-session-id"); + format!("ctx-history-jsonl-v1-{id}") +} + +fn custom_history_cursor_stream(source: &CtxHistoryJsonlSourceRecord) -> String { + let key = custom_history_key(json!({ + "schema": CTX_HISTORY_JSONL_V1_SCHEMA_VERSION, + "kind": "cursor_stream", + "provider_key": source.provider_key, + "source_id": source.source_id, + "source_format": source.source_format, + })); + let stream_id = stable_capture_uuid(&key, "custom-cursor-stream"); + format!("provider:custom:{}:{stream_id}", source.provider_key) +} + +fn custom_history_normalized_cursor_range( + source: &CtxHistoryJsonlSourceRecord, + cursor: &ProviderCursorRange, +) -> ProviderCursorRange { + ProviderCursorRange { + before: cursor + .before + .as_ref() + .map(|checkpoint| custom_history_normalized_cursor_checkpoint(source, checkpoint)), + after: cursor + .after + .as_ref() + .map(|checkpoint| custom_history_normalized_cursor_checkpoint(source, checkpoint)), + } +} + +fn custom_history_normalized_cursor_checkpoint( + source: &CtxHistoryJsonlSourceRecord, + checkpoint: &ProviderCursorCheckpoint, +) -> ProviderCursorCheckpoint { + ProviderCursorCheckpoint { + stream: custom_history_cursor_stream(source), + cursor: checkpoint.cursor.clone(), + observed_at: checkpoint.observed_at, + } +} + +fn custom_history_key(value: Value) -> String { + serde_json::to_string(&value).expect("custom history identity key is serializable") +} + +fn custom_history_metadata(base: Value, custom: Value) -> Value { + let mut map = match base { + Value::Object(map) => map, + Value::Null => serde_json::Map::new(), + other => { + let mut map = serde_json::Map::new(); + map.insert("metadata".to_owned(), other); + map + } + }; + map.insert("ctx_history_jsonl_v1".to_owned(), custom); + Value::Object(map) +} + +fn import_custom_history_edges( + store: &mut Store, + edges: &[(usize, CustomHistoryJsonlV1EdgeImport)], + history_record_id: Option, + allow_partial_failures: bool, + summary: &mut ProviderImportSummary, +) -> Result<()> { + if edges.is_empty() { + return Ok(()); + } + + store.begin_immediate_batch()?; + for (line_number, edge) in edges { + let edge_id = if edge.edge_type == SessionEdgeType::ParentChild { + provider_edge_uuid( + CaptureProvider::Custom, + &edge.to_provider_session_id, + "parent_child", + ) + } else { + let key = custom_history_key(json!({ + "schema": CTX_HISTORY_JSONL_V1_SCHEMA_VERSION, + "kind": "session_edge", + "provider_key": edge.provider_key, + "source_id": edge.source_id, + "from_provider_session_id": edge.from_provider_session_id, + "to_provider_session_id": edge.to_provider_session_id, + "edge_type": edge.edge_type.as_str(), + "edge_id": edge.edge_id, + })); + stable_capture_uuid(&key, "session-edge") + }; + let from_session_id = + provider_session_uuid(CaptureProvider::Custom, &edge.from_provider_session_id); + let to_session_id = + provider_session_uuid(CaptureProvider::Custom, &edge.to_provider_session_id); + let source_id = provider_source_uuid(CaptureProvider::Custom, &edge.to_provider_session_id); + let mut exists_cache = BTreeMap::::new(); + if !provider_session_exists_cached(store, from_session_id, &mut exists_cache)? + || !provider_session_exists_cached(store, to_session_id, &mut exists_cache)? + { + push_provider_import_failure( + summary, + *line_number, + "edge endpoint session was not imported".to_owned(), + ); + if !allow_partial_failures { + let _ = store.rollback_batch(); + return Ok(()); + } + continue; + } + let was_present = store.session_edge_exists(edge_id)?; + let session_edge = SessionEdge { + id: edge_id, + from_session_id, + to_session_id, + edge_type: edge.edge_type, + confidence: edge.confidence, + source_id: Some(source_id), + timestamps: timestamps(edge.occurred_at), + sync: provider_sync_metadata( + edge.fidelity, + json!({ + "provider_key": edge.provider_key, + "source_id": edge.source_id, + "history_record_id": history_record_id, + "metadata": edge.metadata, + }), + ), + }; + store.upsert_session_edge(&session_edge)?; + if edge.edge_type == SessionEdgeType::ParentChild { + let mut child = store.get_session(to_session_id)?; + child.parent_session_id = Some(from_session_id); + if child.root_session_id.is_none() { + child.root_session_id = Some(from_session_id); + } + store.upsert_session(&child)?; + } + if was_present { + summary.skipped_edges += 1; + summary.skipped += 1; + } else { + summary.imported_edges += 1; + summary.imported += 1; + } + } + if let Err(err) = store.commit_batch() { + let _ = store.rollback_batch(); + return Err(err.into()); + } + Ok(()) +} fn collect_jsonl_paths(root: &Path, paths: &mut Vec) -> Result<()> { let metadata = fs::symlink_metadata(root)?; @@ -8714,7 +9671,12 @@ fn import_provider_capture_lines( } } } - resolve_pending_provider_edges(store, &mut summary, &mut caches)?; + if let Err(err) = resolve_pending_provider_edges(store, &mut summary, &mut caches) { + if has_captures && options.wrap_transaction { + let _ = store.rollback_batch(); + } + return Err(err); + } for (line_number, file) in files_touched { if let Err(err) = import_provider_file_touched_line(store, &file, &options) { summary.failed += 1; @@ -8724,6 +9686,12 @@ fn import_provider_capture_lines( }); } } + if summary.failed > 0 && !options.allow_partial_failures { + if has_captures && options.wrap_transaction { + let _ = store.rollback_batch(); + } + return Ok(summary); + } if has_captures && options.wrap_transaction { if let Err(err) = store.commit_batch() { let _ = store.rollback_batch(); @@ -9974,6 +10942,10 @@ mod tests { materialized_fixture("provider-history", name) } + fn custom_history_fixture(name: &str) -> PathBuf { + materialized_fixture("custom-history-jsonl", name) + } + fn materialized_fixture(category: &str, name: &str) -> PathBuf { let source = match category { "provider" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -9982,6 +10954,9 @@ mod tests { "provider-history" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../tests/fixtures/provider-history") .join(name), + "custom-history-jsonl" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/fixtures/custom-history-jsonl") + .join(name), _ => panic!("unknown fixture category {category}"), }; let root = std::env::current_dir() @@ -12530,6 +13505,338 @@ mod tests { assert_eq!(cursor.cursor, "line:3"); } + #[test] + fn custom_history_jsonl_imports_full_shape_and_is_idempotent() { + let temp = tempdir(); + let fixture = custom_history_fixture("basic.jsonl"); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let first = import_custom_history_jsonl_v1( + &fixture, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(fixture.clone()), + imported_at: "2026-06-23T12:10:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + assert_eq!(first.failed, 0, "{:?}", first.failures); + assert_eq!(first.imported_sessions, 2); + assert_eq!(first.imported_events, 2); + assert_eq!(first.imported_edges, 2); + + let root_provider_session_id = + custom_history_internal_session_id("demo-agent", "demo-source", "demo-session"); + let child_provider_session_id = + custom_history_internal_session_id("demo-agent", "demo-source", "demo-session-worker"); + let root_id = provider_session_uuid(CaptureProvider::Custom, &root_provider_session_id); + let child_id = provider_session_uuid(CaptureProvider::Custom, &child_provider_session_id); + let root = store.get_session(root_id).unwrap(); + let child = store.get_session(child_id).unwrap(); + assert_eq!(root.provider, CaptureProvider::Custom); + assert_eq!(child.parent_session_id, Some(root_id)); + assert!(root + .sync + .metadata + .to_string() + .contains("\"provider_key\":\"demo-agent\"")); + let events = store.events_for_session(root_id).unwrap(); + assert_eq!(events.len(), 2); + assert!(events[0].payload.to_string().contains("Add a parser test.")); + + let conn = rusqlite::Connection::open(temp.path().join("work.sqlite")).unwrap(); + let touched: i64 = conn + .query_row("SELECT COUNT(*) FROM files_touched", [], |row| row.get(0)) + .unwrap(); + assert_eq!(touched, 1); + let spawned_edges: i64 = conn + .query_row( + "SELECT COUNT(*) FROM session_edges WHERE edge_type = 'spawned'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(spawned_edges, 1); + let cursor_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sync_cursors WHERE stream LIKE 'provider:custom:demo-agent:%'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(cursor_count, 1); + let cursor: String = conn + .query_row( + "SELECT cursor FROM sync_cursors WHERE stream LIKE 'provider:custom:demo-agent:%'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(cursor, "5"); + let raw_cursor_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sync_cursors WHERE stream = 'demo-agent:demo-source'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(raw_cursor_count, 0); + drop(conn); + + let second = import_custom_history_jsonl_v1( + &fixture, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(fixture.clone()), + imported_at: "2026-06-23T12:10:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + assert_eq!(second.failed, 0); + assert_eq!(second.imported_sessions, 0); + assert_eq!(second.imported_events, 0); + assert_eq!(second.imported_edges, 0); + assert_eq!(second.skipped_events, 2); + assert_eq!(second.skipped_edges, 2); + } + + #[test] + fn custom_history_jsonl_malformed_import_is_atomic_by_default() { + let temp = tempdir(); + let fixture = custom_history_fixture("malformed-partial.jsonl"); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_custom_history_jsonl_v1( + &fixture, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(fixture.clone()), + imported_at: "2026-06-23T13:10:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.imported_sessions, 0); + assert_eq!(summary.imported_events, 0); + assert_eq!(summary.failed, 1); + assert_eq!(store.capture_source_count().unwrap(), 0); + let conn = rusqlite::Connection::open(temp.path().join("work.sqlite")).unwrap(); + let sessions: i64 = conn + .query_row("SELECT COUNT(*) FROM sessions", [], |row| row.get(0)) + .unwrap(); + let events: i64 = conn + .query_row("SELECT COUNT(*) FROM events", [], |row| row.get(0)) + .unwrap(); + assert_eq!(sessions, 0); + assert_eq!(events, 0); + } + + #[test] + fn custom_history_jsonl_preview_overrides_raw_payload_for_searchable_event_payload() { + let temp = tempdir(); + let fixture = temp.path().join("preview-overrides-payload.jsonl"); + fs::write( + &fixture, + [ + r#"{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}"#, + r#"{"record_type":"source","source_id":"src","provider_key":"preview-agent","source_format":"demo"}"#, + r#"{"record_type":"session","source_id":"src","session_id":"run","started_at":"2026-06-23T14:00:00Z"}"#, + r#"{"record_type":"event","source_id":"src","session_id":"run","event_index":0,"event_type":"message","role":"assistant","occurred_at":"2026-06-23T14:00:01Z","payload":{"raw":"unindexed-raw-payload-token"},"preview":"bounded searchable preview text"}"#, + ] + .join("\n"), + ) + .unwrap(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_custom_history_jsonl_v1( + &fixture, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(fixture.clone()), + imported_at: "2026-06-23T14:10:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + let session_id = provider_session_uuid( + CaptureProvider::Custom, + &custom_history_internal_session_id("preview-agent", "src", "run"), + ); + let events = store.events_for_session(session_id).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!( + events[0].payload["body"], + json!({ "text": "bounded searchable preview text" }) + ); + assert!(!events[0] + .payload + .to_string() + .contains("unindexed-raw-payload-token")); + assert_eq!( + events[0].sync.metadata["metadata"]["ctx_history_jsonl_v1"]["raw_payload"]["raw"] + .as_str(), + Some("unindexed-raw-payload-token") + ); + } + + #[test] + fn custom_history_jsonl_namespaces_provider_keys_to_avoid_collisions() { + let temp = tempdir(); + let fixture = temp.path().join("same-native-ids.jsonl"); + fs::write( + &fixture, + [ + r#"{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}"#, + r#"{"record_type":"source","source_id":"src","provider_key":"alpha","source_format":"demo"}"#, + r#"{"record_type":"session","source_id":"src","session_id":"same","started_at":"2026-06-23T14:00:00Z"}"#, + r#"{"record_type":"event","source_id":"src","session_id":"same","event_index":0,"event_type":"message","role":"user","occurred_at":"2026-06-23T14:00:01Z","payload":{"text":"alpha text"}}"#, + r#"{"record_type":"source","source_id":"src-2","provider_key":"beta","source_format":"demo"}"#, + r#"{"record_type":"session","source_id":"src-2","session_id":"same","started_at":"2026-06-23T14:01:00Z"}"#, + r#"{"record_type":"event","source_id":"src-2","session_id":"same","event_index":0,"event_type":"message","role":"user","occurred_at":"2026-06-23T14:01:01Z","payload":{"text":"beta text"}}"#, + ] + .join("\n"), + ) + .unwrap(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_custom_history_jsonl_v1( + &fixture, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(fixture.clone()), + imported_at: "2026-06-23T14:10:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_sessions, 2); + assert_eq!(summary.imported_events, 2); + let alpha_session = provider_session_uuid( + CaptureProvider::Custom, + &custom_history_internal_session_id("alpha", "src", "same"), + ); + let beta_session = provider_session_uuid( + CaptureProvider::Custom, + &custom_history_internal_session_id("beta", "src-2", "same"), + ); + assert_ne!(alpha_session, beta_session); + assert!(store + .events_for_session(alpha_session) + .unwrap() + .iter() + .any(|event| event.payload.to_string().contains("alpha text"))); + assert!(store + .events_for_session(beta_session) + .unwrap() + .iter() + .any(|event| event.payload.to_string().contains("beta text"))); + } + + #[test] + fn custom_history_jsonl_hashes_delimited_identifiers_without_collisions() { + let temp = tempdir(); + let fixture = temp.path().join("delimited-identifiers.jsonl"); + fs::write( + &fixture, + [ + r#"{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}"#, + r#"{"record_type":"source","source_id":"a:b","provider_key":"delim-agent","source_format":"demo"}"#, + r#"{"record_type":"session","source_id":"a:b","session_id":"c","started_at":"2026-06-23T14:00:00Z"}"#, + r#"{"record_type":"event","source_id":"a:b","session_id":"c","event_index":0,"event_type":"message","role":"user","occurred_at":"2026-06-23T14:00:01Z","payload":{"text":"left text"}}"#, + r#"{"record_type":"source","source_id":"a","provider_key":"delim-agent","source_format":"demo"}"#, + r#"{"record_type":"session","source_id":"a","session_id":"b:c","started_at":"2026-06-23T14:01:00Z"}"#, + r#"{"record_type":"event","source_id":"a","session_id":"b:c","event_index":0,"event_type":"message","role":"user","occurred_at":"2026-06-23T14:01:01Z","payload":{"text":"right text"}}"#, + ] + .join("\n"), + ) + .unwrap(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_custom_history_jsonl_v1( + &fixture, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(fixture.clone()), + imported_at: "2026-06-23T14:10:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_sessions, 2); + assert_eq!(summary.imported_events, 2); + let left_session = provider_session_uuid( + CaptureProvider::Custom, + &custom_history_internal_session_id("delim-agent", "a:b", "c"), + ); + let right_session = provider_session_uuid( + CaptureProvider::Custom, + &custom_history_internal_session_id("delim-agent", "a", "b:c"), + ); + assert_ne!(left_session, right_session); + assert!(store + .events_for_session(left_session) + .unwrap() + .iter() + .any(|event| event.payload.to_string().contains("left text"))); + assert!(store + .events_for_session(right_session) + .unwrap() + .iter() + .any(|event| event.payload.to_string().contains("right text"))); + } + + #[test] + fn custom_history_jsonl_dedupes_explicit_parent_child_edge_from_session_parent() { + let temp = tempdir(); + let fixture = temp.path().join("duplicate-parent-child.jsonl"); + fs::write( + &fixture, + [ + r#"{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}"#, + r#"{"record_type":"source","source_id":"src","provider_key":"edge-agent","source_format":"demo"}"#, + r#"{"record_type":"session","source_id":"src","session_id":"root","started_at":"2026-06-23T15:00:00Z"}"#, + r#"{"record_type":"session","source_id":"src","session_id":"child","parent_session_id":"root","started_at":"2026-06-23T15:00:01Z"}"#, + r#"{"record_type":"edge","source_id":"src","from_session_id":"root","to_session_id":"child","edge_type":"parent_child","edge_id":"explicit-parent","occurred_at":"2026-06-23T15:00:02Z"}"#, + ] + .join("\n"), + ) + .unwrap(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_custom_history_jsonl_v1( + &fixture, + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(fixture.clone()), + imported_at: "2026-06-23T15:10:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_edges, 1); + assert_eq!(summary.skipped_edges, 1); + let conn = rusqlite::Connection::open(temp.path().join("work.sqlite")).unwrap(); + let parent_child_edges: i64 = conn + .query_row( + "SELECT COUNT(*) FROM session_edges WHERE edge_type = 'parent_child'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(parent_child_edges, 1); + } + #[test] fn provider_fixture_replay_rejects_malformed_lines_without_partial_import_by_default() { let temp = tempdir(); diff --git a/crates/ctx-history-core/src/history_jsonl.rs b/crates/ctx-history-core/src/history_jsonl.rs new file mode 100644 index 000000000..7ecdaa6b6 --- /dev/null +++ b/crates/ctx-history-core/src/history_jsonl.rs @@ -0,0 +1,213 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::{ + AgentType, Confidence, EventRole, EventType, Fidelity, FileChangeKind, + ProviderArtifactDescriptor, ProviderCursorRange, ProviderRawRetention, + ProviderRedactionBoundary, ProviderSourceTrust, RedactionState, SessionEdgeType, SessionStatus, +}; + +pub const CTX_HISTORY_JSONL_V1_SCHEMA_VERSION: &str = "ctx-history-jsonl-v1"; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "record_type", rename_all = "snake_case")] +pub enum CtxHistoryJsonlRecord { + Manifest(CtxHistoryJsonlManifestRecord), + Source(CtxHistoryJsonlSourceRecord), + Session(CtxHistoryJsonlSessionRecord), + Event(CtxHistoryJsonlEventRecord), + FileTouch(CtxHistoryJsonlFileTouchRecord), + Edge(CtxHistoryJsonlEdgeRecord), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CtxHistoryJsonlManifestRecord { + pub schema_version: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub producer: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exported_at: Option>, + #[serde(default = "super::default_metadata")] + pub metadata: Value, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CtxHistoryJsonlSourceRecord { + pub source_id: String, + pub provider_key: String, + pub source_format: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw_uri: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw_source_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fingerprint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub importer_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub machine_id: Option, + #[serde(default)] + pub raw_retention: ProviderRawRetention, + #[serde(default)] + pub redaction_boundary: ProviderRedactionBoundary, + #[serde(default)] + pub trust: ProviderSourceTrust, + #[serde(default = "default_imported_fidelity")] + pub fidelity: Fidelity, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, + #[serde(default = "super::default_metadata")] + pub metadata: Value, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CtxHistoryJsonlSessionRecord { + pub source_id: String, + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root_session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub external_agent_id: Option, + #[serde(default)] + pub agent_type: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role_hint: Option, + #[serde(default)] + pub is_primary: bool, + #[serde(default = "default_imported_session_status")] + pub status: SessionStatus, + pub started_at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ended_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cwd: Option, + #[serde(default = "default_imported_fidelity")] + pub fidelity: Fidelity, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub idempotency_key: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artifacts: Vec, + #[serde(default = "super::default_metadata")] + pub metadata: Value, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CtxHistoryJsonlEventRecord { + pub source_id: String, + pub session_id: String, + pub event_index: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub event_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_cursor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub event_hash: Option, + #[serde(default)] + pub event_type: EventType, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + pub occurred_at: DateTime, + #[serde(default = "default_imported_fidelity")] + pub fidelity: Fidelity, + #[serde(default)] + pub redaction_state: RedactionState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub idempotency_key: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artifacts: Vec, + #[serde(default = "super::default_metadata")] + pub payload: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preview: Option, + #[serde(default = "super::default_metadata")] + pub metadata: Value, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CtxHistoryJsonlFileTouchRecord { + pub source_id: String, + pub session_id: String, + pub touch_index: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub event_index: Option, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub change_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub old_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub line_count_delta: Option, + #[serde(default)] + pub confidence: Confidence, + pub occurred_at: DateTime, + #[serde(default = "super::default_metadata")] + pub metadata: Value, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CtxHistoryJsonlEdgeRecord { + pub source_id: String, + pub from_session_id: String, + pub to_session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub edge_id: Option, + pub edge_type: SessionEdgeType, + #[serde(default)] + pub confidence: Confidence, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub occurred_at: Option>, + #[serde(default = "default_imported_fidelity")] + pub fidelity: Fidelity, + #[serde(default = "super::default_metadata")] + pub metadata: Value, +} + +const fn default_imported_session_status() -> SessionStatus { + SessionStatus::Imported +} + +const fn default_imported_fidelity() -> Fidelity { + Fidelity::Imported +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ctx_history_jsonl_records_round_trip() { + let raw = r#"{"record_type":"event","source_id":"src-1","session_id":"sess-1","event_index":2,"event_id":"evt-2","native_cursor":"line:3","event_type":"message","role":"assistant","occurred_at":"2026-07-01T12:00:02Z","payload":{"text":"hello"},"preview":"hello"}"#; + let parsed: CtxHistoryJsonlRecord = serde_json::from_str(raw).unwrap(); + let CtxHistoryJsonlRecord::Event(event) = parsed else { + panic!("expected event record"); + }; + assert_eq!(event.source_id, "src-1"); + assert_eq!(event.session_id, "sess-1"); + assert_eq!(event.event_index, 2); + assert_eq!(event.role, Some(EventRole::Assistant)); + assert_eq!( + serde_json::to_value(CtxHistoryJsonlRecord::Event(event)) + .unwrap() + .get("record_type") + .and_then(Value::as_str), + Some("event") + ); + } + + #[test] + fn ctx_history_jsonl_edge_type_is_required() { + let raw = r#"{"record_type":"edge","source_id":"src-1","from_session_id":"root","to_session_id":"child"}"#; + let err = serde_json::from_str::(raw).unwrap_err(); + assert!( + err.to_string().contains("missing field `edge_type`"), + "{err}" + ); + } +} diff --git a/crates/ctx-history-core/src/lib.rs b/crates/ctx-history-core/src/lib.rs index cac38f6ac..f7d819fd7 100644 --- a/crates/ctx-history-core/src/lib.rs +++ b/crates/ctx-history-core/src/lib.rs @@ -97,8 +97,10 @@ macro_rules! text_enum { }; } +mod history_jsonl; mod provider; +pub use history_jsonl::*; pub use provider::*; text_enum! { @@ -184,6 +186,7 @@ text_enum! { Git => "git", Jj => "jj", Gh => "gh", + Custom => "custom", Unknown => "unknown", } default Unknown diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 035983ffa..cc7c0211c 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -471,7 +471,7 @@ const CREATE_TABLES_SQL: &str = r#" CREATE TABLE IF NOT EXISTS capture_sources ( id TEXT PRIMARY KEY NOT NULL, kind TEXT NOT NULL CHECK (kind IN ('provider_import', 'provider_hook', 'direct_cli', 'manual')), - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), machine_id TEXT NOT NULL, process_id INTEGER, cwd TEXT, @@ -488,7 +488,7 @@ CREATE TABLE IF NOT EXISTS capture_sources ( CREATE TABLE IF NOT EXISTS catalog_sessions ( source_path TEXT PRIMARY KEY NOT NULL, - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, external_session_id TEXT, @@ -517,7 +517,7 @@ CREATE TABLE IF NOT EXISTS catalog_sessions ( ); CREATE TABLE IF NOT EXISTS source_import_files ( - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, source_path TEXT NOT NULL, @@ -4850,7 +4850,8 @@ fn migrate_to_v13(conn: &Connection) -> Result<()> { } fn migrate_to_v14(conn: &Connection) -> Result<()> { - conn.execute_batch("BEGIN IMMEDIATE;")?; + let foreign_keys_enabled: i64 = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?; + conn.execute_batch("PRAGMA foreign_keys = OFF; BEGIN IMMEDIATE;")?; let migration = (|| -> Result<()> { conn.execute_batch(CREATE_TABLES_SQL)?; ensure_columns( @@ -4858,8 +4859,12 @@ fn migrate_to_v14(conn: &Connection) -> Result<()> { "catalog_sessions", CATALOG_SESSION_IMPORT_STATE_COLUMNS, )?; + rebuild_capture_sources_provider_check(conn)?; + rebuild_catalog_sessions_provider_check(conn)?; + rebuild_source_import_files_provider_check(conn)?; backfill_catalog_session_import_checkpoints(conn)?; create_stable_sql_views(conn)?; + conn.execute_batch(INDEXES_SQL)?; conn.execute_batch("PRAGMA user_version = 14;")?; Ok(()) })(); @@ -4867,12 +4872,18 @@ fn migrate_to_v14(conn: &Connection) -> Result<()> { match migration { Ok(()) => { conn.execute_batch("COMMIT;")?; + if foreign_keys_enabled != 0 { + conn.execute_batch("PRAGMA foreign_keys = ON;")?; + } Ok(()) } Err(err) => { if let Err(rollback_err) = conn.execute_batch("ROLLBACK;") { return Err(StoreError::Sql(rollback_err)); } + if foreign_keys_enabled != 0 { + conn.execute_batch("PRAGMA foreign_keys = ON;")?; + } Err(err) } } @@ -5081,7 +5092,7 @@ fn rebuild_capture_sources_provider_check(conn: &Connection) -> Result<()> { CREATE TABLE capture_sources_new ( id TEXT PRIMARY KEY NOT NULL, kind TEXT NOT NULL CHECK (kind IN ('provider_import', 'provider_hook', 'direct_cli', 'manual')), - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), machine_id TEXT NOT NULL, process_id INTEGER, cwd TEXT, @@ -5129,7 +5140,7 @@ fn rebuild_catalog_sessions_provider_check(conn: &Connection) -> Result<()> { DROP TABLE IF EXISTS catalog_sessions_new; CREATE TABLE catalog_sessions_new ( source_path TEXT PRIMARY KEY NOT NULL, - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, external_session_id TEXT, @@ -5184,7 +5195,7 @@ fn rebuild_source_import_files_provider_check(conn: &Connection) -> Result<()> { r#" DROP TABLE IF EXISTS source_import_files_new; CREATE TABLE source_import_files_new ( - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, source_path TEXT NOT NULL, @@ -8627,6 +8638,7 @@ mod catalog_tests { for (provider, source_format) in [ ("copilot_cli", "copilot_cli_session_events_jsonl"), ("factory_ai_droid", "factory_ai_droid_sessions_jsonl"), + ("custom", "ctx_history_jsonl_v1"), ] { assert!( schema.contains(provider), @@ -8659,7 +8671,7 @@ mod catalog_tests { let source_count: i64 = store .conn .query_row( - "SELECT COUNT(*) FROM capture_sources WHERE provider IN ('copilot_cli', 'factory_ai_droid')", + "SELECT COUNT(*) FROM capture_sources WHERE provider IN ('copilot_cli', 'factory_ai_droid', 'custom')", [], |row| row.get(0), ) @@ -8667,13 +8679,13 @@ mod catalog_tests { let catalog_count: i64 = store .conn .query_row( - "SELECT COUNT(*) FROM catalog_sessions WHERE provider IN ('copilot_cli', 'factory_ai_droid')", + "SELECT COUNT(*) FROM catalog_sessions WHERE provider IN ('copilot_cli', 'factory_ai_droid', 'custom')", [], |row| row.get(0), ) .unwrap(); - assert_eq!(source_count, 2); - assert_eq!(catalog_count, 2); + assert_eq!(source_count, 3); + assert_eq!(catalog_count, 3); } #[test] diff --git a/docs/cli-reference.md b/docs/cli-reference.md index b00859228..dd9159edb 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -87,6 +87,7 @@ ctx import --provider copilot-cli ctx import --provider factory-ai-droid ctx import --path ~/.codex/sessions ctx import --provider pi --path ~/.pi/sessions.jsonl +ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl ctx import --resume ctx import --json ctx import --progress json --json @@ -99,11 +100,18 @@ creates the data root and default config if needed, reads provider transcript files, and writes indexed source metadata, sessions, events, searchable text, citations, and import totals to SQLite. +Custom history can be imported from an explicit JSONL file with +`--format ctx-history-jsonl-v1 --path `. This path is not discovered or +remembered as a provider home; see `docs/custom-history-import-format.md` for +the schema and incremental semantics. + Import selection rules: - with no arguments or with `--all`, import all discovered auto-importable sources that exist; - with `--provider`, import discovered sources for that provider; +- with `--format ctx-history-jsonl-v1 --path `, import that custom + history JSONL file; - with `--path`, import exactly that path; - with `--path` and no provider, parse the path as Codex format. @@ -213,7 +221,7 @@ optimized for agent reading; use `--verbose` for expanded text diagnostics. Filters: -- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; +- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|antigravity|gemini|cursor|copilot-cli|factory-ai-droid|custom`; - `--workspace `, substring match over stored workspace, cwd, source path, or repository-name text; - `--since d`, for example `2026-06-01T00:00:00Z` or `30d`; diff --git a/docs/custom-history-import-format.md b/docs/custom-history-import-format.md new file mode 100644 index 000000000..703360754 --- /dev/null +++ b/docs/custom-history-import-format.md @@ -0,0 +1,228 @@ +# Custom History Import Format + +`ctx-history-jsonl-v1` is the public JSONL format for importing session history +from tools without a built-in local-history adapter. + +## Transport + +Version 1 uses an explicit local file path: + +```bash +ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl +``` + +ctx does not discover a fixed storage location for this format. The file is not +read from stdin, and ctx does not execute exporter commands in v1. + +Each line is one JSON object. Every object has a `record_type` field with one +of: + +- `manifest` +- `source` +- `session` +- `event` +- `file_touch` +- `edge` + +Record order is flexible, but exporters should write a manifest first, then +source and session records before their dependent events, file touches, and +edges. Unknown fields are ignored unless they are inside `metadata`, `payload`, +or another explicitly documented open object. + +## Manifest + +Exactly one manifest record should appear near the top of the file. + +Required fields: + +- `schema_version`: must be `"ctx-history-jsonl-v1"`. + +Example: + +```json +{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1","metadata":{"exporter":"example"}} +``` + +## Source + +A source describes the exporting system, input corpus, or incremental cursor. + +Required fields: + +- `source_id` +- `provider_key` +- `source_format` + +Optional fields: + +- `raw_uri` +- `raw_source_path` +- `fingerprint` +- `importer_version` +- `observed_at` +- `machine_id` +- `cursor` +- `metadata` + +`provider_key` is the exporter-owned namespace, such as `my-agent` or +`internal-build-bot`. Internally ctx stores these rows under the bounded +provider value `custom`, then derives internal session IDs from the structured +`provider_key`, `source_id`, and `session_id` tuple. Public provider, source, +and session IDs are preserved in metadata for display and lookup. + +`provider_key` must be 1 to 128 bytes, start with a lowercase ASCII letter or +digit, and contain only lowercase ASCII letters, digits, `.`, `_`, or `-`. + +Example: + +```json +{"record_type":"source","source_id":"laptop-main","provider_key":"my-agent","source_format":"my-agent-export-v3","raw_source_path":"/home/me/.my-agent/history.jsonl","cursor":{"after":{"stream":"my-agent:laptop-main","cursor":"171","observed_at":"2026-06-23T12:00:00Z"}},"metadata":{"team":"tools"}} +``` + +## Session + +A session describes one conversation, task, or agent run. + +Required fields: + +- `source_id` +- `session_id` +- `started_at` + +Optional fields: + +- `parent_session_id` +- `root_session_id` +- `native_session_id` +- `cwd` +- `ended_at` +- `agent_type` +- `role_hint` +- `is_primary` +- `status` +- `metadata` + +Use `parent_session_id` and `root_session_id` to model subagents, forks, +handoffs, or resumed tasks when the exporter knows those relationships. + +Example: + +```json +{"record_type":"session","source_id":"laptop-main","session_id":"run-1","native_session_id":"abc123","cwd":"/workspace/app","started_at":"2026-06-23T12:00:00Z","agent_type":"primary","role_hint":"developer","is_primary":true,"status":"completed"} +``` + +## Event + +An event is a time-ordered item inside a session. + +Required fields: + +- `source_id` +- `session_id` +- `event_index`: unsigned 64-bit integer. +- `occurred_at` + +Optional fields: + +- `event_id` +- `native_cursor` +- `event_type` +- `role` +- `payload` +- `preview` +- `metadata` + +`event_index` is the stable exporter order within the session. Use +`native_cursor` for provider cursor tokens or byte offsets that should survive +re-imports. `payload` is open JSON; `preview` should be a bounded searchable +summary when payloads are large or sensitive. When `preview` is present, ctx +uses it as the event's searchable payload and preserves any non-empty `payload` +under import metadata. + +Example: + +```json +{"record_type":"event","source_id":"laptop-main","session_id":"run-1","event_index":0,"event_type":"message","role":"user","occurred_at":"2026-06-23T12:00:01Z","payload":{"text":"Find the failing test."},"preview":"Find the failing test.","native_cursor":"line:42"} +``` + +## File Touch + +A file touch records a path that the session read, wrote, created, deleted, or +renamed. + +Required fields: + +- `source_id` +- `session_id` +- `touch_index`: unsigned 64-bit integer. +- `path` +- `occurred_at` + +Optional fields: + +- `event_index` +- `change_kind` +- `old_path` +- `line_count_delta` +- `confidence` +- `metadata` + +`event_index` links the touch to an event when known. Use `old_path` for +renames, `line_count_delta` for approximate net line changes, and `confidence` +when a touch is inferred from text rather than structured tool output. + +Example: + +```json +{"record_type":"file_touch","source_id":"laptop-main","session_id":"run-1","touch_index":0,"event_index":1,"path":"crates/app/src/lib.rs","change_kind":"modified","line_count_delta":12,"confidence":"high","occurred_at":"2026-06-23T12:00:03Z"} +``` + +## Edge + +An edge records a relationship between two sessions from the same source. + +Required fields: + +- `source_id` +- `from_session_id` +- `to_session_id` +- `edge_type` + +Optional fields: + +- `edge_id` +- `confidence` +- `occurred_at` +- `metadata` + +Example: + +```json +{"record_type":"edge","source_id":"laptop-main","from_session_id":"run-1","to_session_id":"run-1-worker","edge_type":"spawned","confidence":"explicit","occurred_at":"2026-06-23T12:00:05Z"} +``` + +## Incremental Semantics + +v1 imports are explicit, local, and idempotent. On each +`ctx import --format ctx-history-jsonl-v1 --path `, ctx rescans the file +and upserts equivalent records instead of appending duplicates. + +When a source record supplies `cursor`, ctx rewrites its storage stream under a +`provider:custom::` namespace and also preserves the +exporter-supplied cursor object in source metadata. Event `native_cursor` values +are also preserved. ctx does not negotiate with external exporters in v1, does +not call exporter commands, and does not request a delta range; exporter +negotiation is a follow-up capability. + +If an import is interrupted, run the same command again. The expected behavior +is another idempotent rescan of the same JSONL file. + +## Compact Example + +```jsonl +{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"} +{"record_type":"source","source_id":"demo-source","provider_key":"demo-agent","source_format":"demo-jsonl","raw_source_path":"/tmp/demo-history.jsonl","cursor":{"after":{"stream":"demo-agent:demo-source","cursor":"3","observed_at":"2026-06-23T12:00:00Z"}}} +{"record_type":"session","source_id":"demo-source","session_id":"demo-session","cwd":"/workspace/demo","started_at":"2026-06-23T12:00:00Z","agent_type":"primary","role_hint":"developer","is_primary":true,"status":"completed"} +{"record_type":"event","source_id":"demo-source","session_id":"demo-session","event_index":0,"event_type":"message","role":"user","occurred_at":"2026-06-23T12:00:01Z","payload":{"text":"Add a parser test."},"preview":"Add a parser test.","native_cursor":"line:1"} +{"record_type":"file_touch","source_id":"demo-source","session_id":"demo-session","touch_index":0,"event_index":0,"path":"tests/parser.rs","change_kind":"modified","confidence":"high","occurred_at":"2026-06-23T12:00:02Z"} +``` diff --git a/docs/providers.md b/docs/providers.md index 7f554bb5f..ad2400269 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -33,6 +33,13 @@ The current CLI imports local history for: `~/.copilot/session-state`; - Factory AI Droid session JSONL files under `~/.factory/sessions`. +These are built-in provider adapters for native local history. The custom +history format is separate: `ctx import --format ctx-history-jsonl-v1 --path +` reads an explicit JSONL interchange file from any exporter. It is +stored internally under the bounded provider `custom` while preserving the +exporter's `provider_key`, `source_id`, and `session_id` as metadata and ID +namespace components. It is not auto-discovered by `ctx sources`. + Use `ctx sources` for the truth on the current machine: ```bash @@ -87,6 +94,10 @@ Provider imports should be: - clear about which fields were indexed and which were left raw-only; - conservative when a transcript schema is unknown or malformed. +Custom history imports follow the same read-only and idempotent principles, but +their compatibility contract is the `ctx-history-jsonl-v1` schema rather than a +provider-owned native transcript format. + ## Fidelity An imported session may include messages, tool calls, command events, output diff --git a/docs/storage.md b/docs/storage.md index bd59353eb..57fd6e542 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -77,7 +77,7 @@ analytics marker described under network behavior. | `ctx setup` | provider transcript files and home path metadata for source discovery | data root, `work.sqlite`, `config.toml`, and SQLite index | | `ctx status` | data root metadata and existing SQLite store | none | | `ctx sources` | known provider paths under the user's home | none | -| `ctx import` | provider transcript files and path metadata | data root, `config.toml` if missing, and SQLite index | +| `ctx import` | provider transcript files and path metadata, or the explicit custom history JSONL file passed with `--format ctx-history-jsonl-v1 --path` | data root, `config.toml` if missing, and SQLite index | | `ctx show` | SQLite index | selected `--out` path for `show session` when provided | | `ctx locate` | SQLite index and raw source path metadata | none | | `ctx search` | native provider transcript files, path metadata, and SQLite index | SQLite index for newly discovered native provider history | @@ -126,10 +126,16 @@ Re-import or update the index: ctx import --all ctx import --resume ctx import --path ~/.codex/sessions +ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl ``` Current adapters are safe to re-run. They rescan sources idempotently and keep source paths or cursors when available. +Custom history JSONL imports follow the same v1 lifecycle: ctx rescans the +explicit file, upserts already-imported records, stores supplied source cursor +metadata under ctx-owned custom cursor streams, and preserves event native +cursors. The path is not added to `config.toml` or treated as a fixed provider +location. ## Upgrade Reindexing @@ -153,8 +159,9 @@ $EDITOR ~/.ctx/config.toml The current CLI does not add provider source entries to `config.toml`; default provider locations are discovered each time and explicit `--path` imports are -not remembered as future defaults. To remove already indexed data, rebuild the -index and import only the sources you still want. +not remembered as future defaults. Custom history JSONL paths are also +one-shot explicit imports. To remove already indexed data, rebuild the index and +import only the sources you still want. ## SQL Inspection diff --git a/tests/fixtures/custom-history-jsonl/basic.jsonl b/tests/fixtures/custom-history-jsonl/basic.jsonl new file mode 100644 index 000000000..74615e278 --- /dev/null +++ b/tests/fixtures/custom-history-jsonl/basic.jsonl @@ -0,0 +1,8 @@ +{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1","metadata":{"fixture":"basic"}} +{"record_type":"source","source_id":"demo-source","provider_key":"demo-agent","source_format":"demo-jsonl","raw_source_path":"/tmp/demo-history.jsonl","fingerprint":"sha256:demo","importer_version":"1.0.0","observed_at":"2026-06-23T12:00:10Z","machine_id":"fixture-host","cursor":{"after":{"stream":"demo-agent:demo-source","cursor":"5","observed_at":"2026-06-23T12:00:10Z"}},"metadata":{"purpose":"fixture"}} +{"record_type":"session","source_id":"demo-source","session_id":"demo-session","native_session_id":"native-demo-session","cwd":"/workspace/demo","started_at":"2026-06-23T12:00:00Z","ended_at":"2026-06-23T12:05:00Z","agent_type":"primary","role_hint":"developer","is_primary":true,"status":"completed","metadata":{"model":"fixture-model"}} +{"record_type":"event","source_id":"demo-source","session_id":"demo-session","event_index":0,"event_id":"evt-0","native_cursor":"line:3","event_type":"message","role":"user","occurred_at":"2026-06-23T12:00:01Z","payload":{"text":"Add a parser test."},"preview":"Add a parser test.","metadata":{"source":"fixture"}} +{"record_type":"event","source_id":"demo-source","session_id":"demo-session","event_index":1,"event_id":"evt-1","native_cursor":"line:4","event_type":"tool_call","role":"assistant","occurred_at":"2026-06-23T12:00:02Z","payload":{"tool":"exec","args":{"cmd":"cargo test -p ctx-history"}},"preview":"cargo test -p ctx-history","metadata":{"source":"fixture"}} +{"record_type":"file_touch","source_id":"demo-source","session_id":"demo-session","touch_index":0,"event_index":1,"path":"crates/history/src/import.rs","change_kind":"modified","line_count_delta":18,"confidence":"explicit","occurred_at":"2026-06-23T12:00:03Z","metadata":{"source":"fixture"}} +{"record_type":"session","source_id":"demo-source","session_id":"demo-session-worker","parent_session_id":"demo-session","root_session_id":"demo-session","cwd":"/workspace/demo","started_at":"2026-06-23T12:01:00Z","agent_type":"subagent","role_hint":"worker","is_primary":false,"status":"completed","metadata":{"model":"fixture-model"}} +{"record_type":"edge","source_id":"demo-source","from_session_id":"demo-session","to_session_id":"demo-session-worker","edge_type":"spawned","edge_id":"edge-0","confidence":"explicit","occurred_at":"2026-06-23T12:01:00Z","metadata":{"source":"fixture"}} diff --git a/tests/fixtures/custom-history-jsonl/malformed-partial.jsonl b/tests/fixtures/custom-history-jsonl/malformed-partial.jsonl new file mode 100644 index 000000000..b59593713 --- /dev/null +++ b/tests/fixtures/custom-history-jsonl/malformed-partial.jsonl @@ -0,0 +1,5 @@ +{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1","metadata":{"fixture":"malformed-partial"}} +{"record_type":"source","source_id":"partial-source","provider_key":"partial-agent","source_format":"partial-jsonl","cursor":{"after":{"stream":"partial-agent:partial-source","cursor":"1","observed_at":"2026-06-23T13:00:00Z"}}} +{"record_type":"session","source_id":"partial-source","session_id":"partial-session","cwd":"/workspace/demo","started_at":"2026-06-23T13:00:00Z","agent_type":"primary","role_hint":"developer","is_primary":true,"status":"active"} +{"record_type":"event","source_id":"partial-source","session_id":"partial-session","event_index":0,"event_type":"message","role":"user","occurred_at":"2026-06-23T13:00:01Z","payload":{"text":"Valid event before malformed record."},"preview":"Valid event before malformed record."} +{"record_type":"event","source_id":"partial-source","session_id":"partial-session","event_index":"not-a-u64","event_type":"message","role":"assistant","occurred_at":"2026-06-23T13:00:02Z","payload":{"text":"This record is valid JSONL but semantically invalid."},"preview":"This record is valid JSONL but semantically invalid."} From a1af9c7c865fbe82bcaeac91f17e7ffea56ba4c9 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:18:30 -0500 Subject: [PATCH 29/72] Add history source plugin imports --- Cargo.lock | 1 + crates/ctx-cli/Cargo.toml | 1 + crates/ctx-cli/src/docs.rs | 9 + crates/ctx-cli/src/history_source_plugins.rs | 506 +++++++++++++++++++ crates/ctx-cli/src/main.rs | 493 +++++++++++++++++- crates/ctx-cli/tests/cli.rs | 503 ++++++++++++++++++ crates/ctx-history-capture/src/lib.rs | 203 +++++++- docs/cli-reference.md | 55 +- docs/custom-history-import-format.md | 36 +- docs/first-10-minutes.md | 6 +- docs/getting-started.md | 5 +- docs/history-source-plugins.md | 225 +++++++++ docs/providers.md | 10 +- docs/search.md | 15 +- docs/storage.md | 11 +- 15 files changed, 2006 insertions(+), 73 deletions(-) create mode 100644 crates/ctx-cli/src/history_source_plugins.rs create mode 100644 docs/history-source-plugins.md diff --git a/Cargo.lock b/Cargo.lock index eabab954c..db382477d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -272,6 +272,7 @@ dependencies = [ "predicates", "ring", "rusqlite", + "serde", "serde_json", "sha2", "tempfile", diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index 2b7ed2b28..d4f55aa96 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -22,6 +22,7 @@ chrono.workspace = true clap.workspace = true clap_mangen.workspace = true ring.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true ureq = { version = "2.10", default-features = true } diff --git a/crates/ctx-cli/src/docs.rs b/crates/ctx-cli/src/docs.rs index 74b0637b4..55aae6e26 100644 --- a/crates/ctx-cli/src/docs.rs +++ b/crates/ctx-cli/src/docs.rs @@ -223,6 +223,15 @@ const TOPICS: &[DocTopic] = &[ source_path: "docs/custom-history-import-format.md", body: include_str!("../../../docs/custom-history-import-format.md"), }, + DocTopic { + id: "history-source-plugins", + title: "History Source Plugins", + audience: "integrator-agent", + summary: "Local plugin manifests, stdout import, cursor handoff, and adapter shapes.", + tags: &["providers", "plugins", "imports", "custom"], + source_path: "docs/history-source-plugins.md", + body: include_str!("../../../docs/history-source-plugins.md"), + }, DocTopic { id: "provider-support", title: "Provider Support", diff --git a/crates/ctx-cli/src/history_source_plugins.rs b/crates/ctx-cli/src/history_source_plugins.rs new file mode 100644 index 000000000..50ed14527 --- /dev/null +++ b/crates/ctx-cli/src/history_source_plugins.rs @@ -0,0 +1,506 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + env, + fs::{self, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, +}; + +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; + +use anyhow::{anyhow, Context, Result}; +use serde::Deserialize; +use uuid::Uuid; + +const PLUGIN_MANIFEST_FILE: &str = "ctx-history-plugin.json"; +const LEGACY_PLUGIN_MANIFEST_FILE: &str = "plugin.json"; +const DEFAULT_PLUGIN_TIMEOUT_SECONDS: u64 = 300; +const MAX_PLUGIN_STDERR_SNIPPET_BYTES: usize = 4096; +const MAX_INLINE_CURSOR_ENV_BYTES: usize = 8192; +const SAFE_PLUGIN_ENV: &[&str] = &[ + "PATH", + "HOME", + "USER", + "LOGNAME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "TEMP", + "TMP", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_CACHE_HOME", + "XDG_STATE_HOME", +]; + +#[derive(Debug, Clone)] +pub struct HistorySourcePluginSource { + pub plugin_name: String, + pub plugin_display_name: Option, + pub plugin_version: Option, + pub manifest_path: PathBuf, + pub manifest_dir: PathBuf, + pub id: String, + pub display_name: Option, + pub provider_key: String, + pub source_id: String, + pub source_format: String, + pub command: Vec, + pub working_dir: Option, + pub env: BTreeMap, + pub enabled: bool, + pub refresh: HistorySourcePluginRefresh, + pub timeout: Duration, +} + +impl HistorySourcePluginSource { + pub fn label(&self) -> String { + format!("{}/{}", self.plugin_name, self.id) + } + + pub fn cursor_stream(&self) -> String { + ctx_history_capture::custom_history_jsonl_v1_cursor_stream( + &self.provider_key, + &self.source_id, + &self.source_format, + ) + } + + pub fn matches_selector(&self, selector: &str) -> bool { + selector == self.plugin_name + || selector == self.id + || selector == self.label() + || selector == self.provider_key + || selector == format!("{}/{}", self.provider_key, self.source_id) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HistorySourcePluginRefresh { + Manual, + Auto, +} + +impl Default for HistorySourcePluginRefresh { + fn default() -> Self { + Self::Manual + } +} + +#[derive(Debug, Clone)] +pub struct HistorySourcePluginRun { + pub stdout: Vec, + pub stderr: String, +} + +#[derive(Debug, Clone)] +pub struct HistorySourcePluginRunOptions<'a> { + pub data_root: &'a Path, + pub machine_id: &'a str, + pub cursor: Option<&'a str>, + pub cursor_stream: &'a str, + pub full_rescan: bool, +} + +#[derive(Debug, Deserialize)] +struct HistorySourcePluginManifest { + schema_version: u32, + name: String, + #[serde(default)] + display_name: Option, + #[serde(default)] + version: Option, + #[serde(default)] + history_sources: Vec, +} + +#[derive(Debug, Deserialize)] +struct HistorySourcePluginSourceManifest { + id: String, + #[serde(default)] + display_name: Option, + #[serde(default)] + provider_key: Option, + #[serde(default)] + source_id: Option, + source_format: String, + command: Vec, + #[serde(default)] + working_dir: Option, + #[serde(default)] + env: BTreeMap, + #[serde(default)] + enabled: bool, + #[serde(default)] + refresh: HistorySourcePluginRefresh, + #[serde(default)] + timeout_seconds: Option, +} + +pub fn discover_history_source_plugins( + data_root: &Path, + extra_manifests: &[PathBuf], +) -> Result> { + let mut sources = Vec::new(); + for manifest_path in plugin_manifest_paths(data_root) { + match read_plugin_manifest(&manifest_path) { + Ok(mut manifest_sources) => sources.append(&mut manifest_sources), + Err(_) => continue, + } + } + for manifest_path in explicit_plugin_manifest_paths(extra_manifests)? { + let mut manifest_sources = read_plugin_manifest(&manifest_path)?; + sources.append(&mut manifest_sources); + } + sources.sort_by(|left, right| left.label().cmp(&right.label())); + Ok(sources) +} + +pub fn run_history_source_plugin( + source: &HistorySourcePluginSource, + options: HistorySourcePluginRunOptions<'_>, +) -> Result { + let (program, args) = source.command.split_first().ok_or_else(|| { + anyhow!( + "history source plugin {} has an empty command", + source.label() + ) + })?; + let mut command = Command::new(program); + command.env_clear(); + inherit_safe_plugin_env(&mut command); + command.args(args); + command.stdin(Stdio::null()); + command.stdout(Stdio::piped()); + command.stderr(Stdio::piped()); + if let Some(working_dir) = &source.working_dir { + command.current_dir(resolve_manifest_path(&source.manifest_dir, working_dir)); + } + for (key, value) in &source.env { + command.env(key, value); + } + command.env("CTX_DATA_ROOT", options.data_root); + command.env("CTX_HISTORY_PLUGIN", "1"); + command.env("CTX_HISTORY_PLUGIN_NAME", &source.plugin_name); + command.env("CTX_HISTORY_PLUGIN_MANIFEST", &source.manifest_path); + command.env("CTX_HISTORY_SOURCE", source.label()); + command.env("CTX_HISTORY_SOURCE_ID", &source.source_id); + command.env("CTX_HISTORY_PROVIDER_KEY", &source.provider_key); + command.env("CTX_HISTORY_SOURCE_FORMAT", &source.source_format); + command.env("CTX_HISTORY_CURSOR_STREAM", options.cursor_stream); + command.env("CTX_HISTORY_MACHINE_ID", options.machine_id); + command.env( + "CTX_HISTORY_FULL_RESCAN", + if options.full_rescan { "1" } else { "0" }, + ); + let cursor_file = if let Some(cursor) = options.cursor { + let path = write_private_temp_file("ctx-history-cursor", cursor).with_context(|| { + format!("write history source plugin {} cursor file", source.label()) + })?; + if cursor.len() <= MAX_INLINE_CURSOR_ENV_BYTES { + command.env("CTX_HISTORY_CURSOR", cursor); + command.env("CTX_HISTORY_CURSOR_JSON", cursor); + } else { + command.env_remove("CTX_HISTORY_CURSOR"); + command.env_remove("CTX_HISTORY_CURSOR_JSON"); + } + command.env("CTX_HISTORY_CURSOR_FILE", &path); + Some(path) + } else { + command.env_remove("CTX_HISTORY_CURSOR"); + command.env_remove("CTX_HISTORY_CURSOR_JSON"); + command.env_remove("CTX_HISTORY_CURSOR_FILE"); + None + }; + let mut child = match command.spawn() { + Ok(child) => child, + Err(err) => { + cleanup_cursor_file(cursor_file.as_ref()); + return Err(err).with_context(|| { + format!( + "spawn history source plugin {} command {}", + source.label(), + shell_like_command(&source.command) + ) + }); + } + }; + let mut stdout = child + .stdout + .take() + .context("history source plugin stdout was not piped")?; + let mut stderr = child + .stderr + .take() + .context("history source plugin stderr was not piped")?; + let stdout_handle = thread::spawn(move || { + let mut bytes = Vec::new(); + stdout.read_to_end(&mut bytes).map(|_| bytes) + }); + let stderr_handle = thread::spawn(move || { + let mut bytes = Vec::new(); + stderr.read_to_end(&mut bytes).map(|_| bytes) + }); + + let started = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait()? { + break status; + } + if started.elapsed() >= source.timeout { + let _ = child.kill(); + let _ = child.wait(); + cleanup_cursor_file(cursor_file.as_ref()); + return Err(anyhow!( + "history source plugin {} timed out after {}s", + source.label(), + source.timeout.as_secs() + )); + } + thread::sleep(Duration::from_millis(25)); + }; + + let stdout = stdout_handle + .join() + .map_err(|_| anyhow!("history source plugin stdout reader panicked"))??; + let stderr = stderr_handle + .join() + .map_err(|_| anyhow!("history source plugin stderr reader panicked"))??; + cleanup_cursor_file(cursor_file.as_ref()); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if !status.success() { + let detail = if stderr.is_empty() { + format!("exit status {status}") + } else { + format!("exit status {status}: {}", stderr_snippet(&stderr)) + }; + return Err(anyhow!( + "history source plugin {} failed: {detail}", + source.label() + )); + } + Ok(HistorySourcePluginRun { stdout, stderr }) +} + +fn inherit_safe_plugin_env(command: &mut Command) { + for key in SAFE_PLUGIN_ENV { + if let Some(value) = env::var_os(key) { + command.env(key, value); + } + } +} + +fn write_private_temp_file(prefix: &str, contents: &str) -> Result { + for _ in 0..16 { + let path = env::temp_dir().join(format!("{prefix}-{}.json", Uuid::new_v4())); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + match options.open(&path) { + Ok(mut file) => { + file.write_all(contents.as_bytes())?; + return Ok(path); + } + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(err) => { + return Err(err) + .with_context(|| format!("create private temp file {}", path.display())); + } + } + } + Err(anyhow!("failed to allocate unique private temp file")) +} + +fn cleanup_cursor_file(path: Option<&PathBuf>) { + if let Some(path) = path { + let _ = fs::remove_file(path); + } +} + +fn read_plugin_manifest(path: &Path) -> Result> { + let raw = fs::read_to_string(path) + .with_context(|| format!("read history source plugin manifest {}", path.display()))?; + let manifest: HistorySourcePluginManifest = serde_json::from_str(&raw) + .with_context(|| format!("parse history source plugin manifest {}", path.display()))?; + validate_plugin_id("plugin name", &manifest.name)?; + if manifest.schema_version != 1 { + return Err(anyhow!( + "history source plugin manifest {} has unsupported schema_version {}; expected 1", + path.display(), + manifest.schema_version + )); + } + let manifest_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let mut sources = Vec::new(); + for source in manifest.history_sources { + validate_plugin_id("history source id", &source.id)?; + let provider_key = source.provider_key.unwrap_or_else(|| manifest.name.clone()); + validate_plugin_id("provider_key", &provider_key)?; + let source_id = source.source_id.unwrap_or_else(|| source.id.clone()); + if source.source_format.trim().is_empty() { + return Err(anyhow!( + "history source plugin manifest {} source {} has empty source_format", + path.display(), + source.id + )); + } + if source.command.is_empty() || source.command.iter().any(|part| part.trim().is_empty()) { + return Err(anyhow!( + "history source plugin manifest {} source {} has empty command", + path.display(), + source.id + )); + } + sources.push(HistorySourcePluginSource { + plugin_name: manifest.name.clone(), + plugin_display_name: manifest.display_name.clone(), + plugin_version: manifest.version.clone(), + manifest_path: path.to_path_buf(), + manifest_dir: manifest_dir.clone(), + id: source.id, + display_name: source.display_name, + provider_key, + source_id, + source_format: source.source_format, + command: source.command, + working_dir: source.working_dir, + env: source.env, + enabled: source.enabled, + refresh: source.refresh, + timeout: Duration::from_secs( + source + .timeout_seconds + .unwrap_or(DEFAULT_PLUGIN_TIMEOUT_SECONDS) + .max(1), + ), + }); + } + Ok(sources) +} + +fn plugin_manifest_paths(data_root: &Path) -> Vec { + let mut candidates = BTreeSet::new(); + collect_manifest_path_candidates(&data_root.join("plugins"), &mut candidates); + if let Some(paths) = env::var_os("CTX_HISTORY_PLUGIN_PATH") { + for path in env::split_paths(&paths) { + collect_manifest_path_candidates(&path, &mut candidates); + } + } + if let Some(paths) = env::var_os("CTX_PLUGIN_PATH") { + for path in env::split_paths(&paths) { + collect_manifest_path_candidates(&path, &mut candidates); + } + } + candidates.into_iter().collect() +} + +fn explicit_plugin_manifest_paths(extra_manifests: &[PathBuf]) -> Result> { + let mut candidates = BTreeSet::new(); + for path in extra_manifests { + let before = candidates.len(); + collect_manifest_path_candidates(path, &mut candidates); + if candidates.len() == before { + return Err(anyhow!( + "history source plugin manifest path {} did not contain {}", + path.display(), + PLUGIN_MANIFEST_FILE + )); + } + } + Ok(candidates.into_iter().collect()) +} + +fn collect_manifest_path_candidates(path: &Path, candidates: &mut BTreeSet) { + if path.is_file() { + candidates.insert(path.to_path_buf()); + return; + } + if !path.is_dir() { + return; + } + let direct = path.join(PLUGIN_MANIFEST_FILE); + if direct.is_file() { + candidates.insert(direct); + } + let legacy = path.join(LEGACY_PLUGIN_MANIFEST_FILE); + if legacy.is_file() { + candidates.insert(legacy); + } + let Ok(entries) = fs::read_dir(path) else { + return; + }; + for entry in entries.flatten() { + let child = entry.path(); + if child.is_file() + && child + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name == PLUGIN_MANIFEST_FILE) + { + candidates.insert(child); + continue; + } + if child.is_dir() { + let manifest = child.join(PLUGIN_MANIFEST_FILE); + if manifest.is_file() { + candidates.insert(manifest); + } + } + } +} + +fn validate_plugin_id(label: &str, value: &str) -> Result<()> { + let valid = !value.is_empty() + && value.len() <= 128 + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'.' | b'_' | b'-') + }) + && value + .bytes() + .next() + .is_some_and(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()); + if valid { + Ok(()) + } else { + Err(anyhow!( + "{label} must be 1 to 128 bytes, start with a lowercase ASCII letter or digit, and use only lowercase ASCII letters, digits, '.', '_', or '-'" + )) + } +} + +fn resolve_manifest_path(manifest_dir: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + manifest_dir.join(path) + } +} + +fn shell_like_command(command: &[String]) -> String { + command.join(" ") +} + +fn stderr_snippet(value: &str) -> String { + let mut snippet = value + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .take(12) + .collect::>() + .join(" | "); + if snippet.len() > MAX_PLUGIN_STDERR_SNIPPET_BYTES { + snippet.truncate(MAX_PLUGIN_STDERR_SNIPPET_BYTES); + snippet.push_str("..."); + } + snippet +} diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index c10a804c2..e16e79d22 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -1,6 +1,6 @@ use std::{ env, fs, - io::{IsTerminal, Read, Write}, + io::{Cursor, IsTerminal, Read, Write}, path::{Path, PathBuf}, str::FromStr, sync::{Arc, Mutex}, @@ -18,6 +18,7 @@ use uuid::Uuid; mod analytics; mod config; mod docs; +mod history_source_plugins; mod identity; mod mcp; mod net; @@ -31,11 +32,13 @@ use ctx_history_capture::{ import_codex_history_jsonl, import_codex_session_jsonl, import_codex_session_jsonl_tail, import_codex_session_paths, import_codex_session_tree, import_copilot_cli_session_events, import_cursor_native_history, import_custom_history_jsonl_v1, - import_factory_ai_droid_sessions, import_gemini_cli_history, import_hermes_sqlite, - import_nanoclaw_project, import_openclaw_history, import_opencode_sqlite, - import_pi_session_jsonl, provider_source_for_path, provider_source_spec, stable_capture_uuid, - validate_custom_history_jsonl_v1, AntigravityCliImportOptions, AstrBotSqliteImportOptions, - CatalogSummary, ClaudeProjectsImportOptions, CodexEventImportMode, CodexHistoryImportOptions, + import_custom_history_jsonl_v1_reader, import_factory_ai_droid_sessions, + import_gemini_cli_history, import_hermes_sqlite, import_nanoclaw_project, + import_openclaw_history, import_opencode_sqlite, import_pi_session_jsonl, + provider_source_for_path, provider_source_spec, stable_capture_uuid, + validate_custom_history_jsonl_v1, validate_custom_history_jsonl_v1_reader, + AntigravityCliImportOptions, AstrBotSqliteImportOptions, CatalogSummary, + ClaudeProjectsImportOptions, CodexEventImportMode, CodexHistoryImportOptions, CodexSessionCatalogOptions, CodexSessionImportOptions, CodexSessionImportProgress, CodexSessionImportProgressCallback, CodexToolOutputMode, CopilotCliImportOptions, CursorNativeImportOptions, CustomHistoryJsonlV1ImportOptions, FactoryAiDroidImportOptions, @@ -45,8 +48,8 @@ use ctx_history_capture::{ }; use ctx_history_core::{ database_path, default_data_root, utc_now, CaptureProvider, ContextCitation, - ContextCitationType, Event, EventRole, EventType, HistoryRecord, ProviderRawRetention, - RedactionState, Session, + ContextCitationType, CtxHistoryJsonlRecord, Event, EventRole, EventType, HistoryRecord, + ProviderRawRetention, RedactionState, Session, }; use ctx_history_store::{ CatalogSession, CatalogSourceIndexUpdate, RawSqlOptions, RawSqlResult, RawSqlValue, @@ -54,6 +57,10 @@ use ctx_history_store::{ RAW_SQL_DEFAULT_MAX_ROWS, RAW_SQL_DEFAULT_MAX_SQL_BYTES, RAW_SQL_DEFAULT_MAX_VALUE_BYTES, RAW_SQL_MAX_TIMEOUT, }; +use history_source_plugins::{ + discover_history_source_plugins, run_history_source_plugin, HistorySourcePluginRefresh, + HistorySourcePluginRunOptions, HistorySourcePluginSource, +}; const WAL_TRUNCATE_MIN_BYTES: u64 = 64 * 1024 * 1024; const LARGE_IMPORT_SOURCE_FILES_WARNING: usize = 10_000; @@ -127,14 +134,28 @@ struct ImportArgs { provider: Option, #[arg(long)] path: Option, + #[arg( + long = "history-source", + alias = "plugin", + conflicts_with_all = ["provider", "path", "format", "all"] + )] + history_source: Option, + #[arg( + long = "history-source-manifest", + alias = "plugin-manifest", + conflicts_with_all = ["provider", "path", "format"] + )] + history_source_manifest: Vec, + #[arg(long = "reset-cursor")] + reset_cursor: bool, #[arg( long, value_enum, requires = "path", - conflicts_with_all = ["provider", "all"] + conflicts_with_all = ["provider", "all", "history_source"] )] format: Option, - #[arg(long, conflicts_with_all = ["provider", "path", "format"])] + #[arg(long, conflicts_with_all = ["provider", "path", "format", "history_source"])] all: bool, #[arg(long)] resume: bool, @@ -691,6 +712,7 @@ struct ImportRunOptions { json: bool, print_human: bool, allow_empty_sources: bool, + include_history_source_plugins: bool, operation: &'static str, } @@ -1246,7 +1268,9 @@ fn main() -> Result<()> { let result = match cli.command { CommandRoot::Setup(args) => run_setup(args, data_root.clone(), &mut analytics_properties), CommandRoot::Status(args) => run_status(args, data_root.clone(), &mut analytics_properties), - CommandRoot::Sources(args) => run_sources(args, &mut analytics_properties), + CommandRoot::Sources(args) => { + run_sources(args, data_root.clone(), &mut analytics_properties) + } CommandRoot::Import(args) => run_import(args, data_root.clone(), &mut analytics_properties), CommandRoot::Show(args) => run_show(args, data_root.clone(), &mut analytics_properties), CommandRoot::Locate(args) => run_locate(args, data_root.clone(), &mut analytics_properties), @@ -1299,6 +1323,8 @@ fn command_analytics_properties(command: &CommandRoot) -> AnalyticsProperties { "source_mode", if args.format.is_some() { "explicit_format" + } else if args.history_source.is_some() { + "history_source_plugin" } else if args.path.is_some() { "explicit_path" } else if args.all { @@ -1316,6 +1342,7 @@ fn command_analytics_properties(command: &CommandRoot) -> AnalyticsProperties { provider.capture_provider().as_str(), ); } + analytics::insert_bool(&mut properties, "reset_cursor", args.reset_cursor); analytics::insert_str( &mut properties, "progress_mode", @@ -1470,6 +1497,9 @@ fn run_setup( let import_args = ImportArgs { provider: None, path: None, + history_source: None, + history_source_manifest: Vec::new(), + reset_cursor: false, format: None, all: true, resume: false, @@ -1485,6 +1515,7 @@ fn run_setup( json: args.json, print_human: !args.json, allow_empty_sources: true, + include_history_source_plugins: false, operation: "setup", }, )?) @@ -1728,8 +1759,13 @@ fn run_status( Ok(()) } -fn run_sources(args: JsonArgs, analytics_properties: &mut AnalyticsProperties) -> Result<()> { +fn run_sources( + args: JsonArgs, + data_root: PathBuf, + analytics_properties: &mut AnalyticsProperties, +) -> Result<()> { let sources = discovered_sources(); + let plugin_sources = discover_history_source_plugins(&data_root, &[])?; let existing = sources.iter().filter(|source| source.exists).count(); let importable = sources .iter() @@ -1742,7 +1778,7 @@ fn run_sources(args: JsonArgs, analytics_properties: &mut AnalyticsProperties) - analytics::insert_count_bucket( analytics_properties, "providers_detected_bucket", - sources.len() as u64, + sources.len().saturating_add(plugin_sources.len()) as u64, ); analytics::insert_count_bucket( analytics_properties, @@ -1755,9 +1791,11 @@ fn run_sources(args: JsonArgs, analytics_properties: &mut AnalyticsProperties) - importable as u64, ); if args.json { + let mut source_values = sources_json(&sources); + source_values.extend(plugin_sources_json(&plugin_sources)); print_json(json!({ "schema_version": 1, - "sources": sources_json(&sources), + "sources": source_values, }))?; } else { for source in sources { @@ -1769,6 +1807,13 @@ fn run_sources(args: JsonArgs, analytics_properties: &mut AnalyticsProperties) - source.source_format ); } + for source in plugin_sources { + println!( + "custom {} available (history-source-plugin:{})", + source.label(), + source.source_format + ); + } } Ok(()) } @@ -1830,6 +1875,7 @@ fn run_import( json, print_human: !json, allow_empty_sources: false, + include_history_source_plugins: true, operation: "import", }, )?; @@ -1844,7 +1890,7 @@ fn run_import_internal( ) -> Result { fs::create_dir_all(&data_root)?; config::write_default_config(&data_root)?; - let db_path = database_path(data_root); + let db_path = database_path(data_root.clone()); let mut store = Store::open(&db_path)?; let mut totals = ImportTotals::default(); let mut imported_sources = Vec::new(); @@ -1861,12 +1907,17 @@ fn run_import_internal( } let requests = import_requests(args)?; - if requests.is_empty() { + let plugin_requests = history_source_plugin_import_requests( + args, + &data_root, + options.include_history_source_plugins, + )?; + if requests.is_empty() && plugin_requests.is_empty() { if options.allow_empty_sources { return Ok(ImportReport::empty(args.resume)); } return Err(anyhow!( - "no importable provider history sources found; use --path or run `ctx sources`" + "no importable provider history sources found; use --path, --history-source, or run `ctx sources`" )); } @@ -1881,7 +1932,7 @@ fn run_import_internal( analytics::insert_count_bucket( analytics_properties, "sources_seen_bucket", - planned_sources.len() as u64, + planned_sources.len().saturating_add(plugin_requests.len()) as u64, ); analytics::insert_bytes_bucket( analytics_properties, @@ -1900,7 +1951,7 @@ fn run_import_internal( "discovering", format!( "found {} import source(s), {}", - planned_sources.len(), + planned_sources.len().saturating_add(plugin_requests.len()), format_bytes(planned_total_bytes) ), ); @@ -1911,6 +1962,64 @@ fn run_import_internal( progress.warning(warning); } + for plugin_source in plugin_requests { + if options.print_human { + progress.finish_line(); + println!("importing history source plugin {}", plugin_source.label()); + } + progress.message( + "indexing", + format!("running history source plugin {}", plugin_source.label()), + ); + match import_history_source_plugin( + &mut store, + &plugin_source, + &data_root, + args.resume || args.reset_cursor, + ) { + Ok((summary, stats)) => { + totals.add(&summary, &stats); + progress.done( + "indexing", + format!("imported history source plugin {}", plugin_source.label()), + planned_total_bytes, + ); + if options.print_human { + progress.finish_line(); + print_history_source_plugin_imported(&plugin_source, &summary); + } + imported_sources.push(history_source_plugin_import_json( + &plugin_source, + &stats, + &summary, + )); + } + Err(err) => { + let error = error_summary(&err); + if allow_source_failures && !import_error_is_systemic(&error) { + totals.add_source_failure(&SourceStats::default()); + progress.done( + "indexing", + format!( + "skipped history source plugin {}: {}", + plugin_source.label(), + one_line_error(&error) + ), + planned_total_bytes, + ); + if options.print_human { + progress.finish_line(); + print_history_source_plugin_failed(&plugin_source, &error); + } + imported_sources + .push(history_source_plugin_failure_json(&plugin_source, &error)); + } else { + return Err(err); + } + } + } + } + if should_parallelize_import(&planned_sources) { let final_refresh_required = store.event_search_projection_needs_backfill()? || planned_sources @@ -2185,7 +2294,12 @@ fn run_import_internal( ); analytics::insert_count_bucket(analytics_properties, "failed_bucket", totals.failed as u64); if totals.imported_sources == 0 && totals.failed_sources > 0 { - return Err(anyhow!("all import sources failed")); + let detail = imported_sources + .iter() + .find_map(|source| source.get("error").and_then(Value::as_str)) + .map(|error| format!("; first failure: {error}")) + .unwrap_or_default(); + return Err(anyhow!("all import sources failed{detail}")); } Ok(ImportReport { resume: args.resume, @@ -2483,6 +2597,32 @@ fn custom_format_import_json( }) } +fn history_source_plugin_import_json( + source: &HistorySourcePluginSource, + stats: &SourceStats, + summary: &ProviderImportSummary, +) -> Value { + json!({ + "status": "imported", + "provider": CaptureProvider::Custom.as_str(), + "kind": "history_source_plugin", + "plugin": source.plugin_name, + "history_source": source.label(), + "provider_key": source.provider_key, + "source_id": source.source_id, + "source_format": source.source_format, + "manifest_path": source.manifest_path, + "source_files": stats.files, + "source_bytes": stats.bytes, + "imported_sessions": summary.imported_sessions, + "imported_events": summary.imported_events, + "imported_edges": summary.imported_edges, + "skipped": summary.skipped, + "failed": summary.failed, + "failures": provider_failures_json(summary), + }) +} + fn provider_failures_json(summary: &ProviderImportSummary) -> Vec { summary .failures @@ -2509,6 +2649,23 @@ fn source_failure_json(failure: &ImportSourceFailure) -> Value { }) } +fn history_source_plugin_failure_json(source: &HistorySourcePluginSource, error: &str) -> Value { + json!({ + "status": "failed", + "provider": CaptureProvider::Custom.as_str(), + "kind": "history_source_plugin", + "plugin": source.plugin_name, + "history_source": source.label(), + "provider_key": source.provider_key, + "source_id": source.source_id, + "source_format": source.source_format, + "manifest_path": source.manifest_path, + "source_files": 0, + "source_bytes": 0, + "error": one_line_error(error), + }) +} + fn print_source_imported(source: &SourceInfo, summary: &ProviderImportSummary) { println!( "imported {}: sessions={} events={} edges={} skipped={} failed={}", @@ -2521,6 +2678,21 @@ fn print_source_imported(source: &SourceInfo, summary: &ProviderImportSummary) { ); } +fn print_history_source_plugin_imported( + source: &HistorySourcePluginSource, + summary: &ProviderImportSummary, +) { + println!( + "imported history source plugin {}: sessions={} events={} edges={} skipped={} failed={}", + source.label(), + summary.imported_sessions, + summary.imported_events, + summary.imported_edges, + summary.skipped, + summary.failed + ); +} + fn print_source_failed(failure: &ImportSourceFailure) { println!( "skipped {}: {}", @@ -2530,6 +2702,15 @@ fn print_source_failed(failure: &ImportSourceFailure) { println!(" path: {}", failure.source.path.display()); } +fn print_history_source_plugin_failed(source: &HistorySourcePluginSource, error: &str) { + println!( + "skipped history source plugin {}: {}", + source.label(), + one_line_error(error) + ); + println!(" manifest: {}", source.manifest_path.display()); +} + fn source_error_reason(source: &SourceInfo, error: &str) -> String { let error = one_line_error(error); let prefix = format!( @@ -4437,6 +4618,9 @@ fn run_doctor( } fn import_requests(args: &ImportArgs) -> Result> { + if args.history_source.is_some() || !args.history_source_manifest.is_empty() { + return Ok(Vec::new()); + } if let Some(path) = &args.path { let provider = args .provider @@ -4523,6 +4707,209 @@ fn no_importable_provider_sources_error( anyhow!(message) } +fn history_source_plugin_import_requests( + args: &ImportArgs, + data_root: &Path, + include_plugins: bool, +) -> Result> { + if !include_plugins { + return Ok(Vec::new()); + } + if !args.all && args.history_source.is_none() && args.history_source_manifest.is_empty() { + return Ok(Vec::new()); + } + let sources = discover_history_source_plugins(data_root, &args.history_source_manifest)?; + if let Some(selector) = &args.history_source { + let matches = sources + .into_iter() + .filter(|source| source.matches_selector(selector)) + .collect::>(); + if matches.is_empty() { + return Err(anyhow!( + "no history source plugin matched `{selector}`; use `ctx sources` to inspect configured plugins" + )); + } + if matches.len() > 1 { + let labels = matches + .iter() + .map(HistorySourcePluginSource::label) + .collect::>() + .join(", "); + return Err(anyhow!( + "history source plugin selector `{selector}` matched multiple sources ({labels}); use plugin/source or provider_key/source_id" + )); + } + return Ok(matches); + } + if args.all { + return Ok(sources + .into_iter() + .filter(|source| source.enabled) + .collect()); + } + Ok(sources + .into_iter() + .filter(|source| { + args.history_source_manifest + .iter() + .any(|path| manifest_arg_matches_source(path, &source.manifest_path)) + }) + .collect()) +} + +fn manifest_arg_matches_source(arg: &Path, manifest_path: &Path) -> bool { + if arg.is_file() { + return same_pathish(arg, manifest_path); + } + if arg.is_dir() { + return manifest_path.starts_with(arg); + } + same_pathish(arg, manifest_path) +} + +fn same_pathish(left: &Path, right: &Path) -> bool { + if left == right { + return true; + } + let left = fs::canonicalize(left).unwrap_or_else(|_| left.to_path_buf()); + let right = fs::canonicalize(right).unwrap_or_else(|_| right.to_path_buf()); + left == right +} + +fn import_history_source_plugin( + store: &mut Store, + source: &HistorySourcePluginSource, + data_root: &Path, + full_rescan: bool, +) -> Result<(ProviderImportSummary, SourceStats)> { + let record = import_record_for_history_source_plugin(source); + let record_id = record.id; + let options = CustomHistoryJsonlV1ImportOptions::default(); + let machine_id = options.machine_id.clone(); + let cursor_stream = source.cursor_stream(); + let previous_cursor = if full_rescan { + None + } else { + store + .get_sync_cursor(None, &machine_id, &cursor_stream)? + .map(|cursor| cursor.cursor) + }; + let run = run_history_source_plugin( + source, + HistorySourcePluginRunOptions { + data_root, + machine_id: &machine_id, + cursor: previous_cursor.as_deref(), + cursor_stream: &cursor_stream, + full_rescan, + }, + )?; + let _plugin_stderr = &run.stderr; + validate_history_source_plugin_output(source, &run.stdout, &machine_id)?; + let validation = validate_custom_history_jsonl_v1_reader(Cursor::new(run.stdout.as_slice())) + .map_err(anyhow::Error::from)?; + if validation.failed > 0 { + return Err(history_source_plugin_import_failure(source, &validation)); + } + let stats = SourceStats { + files: 1, + bytes: run.stdout.len() as u64, + }; + store.upsert_record(&record)?; + let summary = import_custom_history_jsonl_v1_reader( + Cursor::new(run.stdout), + store, + CustomHistoryJsonlV1ImportOptions { + machine_id, + source_path: Some(source.manifest_path.clone()), + history_record_id: Some(record_id), + allow_partial_failures: false, + ..options + }, + ) + .map_err(anyhow::Error::from)?; + if summary.failed > 0 { + return Err(history_source_plugin_import_failure(source, &summary)); + } + Ok((summary, stats)) +} + +fn validate_history_source_plugin_output( + source: &HistorySourcePluginSource, + stdout: &[u8], + machine_id: &str, +) -> Result<()> { + let text = std::str::from_utf8(stdout).with_context(|| { + format!( + "history source plugin {} emitted non-UTF-8 ctx-history-jsonl-v1 output", + source.label() + ) + })?; + let mut saw_source = false; + for (index, line) in text.lines().enumerate() { + let line_number = index + 1; + if line.trim().is_empty() { + continue; + } + let record: CtxHistoryJsonlRecord = serde_json::from_str(line).with_context(|| { + format!( + "history source plugin {} emitted invalid ctx-history-jsonl-v1 at line {line_number}", + source.label() + ) + })?; + let CtxHistoryJsonlRecord::Source(source_record) = record else { + continue; + }; + saw_source = true; + if source_record.provider_key != source.provider_key + || source_record.source_id != source.source_id + || source_record.source_format != source.source_format + { + return Err(anyhow!( + "history source plugin {} emitted source identity {}/{}/{} but manifest declares {}/{}/{}", + source.label(), + source_record.provider_key, + source_record.source_id, + source_record.source_format, + source.provider_key, + source.source_id, + source.source_format + )); + } + if let Some(source_machine_id) = source_record.machine_id { + if source_machine_id != machine_id { + return Err(anyhow!( + "history source plugin {} emitted machine_id `{source_machine_id}` but ctx is importing as `{machine_id}`; omit machine_id or set it to CTX_HISTORY_MACHINE_ID", + source.label() + )); + } + } + } + if !saw_source { + return Err(anyhow!( + "history source plugin {} emitted no source record", + source.label() + )); + } + Ok(()) +} + +fn history_source_plugin_import_failure( + source: &HistorySourcePluginSource, + summary: &ProviderImportSummary, +) -> anyhow::Error { + let detail = summary + .failures + .first() + .map(|failure| format!("line {}: {}", failure.line, failure.error)) + .unwrap_or_else(|| "unknown validation failure".to_owned()); + anyhow!( + "history source plugin {} import failed with {} failure(s); first failure: {detail}", + source.label(), + summary.failed + ) +} + fn validate_source_import_supported(source: &SourceInfo) -> Result<()> { match source.import_support { ProviderImportSupport::Native => Ok(()), @@ -5378,6 +5765,35 @@ fn import_record_for_custom_history(path: &Path, format: ImportFormatArg) -> His record } +fn import_record_for_history_source_plugin(source: &HistorySourcePluginSource) -> HistoryRecord { + let key = format!( + "history-source-plugin:{}:{}:{}:{}:{}", + source.plugin_name, source.id, source.provider_key, source.source_id, source.source_format + ); + let mut record = HistoryRecord::new( + format!("history source plugin {}", source.label()), + format!( + "Indexed custom agent history from history source plugin {} ({})", + source.label(), + source.source_format + ), + vec![ + "agent-history".into(), + "custom".into(), + "history-source-plugin".into(), + source.provider_key.clone(), + source.source_format.clone(), + ], + "agent_history", + source + .manifest_path + .parent() + .map(|path| path.display().to_string()), + ); + record.id = stable_capture_uuid(&key, "record"); + record +} + fn discovered_sources() -> Vec { home_dir() .as_deref() @@ -5421,6 +5837,43 @@ fn sources_json(sources: &[SourceInfo]) -> Vec { .collect() } +fn plugin_sources_json(sources: &[HistorySourcePluginSource]) -> Vec { + sources + .iter() + .map(|source| { + json!({ + "provider": CaptureProvider::Custom.as_str(), + "kind": "history_source_plugin", + "plugin": source.plugin_name, + "plugin_display_name": source.plugin_display_name, + "plugin_version": source.plugin_version, + "history_source": source.label(), + "history_source_id": source.id, + "display_name": source.display_name, + "provider_key": source.provider_key, + "source_id": source.source_id, + "source_format": source.source_format, + "manifest_path": source.manifest_path, + "enabled": source.enabled, + "refresh": history_source_plugin_refresh_json(source.refresh), + "status": "available", + "import_support": "history_source_plugin", + "native_import": false, + "importable": true, + "raw_retention": "metadata_only", + "unsupported_reason": null, + }) + }) + .collect() +} + +fn history_source_plugin_refresh_json(refresh: HistorySourcePluginRefresh) -> &'static str { + match refresh { + HistorySourcePluginRefresh::Manual => "manual", + HistorySourcePluginRefresh::Auto => "auto", + } +} + fn import_support_json(support: ProviderImportSupport) -> &'static str { match support { ProviderImportSupport::Native => "native", diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 48960a65d..874d5bc1a 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -36,6 +36,180 @@ fn custom_history_fixture(name: &str) -> String { materialized_fixture("custom-history-jsonl", name) } +#[derive(Debug)] +struct HistorySourcePluginFixture { + manifest_dir: PathBuf, + run_marker: PathBuf, +} + +fn write_history_source_plugin( + temp: &TempDir, + provider: &str, + enabled: bool, + cursor_log: Option<&Path>, +) -> HistorySourcePluginFixture { + write_history_source_plugin_at( + &temp.path().join("history-plugins"), + provider, + enabled, + cursor_log, + ) +} + +fn write_history_source_plugin_at( + root: &Path, + provider: &str, + enabled: bool, + cursor_log: Option<&Path>, +) -> HistorySourcePluginFixture { + let manifest_dir = root.join(provider); + fs::create_dir_all(&manifest_dir).unwrap(); + let script = manifest_dir.join("export.py"); + let run_marker = manifest_dir.join("ran"); + let run_marker_json = Value::String(run_marker.display().to_string()); + let cursor_log_py = cursor_log + .map(|path| { + serde_json::to_string(&path.display().to_string()) + .expect("cursor log path is JSON-serializable") + }) + .unwrap_or_else(|| "None".to_owned()); + let script_body = format!( + r#"#!/usr/bin/env python3 +import json +import os +import pathlib +import sys + +provider = sys.argv[1] +source_id = os.environ["CTX_HISTORY_SOURCE_ID"] +provider_key = os.environ["CTX_HISTORY_PROVIDER_KEY"] +source_format = os.environ["CTX_HISTORY_SOURCE_FORMAT"] +cursor_stream = os.environ["CTX_HISTORY_CURSOR_STREAM"] +cursor_json = os.environ.get("CTX_HISTORY_CURSOR_JSON") +cursor_file = os.environ.get("CTX_HISTORY_CURSOR_FILE") +pathlib.Path({run_marker_json}).write_text("ran\n") +cursor_log = {cursor_log_py} +cursor_text = cursor_json +if not cursor_text and cursor_file: + cursor_text = pathlib.Path(cursor_file).read_text() +if cursor_log and cursor_text: + file_text = pathlib.Path(cursor_file).read_text() if cursor_file else "" + with open(cursor_log, "a", encoding="utf-8") as handle: + handle.write(cursor_text + "\n") + handle.write("cursor_file=" + file_text + "\n") + +cursor_shapes = {{ + "dorkos": {{"files": {{"/tmp/dorkos.jsonl": {{"offset": 128, "size": 128, "mtimeMs": 1}}}}}}, + "disabled-dorkos": {{"files": {{"/tmp/disabled-dorkos.jsonl": {{"offset": 128, "size": 128, "mtimeMs": 1}}}}}}, + "openclaw": {{"backend": "openclaw-file", "transcripts": {{"/tmp/openclaw.jsonl": {{"offset": 256, "size": 256, "lastRecordId": "rec-1"}}}}}}, + "hermes": {{"message_id": 7}}, + "nanoclaw": {{"sessions": {{"sess-1": 42}}}}, +}} +next_cursor = cursor_shapes[provider] +if cursor_text: + if provider == "hermes": + next_cursor = {{"message_id": 8}} + elif provider == "nanoclaw": + next_cursor = {{"sessions": {{"sess-1": 44}}}} + elif provider == "openclaw": + next_cursor = {{"backend": "openclaw-file", "transcripts": {{"/tmp/openclaw.jsonl": {{"offset": 512, "size": 512, "lastRecordId": "rec-2"}}}}}} + else: + next_cursor = {{"files": {{"/tmp/" + provider + ".jsonl": {{"offset": 256, "size": 256, "mtimeMs": 2}}}}}} + +event_index = 1 if cursor_text else 0 +phase = "incremental" if cursor_text else "initial" +observed = "2026-07-01T12:00:00Z" +cursor = {{ + "after": {{ + "stream": cursor_stream, + "cursor": json.dumps(next_cursor, separators=(",", ":")), + "observed_at": observed, + }} +}} +if cursor_text: + cursor["before"] = {{ + "stream": cursor_stream, + "cursor": cursor_text, + "observed_at": observed, + }} + +records = [ + {{"record_type": "manifest", "schema_version": "ctx-history-jsonl-v1", "producer": provider + "-fixture"}}, + {{"record_type": "source", "source_id": source_id, "provider_key": provider_key, "source_format": source_format, "observed_at": observed, "cursor": cursor, "metadata": {{"fixture_provider": provider}}}}, + {{"record_type": "session", "source_id": source_id, "session_id": provider + "-session", "started_at": "2026-07-01T11:59:00Z", "cwd": "/workspace/" + provider, "agent_type": "primary", "is_primary": True, "status": "completed"}}, + {{"record_type": "event", "source_id": source_id, "session_id": provider + "-session", "event_index": event_index, "event_id": provider + "-event-" + str(event_index), "native_cursor": phase, "event_type": "message", "role": "assistant", "occurred_at": observed, "payload": {{"text": provider + " plugin " + phase + " marker"}}, "preview": provider + " plugin " + phase + " marker"}}, +] +for record in records: + print(json.dumps(record, separators=(",", ":"))) +"#, + run_marker_json = run_marker_json, + cursor_log_py = cursor_log_py + ); + fs::write(&script, script_body).unwrap(); + let manifest = json!({ + "schema_version": 1, + "name": provider, + "display_name": format!("{provider} history"), + "version": "0.1.0", + "history_sources": [{ + "id": "default", + "provider_key": provider, + "source_id": "default", + "source_format": format!("{provider}-history-v1"), + "enabled": enabled, + "command": [python_command(), script.display().to_string(), provider], + "timeout_seconds": 10 + }] + }); + fs::write( + manifest_dir.join("ctx-history-plugin.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + HistorySourcePluginFixture { + manifest_dir, + run_marker, + } +} + +fn python_command() -> String { + std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()) +} + +fn write_raw_history_source_plugin( + temp: &TempDir, + provider: &str, + script_body: &str, +) -> HistorySourcePluginFixture { + let manifest_dir = temp.path().join("history-plugins").join(provider); + fs::create_dir_all(&manifest_dir).unwrap(); + let script = manifest_dir.join("export.py"); + let run_marker = manifest_dir.join("ran"); + fs::write(&script, script_body).unwrap(); + let manifest = json!({ + "schema_version": 1, + "name": provider, + "history_sources": [{ + "id": "default", + "provider_key": provider, + "source_id": "default", + "source_format": format!("{provider}-history-v1"), + "enabled": false, + "command": [python_command(), script.display().to_string()], + "timeout_seconds": 10 + }] + }); + fs::write( + manifest_dir.join("ctx-history-plugin.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + HistorySourcePluginFixture { + manifest_dir, + run_marker, + } +} + fn redaction_fixture(name: &str) -> String { materialized_fixture("redaction", name) } @@ -857,6 +1031,335 @@ fn import_custom_history_format_is_not_a_native_provider_importer() { assert!(stderr.contains("--all"), "{stderr}"); } +#[test] +fn history_source_plugins_are_listed_without_running() { + let temp = tempdir(); + let plugin = write_history_source_plugin(&temp, "dorkos", false, None); + + let sources = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["sources", "--json"]), + ); + let plugin_source = sources["sources"] + .as_array() + .unwrap() + .iter() + .find(|source| source["history_source"] == "dorkos/default") + .unwrap(); + assert_eq!(plugin_source["kind"], "history_source_plugin"); + assert_eq!(plugin_source["provider_key"], "dorkos"); + assert_eq!(plugin_source["enabled"], false); + assert!(!plugin.run_marker.exists()); +} + +#[test] +fn setup_does_not_execute_enabled_history_source_plugins() { + let temp = tempdir(); + let plugin = write_history_source_plugin(&temp, "dorkos", true, None); + + json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["setup", "--json", "--progress", "none"]), + ); + + assert!(!plugin.run_marker.exists()); +} + +#[test] +fn ambiguous_history_source_plugin_selector_fails_before_execution() { + let temp = tempdir(); + let plugin_root = temp.path().join("history-plugins"); + let dorkos = write_history_source_plugin_at(&plugin_root, "dorkos", false, None); + let hermes = write_history_source_plugin_at(&plugin_root, "hermes", false, None); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) + .args([ + "import", + "--history-source", + "default", + "--progress", + "none", + ]), + ); + + assert!(stderr.contains("matched multiple sources"), "{stderr}"); + assert!(!dorkos.run_marker.exists()); + assert!(!hermes.run_marker.exists()); +} + +#[test] +fn explicit_history_source_manifest_reports_parse_errors() { + let temp = tempdir(); + let bad_manifest = temp.path().join("bad-plugin.json"); + fs::write(&bad_manifest, "{not-json").unwrap(); + + let stderr = failure_stderr(ctx(&temp).args([ + "import", + "--history-source-manifest", + bad_manifest.to_str().unwrap(), + "--progress", + "none", + ])); + + assert!( + stderr.contains("parse history source plugin manifest"), + "{stderr}" + ); +} + +#[test] +fn failed_history_source_plugin_import_does_not_leave_record_metadata() { + let temp = tempdir(); + let script = r#"#!/usr/bin/env python3 +import json +provider = "badplugin" +records = [ + {"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}, + {"record_type":"source","source_id":"default","provider_key":provider,"source_format":"badplugin-history-v1"}, + {"record_type":"event","source_id":"default","session_id":"missing","event_index":0,"event_type":"message","role":"assistant","occurred_at":"2026-07-01T12:00:00Z","preview":"should not import"} +] +for record in records: + print(json.dumps(record)) +"#; + let plugin = write_raw_history_source_plugin(&temp, "badplugin", script); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "badplugin/default", + "--progress", + "none", + ]), + ); + + assert!(stderr.contains("import failed"), "{stderr}"); + let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); + assert_eq!( + sqlite_count(&conn, "SELECT COUNT(*) FROM history_records"), + 0 + ); + assert_eq!(sqlite_count(&conn, "SELECT COUNT(*) FROM sessions"), 0); + assert_eq!(sqlite_count(&conn, "SELECT COUNT(*) FROM events"), 0); +} + +#[test] +fn history_source_plugin_rejects_mismatched_machine_id_before_import() { + let temp = tempdir(); + let script = r#"#!/usr/bin/env python3 +import json +records = [ + {"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}, + {"record_type":"source","source_id":"default","provider_key":"machineplugin","source_format":"machineplugin-history-v1","machine_id":"other-machine"}, + {"record_type":"session","source_id":"default","session_id":"run","started_at":"2026-07-01T12:00:00Z"}, +] +for record in records: + print(json.dumps(record)) +"#; + let plugin = write_raw_history_source_plugin(&temp, "machineplugin", script); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "machineplugin/default", + "--progress", + "none", + ]), + ); + + assert!(stderr.contains("machine_id"), "{stderr}"); + let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); + assert_eq!( + sqlite_count(&conn, "SELECT COUNT(*) FROM history_records"), + 0 + ); +} + +#[test] +fn large_history_source_plugin_cursor_uses_cursor_file_without_inline_env() { + let temp = tempdir(); + let log = temp.path().join("large-cursor.log"); + let log_json = serde_json::to_string(&log.display().to_string()).unwrap(); + let script = format!( + r#"#!/usr/bin/env python3 +import json +import os +import pathlib + +cursor_file = os.environ.get("CTX_HISTORY_CURSOR_FILE") +inline = os.environ.get("CTX_HISTORY_CURSOR_JSON") +cursor_text = pathlib.Path(cursor_file).read_text() if cursor_file else inline +if cursor_text: + with open({log_json}, "a", encoding="utf-8") as handle: + handle.write("inline=" + ("1" if inline else "0") + "\n") + handle.write("file_len=" + str(len(cursor_text)) + "\n") +next_cursor = "x" * 9000 if not cursor_text else "done" +observed = "2026-07-01T12:00:00Z" +records = [ + {{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}}, + {{"record_type":"source","source_id":"default","provider_key":"largecursor","source_format":"largecursor-history-v1","cursor":{{"after":{{"stream":os.environ["CTX_HISTORY_CURSOR_STREAM"],"cursor":next_cursor,"observed_at":observed}}}}}}, + {{"record_type":"session","source_id":"default","session_id":"run","started_at":"2026-07-01T12:00:00Z"}}, + {{"record_type":"event","source_id":"default","session_id":"run","event_index":1 if cursor_text else 0,"event_type":"message","role":"assistant","occurred_at":observed,"preview":"large cursor marker"}}, +] +for record in records: + print(json.dumps(record)) +"# + ); + let plugin = write_raw_history_source_plugin(&temp, "largecursor", &script); + + json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "largecursor/default", + "--json", + "--progress", + "none", + ]), + ); + json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "largecursor/default", + "--json", + "--progress", + "none", + ]), + ); + + let log = fs::read_to_string(log).unwrap(); + assert!(log.contains("inline=0"), "{log}"); + assert!(log.contains("file_len=9000"), "{log}"); +} + +#[test] +fn import_history_source_plugin_is_searchable_and_receives_cursor() { + let temp = tempdir(); + let cursor_log = temp.path().join("cursor-log.txt"); + let plugin = write_history_source_plugin(&temp, "hermes", false, Some(&cursor_log)); + + let first = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "hermes", + "--json", + "--progress", + "none", + ]), + ); + assert_eq!(first["totals"]["imported_sessions"], 1); + assert_eq!(first["totals"]["imported_events"], 1); + assert_eq!(first["sources"][0]["history_source"], "hermes/default"); + + let initial = json_output(ctx(&temp).args([ + "search", + "hermes plugin initial marker", + "--provider", + "custom", + "--refresh", + "off", + "--json", + ])); + assert!( + !initial["results"].as_array().unwrap().is_empty(), + "initial plugin import was not searchable: {initial:#}" + ); + + let second = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "hermes", + "--json", + "--progress", + "none", + ]), + ); + assert_eq!(second["totals"]["imported_sessions"], 0); + assert_eq!(second["totals"]["imported_events"], 1); + + let incremental = json_output(ctx(&temp).args([ + "search", + "hermes plugin incremental marker", + "--provider", + "custom", + "--refresh", + "off", + "--json", + ])); + assert!( + !incremental["results"].as_array().unwrap().is_empty(), + "incremental plugin import was not searchable: {incremental:#}" + ); + let cursor_log = fs::read_to_string(cursor_log).unwrap(); + assert!(cursor_log.contains(r#""message_id":7"#), "{cursor_log}"); + assert!(cursor_log.contains("cursor_file="), "{cursor_log}"); +} + +#[test] +fn import_all_runs_enabled_history_source_plugins_for_external_shapes() { + let temp = tempdir(); + let plugin_root = temp.path().join("history-plugins"); + let providers = ["dorkos", "openclaw", "hermes", "nanoclaw"]; + for provider in providers { + write_history_source_plugin_at(&plugin_root, provider, true, None); + } + write_history_source_plugin_at(&plugin_root, "disabled-dorkos", false, None); + + let imported = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) + .args(["import", "--all", "--json", "--progress", "none"]), + ); + assert_eq!(imported["totals"]["imported_sources"], 4); + assert_eq!(imported["totals"]["imported_sessions"], 4); + assert_eq!(imported["totals"]["imported_events"], 4); + let sources = imported["sources"].as_array().unwrap(); + for provider in providers { + assert!( + sources + .iter() + .any(|source| source["history_source"] == format!("{provider}/default")), + "missing import source for {provider}: {sources:#?}" + ); + let search = json_output(ctx(&temp).args([ + "search", + &format!("{provider} plugin initial marker"), + "--provider", + "custom", + "--refresh", + "off", + "--json", + ])); + assert!( + !search["results"].as_array().unwrap().is_empty(), + "{provider} plugin result was not searchable: {search:#}" + ); + } + assert!(!sources + .iter() + .any(|source| source["history_source"] == "disabled-dorkos/default")); +} + #[test] fn import_all_discovers_and_imports_providers_together() { let temp = tempdir(); diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 6bceead7e..ab6408270 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -221,7 +221,7 @@ impl Default for CustomHistoryJsonlV1ImportOptions { Self { machine_id: default_machine_id(), source_path: None, - imported_at: Utc::now(), + imported_at: utc_now(), history_record_id: None, allow_partial_failures: false, } @@ -1909,6 +1909,49 @@ pub fn import_custom_history_jsonl_v1( options.allow_partial_failures, &mut summary, )?; + import_custom_history_source_cursors(store, &normalization.source_cursors)?; + Ok(summary) +} + +pub fn import_custom_history_jsonl_v1_reader( + reader: impl BufRead, + store: &mut Store, + options: CustomHistoryJsonlV1ImportOptions, +) -> Result { + let normalization = normalize_custom_history_jsonl_v1_reader( + reader, + &ProviderAdapterContext { + machine_id: options.machine_id, + source_path: options.source_path, + imported_at: options.imported_at, + tool_output_mode: CodexToolOutputMode::Full, + event_mode: CodexEventImportMode::Rich, + include_notices: true, + }, + )?; + if normalization.provider.summary.failed > 0 && !options.allow_partial_failures { + return Ok(normalization.provider.summary); + } + + let mut summary = import_normalized_provider_captures( + store, + normalization.provider, + NormalizedProviderImportOptions { + history_record_id: options.history_record_id, + allow_partial_failures: options.allow_partial_failures, + persist_cursors: true, + wrap_transaction: true, + fast_event_inserts: true, + }, + )?; + import_custom_history_edges( + store, + &normalization.edges, + options.history_record_id, + options.allow_partial_failures, + &mut summary, + )?; + import_custom_history_source_cursors(store, &normalization.source_cursors)?; Ok(summary) } @@ -1924,6 +1967,14 @@ pub fn validate_custom_history_jsonl_v1(path: impl AsRef) -> Result Result { + let normalization = + normalize_custom_history_jsonl_v1_reader(reader, &ProviderAdapterContext::default())?; + Ok(normalization.provider.summary) +} + pub fn import_codex_history_jsonl( path: impl AsRef, store: &mut Store, @@ -3620,6 +3671,13 @@ const CODEX_FAST_IMPORT_PASSIVE_CHECKPOINT_MIN_BYTES: u64 = 2 * 1024 * 1024 * 10 struct CustomHistoryJsonlV1NormalizationResult { provider: ProviderNormalizationResult, edges: Vec<(usize, CustomHistoryJsonlV1EdgeImport)>, + source_cursors: Vec, +} + +#[derive(Debug, Clone)] +struct CustomHistoryJsonlV1SourceCursorImport { + machine_id: String, + checkpoint: ProviderCursorCheckpoint, } #[derive(Debug, Clone)] @@ -3643,6 +3701,13 @@ fn normalize_custom_history_jsonl_v1( ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; let reader = BufReader::new(file); + normalize_custom_history_jsonl_v1_reader(reader, context) +} + +fn normalize_custom_history_jsonl_v1_reader( + reader: impl BufRead, + context: &ProviderAdapterContext, +) -> Result { let mut summary = ProviderImportSummary::default(); let mut records = Vec::new(); @@ -3852,6 +3917,23 @@ fn normalize_custom_history_jsonl_v1( summary, ..ProviderNormalizationResult::default() }; + let mut source_cursors = Vec::new(); + for (_, source) in sources.values() { + let machine_id = source + .machine_id + .clone() + .unwrap_or_else(|| context.machine_id.clone()); + if let Some(after) = source + .cursor + .as_ref() + .and_then(|cursor| custom_history_normalized_cursor_range(source, cursor).after) + { + source_cursors.push(CustomHistoryJsonlV1SourceCursorImport { + machine_id, + checkpoint: after, + }); + } + } for (line_number, session) in sessions.values() { let source = &sources .get(&session.source_id) @@ -3902,6 +3984,7 @@ fn normalize_custom_history_jsonl_v1( Ok(CustomHistoryJsonlV1NormalizationResult { provider: result, edges: custom_edges, + source_cursors, }) } @@ -3914,6 +3997,7 @@ fn custom_history_failed_normalization( ..ProviderNormalizationResult::default() }, edges: Vec::new(), + source_cursors: Vec::new(), } } @@ -4340,15 +4424,27 @@ fn custom_history_internal_session_id( } fn custom_history_cursor_stream(source: &CtxHistoryJsonlSourceRecord) -> String { + custom_history_jsonl_v1_cursor_stream( + &source.provider_key, + &source.source_id, + &source.source_format, + ) +} + +pub fn custom_history_jsonl_v1_cursor_stream( + provider_key: &str, + source_id: &str, + source_format: &str, +) -> String { let key = custom_history_key(json!({ "schema": CTX_HISTORY_JSONL_V1_SCHEMA_VERSION, "kind": "cursor_stream", - "provider_key": source.provider_key, - "source_id": source.source_id, - "source_format": source.source_format, + "provider_key": provider_key, + "source_id": source_id, + "source_format": source_format, })); let stream_id = stable_capture_uuid(&key, "custom-cursor-stream"); - format!("provider:custom:{}:{stream_id}", source.provider_key) + format!("provider:custom:{provider_key}:{stream_id}") } fn custom_history_normalized_cursor_range( @@ -4491,6 +4587,32 @@ fn import_custom_history_edges( Ok(()) } +fn import_custom_history_source_cursors( + store: &mut Store, + cursors: &[CustomHistoryJsonlV1SourceCursorImport], +) -> Result<()> { + for cursor in cursors { + store.upsert_sync_cursor(&SyncCursor { + id: stable_capture_uuid( + &format!( + "provider-cursor:{}:{}:{}", + CaptureProvider::Custom.as_str(), + cursor.machine_id, + cursor.checkpoint.stream + ), + "provider-sync-cursor", + ), + team_id: None, + device_id: cursor.machine_id.clone(), + stream: cursor.checkpoint.stream.clone(), + cursor: cursor.checkpoint.cursor.clone(), + last_synced_at: Some(cursor.checkpoint.observed_at), + timestamps: timestamps(cursor.checkpoint.observed_at), + })?; + } + Ok(()) +} + fn collect_jsonl_paths(root: &Path, paths: &mut Vec) -> Result<()> { let metadata = fs::symlink_metadata(root)?; let file_type = metadata.file_type(); @@ -13602,6 +13724,77 @@ mod tests { assert_eq!(second.skipped_edges, 2); } + #[test] + fn custom_history_jsonl_reader_import_persists_normalized_cursor() { + let temp = tempdir(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let input = [ + r#"{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}"#, + r#"{"record_type":"source","source_id":"src","provider_key":"stream-agent","source_format":"stream-v1","cursor":{"after":{"stream":"native-stream","cursor":"{\"message_id\":7}","observed_at":"2026-07-01T12:00:00Z"}}}"#, + r#"{"record_type":"session","source_id":"src","session_id":"run","started_at":"2026-07-01T11:59:00Z"}"#, + r#"{"record_type":"event","source_id":"src","session_id":"run","event_index":0,"event_type":"message","role":"assistant","occurred_at":"2026-07-01T12:00:00Z","preview":"stream import marker"}"#, + ] + .join("\n"); + + let summary = import_custom_history_jsonl_v1_reader( + std::io::Cursor::new(input.into_bytes()), + &mut store, + CustomHistoryJsonlV1ImportOptions { + source_path: Some(PathBuf::from("plugin://stream-agent/default")), + imported_at: "2026-07-01T12:01:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_sessions, 1); + assert_eq!(summary.imported_events, 1); + let cursor = store + .get_sync_cursor( + None, + &CustomHistoryJsonlV1ImportOptions::default().machine_id, + &custom_history_jsonl_v1_cursor_stream("stream-agent", "src", "stream-v1"), + ) + .unwrap() + .unwrap(); + assert_eq!(cursor.cursor, r#"{"message_id":7}"#); + } + + #[test] + fn custom_history_jsonl_reader_persists_source_only_cursor() { + let temp = tempdir(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let input = [ + r#"{"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}"#, + r#"{"record_type":"source","source_id":"src","provider_key":"stream-agent","source_format":"stream-v1","cursor":{"after":{"stream":"native-stream","cursor":"{\"message_id\":9}","observed_at":"2026-07-01T12:02:00Z"}}}"#, + ] + .join("\n"); + + let summary = import_custom_history_jsonl_v1_reader( + std::io::Cursor::new(input.into_bytes()), + &mut store, + CustomHistoryJsonlV1ImportOptions { + imported_at: "2026-07-01T12:03:00Z".parse().unwrap(), + ..CustomHistoryJsonlV1ImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_sessions, 0); + assert_eq!(summary.imported_events, 0); + let cursor = store + .get_sync_cursor( + None, + &CustomHistoryJsonlV1ImportOptions::default().machine_id, + &custom_history_jsonl_v1_cursor_stream("stream-agent", "src", "stream-v1"), + ) + .unwrap() + .unwrap(); + assert_eq!(cursor.cursor, r#"{"message_id":9}"#); + } + #[test] fn custom_history_jsonl_malformed_import_is_atomic_by_default() { let temp = tempdir(); diff --git a/docs/cli-reference.md b/docs/cli-reference.md index dd9159edb..4224e261a 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -28,8 +28,9 @@ ctx doctor --json - `setup` creates the data root, opens or creates `work.sqlite`, writes `config.toml` when needed, discovers known provider history locations, - catalogs Codex sessions, imports all discovered auto-importable sources, - optimizes the local search index, and prints next steps. + catalogs Codex sessions, imports discovered native provider sources, optimizes + the local search index, and prints next steps. It does not execute + history-source plugin commands. - `setup --catalog-only` stops after discovery/cataloging. It is useful for fast inventory or troubleshooting, but it does not make history searchable. - `status` reports the ctx root, database path, config path, indexed item @@ -60,12 +61,17 @@ machine. Current rows include: - native rows for supported Antigravity, Claude, OpenCode, OpenClaw, Hermes, Gemini, Cursor, Copilot CLI, and Factory AI Droid local history locations; - preview rows for NanoClaw project roots and AstrBot SQLite history when those - paths are discoverable. - -Each JSON row includes `provider`, `path`, `exists`, `source_format`, `status`, -`import_support`, `native_import`, `importable`, `raw_retention`, and any -`unsupported_reason`. `sources` reads home-directory path metadata and writes -nothing to provider files or source repositories. + paths are discoverable; +- local history-source plugin manifests under `$CTX_DATA_ROOT/plugins` or + `CTX_HISTORY_PLUGIN_PATH`. + +Native JSON rows include `provider`, `path`, `exists`, `source_format`, +`status`, `import_support`, `native_import`, `importable`, `raw_retention`, and +any `unsupported_reason`. Plugin JSON rows use +`kind: "history_source_plugin"` and include `plugin`, `history_source`, +`provider_key`, `source_id`, `manifest_path`, and `enabled`. `sources` reads +path metadata and plugin manifests, writes nothing to provider files or source +repositories, and does not execute plugin commands. ## Import @@ -88,13 +94,19 @@ ctx import --provider factory-ai-droid ctx import --path ~/.codex/sessions ctx import --provider pi --path ~/.pi/sessions.jsonl ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl +ctx import --history-source dorkos +ctx import --plugin dorkos/default +ctx import --history-source-manifest ./ctx-history-plugin.json +ctx import --plugin-manifest ./ctx-history-plugin.json +ctx import --history-source hermes --reset-cursor ctx import --resume ctx import --json ctx import --progress json --json ``` `import` explicitly indexes provider history into the local SQLite store. The -normal first-run path is `ctx setup`, which already imports discovered sources. +normal first-run path is `ctx setup`, which already imports discovered native +provider sources. Use `import` to repair, re-run, resume, or target a specific provider/path. It creates the data root and default config if needed, reads provider transcript files, and writes indexed source metadata, sessions, events, searchable text, @@ -105,13 +117,23 @@ Custom history can be imported from an explicit JSONL file with remembered as a provider home; see `docs/custom-history-import-format.md` for the schema and incremental semantics. +History-source plugins are local command adapters that stream +`ctx-history-jsonl-v1` to stdout. Use `--history-source ` for an +explicit plugin import, or `--history-source-manifest ` to test a manifest +without installing it. `--plugin` and `--plugin-manifest` are aliases. +`--reset-cursor` withholds the previous plugin cursor for that run and asks the +plugin to perform a full rescan. See `docs/history-source-plugins.md`. + Import selection rules: -- with no arguments or with `--all`, import all discovered auto-importable - sources that exist; +- with no arguments, import discovered native sources that exist; +- with `--all`, import discovered native sources that exist and enabled + history-source plugin sources; - with `--provider`, import discovered sources for that provider; - with `--format ctx-history-jsonl-v1 --path `, import that custom history JSONL file; +- with `--history-source`, import matching local plugin sources; +- with `--history-source-manifest`, import sources from that manifest path; - with `--path`, import exactly that path; - with `--path` and no provider, parse the path as Codex format. @@ -180,11 +202,12 @@ ctx search "this current task" --include-current-session `search` defaults to `--refresh auto`, which quietly refreshes discovered native provider sources before querying indexed sessions and events. The refresh is -best-effort and keeps JSON stdout reserved for the search result object. On -large discovered sources or already-cataloged indexes, `auto` serves current -results without a foreground catch-up scan; use `--refresh strict` or -`ctx import --all` when you need a full catch-up before querying. Use -`--refresh off` to search the existing index without refreshing, or +best-effort and keeps JSON stdout reserved for the search result object. +History-source plugin commands are not executed by search refresh. On large +discovered sources or already-cataloged indexes, `auto` serves current results +without a foreground catch-up scan; use `--refresh strict` or `ctx import --all` +when you need a full catch-up before querying. Use `--refresh off` to search the +existing index without refreshing, or `--refresh strict` to fail when the pre-search refresh cannot run or import successfully. Preview native sources such as NanoClaw and AstrBot are searched from the existing index until they are explicitly imported through a supported diff --git a/docs/custom-history-import-format.md b/docs/custom-history-import-format.md index 703360754..704209db8 100644 --- a/docs/custom-history-import-format.md +++ b/docs/custom-history-import-format.md @@ -3,16 +3,23 @@ `ctx-history-jsonl-v1` is the public JSONL format for importing session history from tools without a built-in local-history adapter. -## Transport +## Transports -Version 1 uses an explicit local file path: +The same JSONL schema can be imported from an explicit local file path: ```bash ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl ``` -ctx does not discover a fixed storage location for this format. The file is not -read from stdin, and ctx does not execute exporter commands in v1. +or from a local history-source plugin command: + +```bash +ctx import --history-source my-agent +``` + +ctx does not discover a fixed storage location for this format. File imports +are explicit paths. Plugin imports are explicit local command adapters declared +by a local manifest; see `docs/history-source-plugins.md`. Each line is one JSON object. Every object has a `record_type` field with one of: @@ -203,19 +210,24 @@ Example: ## Incremental Semantics -v1 imports are explicit, local, and idempotent. On each -`ctx import --format ctx-history-jsonl-v1 --path `, ctx rescans the file -and upserts equivalent records instead of appending duplicates. +v1 imports are explicit, local, and idempotent. On each file import, ctx rescans +the file and upserts equivalent records instead of appending duplicates. On each +plugin import, ctx invokes the plugin, validates stdout atomically, and upserts +the emitted records. When a source record supplies `cursor`, ctx rewrites its storage stream under a `provider:custom::` namespace and also preserves the exporter-supplied cursor object in source metadata. Event `native_cursor` values -are also preserved. ctx does not negotiate with external exporters in v1, does -not call exporter commands, and does not request a delta range; exporter -negotiation is a follow-up capability. +are also preserved. + +For plugin imports, ctx passes the previously stored source cursor to the next +command through `CTX_HISTORY_CURSOR_JSON` and `CTX_HISTORY_CURSOR_FILE`. The +cursor string remains exporter-owned, so it can encode byte offsets, SQLite row +ids, session sequence maps, or another native high-water mark. -If an import is interrupted, run the same command again. The expected behavior -is another idempotent rescan of the same JSONL file. +If an import is interrupted, run the same command again. File imports perform +another idempotent rescan. Plugin imports receive the last successfully stored +cursor; failed plugin runs do not advance it. ## Compact Example diff --git a/docs/first-10-minutes.md b/docs/first-10-minutes.md index daf17ea10..3a34ae0b9 100644 --- a/docs/first-10-minutes.md +++ b/docs/first-10-minutes.md @@ -26,9 +26,9 @@ ctx status --json ``` `ctx setup` creates local storage, discovers supported provider history, -catalogs Codex sessions, imports discovered auto-importable sources, and -optimizes the local search index. The default root is `~/.ctx`. Use a temporary -root for trials: +catalogs Codex sessions, imports discovered native provider sources, and +optimizes the local search index. It does not execute history-source plugin +commands. The default root is `~/.ctx`. Use a temporary root for trials: ```bash ctx --data-root /tmp/ctx-first-10 setup diff --git a/docs/getting-started.md b/docs/getting-started.md index 5bfd73724..4b9584208 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -28,8 +28,9 @@ ctx status Setup creates the configured ctx data root, initializes SQLite, writes `config.toml` when missing, discovers known provider history paths, catalogs -Codex sessions, imports discovered sources, optimizes the local search index, -and prints next steps. The default data root is `~/.ctx`. +Codex sessions, imports discovered native provider sources, optimizes the local +search index, and prints next steps. It does not execute history-source plugin +commands. The default data root is `~/.ctx`. Use a different root when testing: diff --git a/docs/history-source-plugins.md b/docs/history-source-plugins.md new file mode 100644 index 000000000..66274bb40 --- /dev/null +++ b/docs/history-source-plugins.md @@ -0,0 +1,225 @@ +# History Source Plugins + +History source plugins let third-party tools make their local histories +searchable in ctx without ctx owning their storage schemas. + +The narrow waist is: + +1. A local manifest declares one or more history sources. +2. ctx invokes the declared command only during explicit import. +3. The command writes `ctx-history-jsonl-v1` records to stdout. +4. ctx validates and imports that stream atomically. +5. ctx passes the previous source cursor back on the next run. + +Plugins are command-line adapters, not an in-process ABI and not a hosted plugin +store. Plugin authors own their native JSONL, SQLite, or API reads. ctx owns the +manifest, cursor handoff, validation, import, and search index. + +## Install And Discover + +Put a manifest at one of: + +- `$CTX_DATA_ROOT/plugins//ctx-history-plugin.json`; +- any directory or manifest file listed in `CTX_HISTORY_PLUGIN_PATH`; +- any directory or manifest file listed in `CTX_PLUGIN_PATH`. + +`ctx sources` and `ctx sources --json` list plugin sources without executing +their commands. + +Manifest example: + +```json +{ + "schema_version": 1, + "name": "dorkos", + "display_name": "DorkOS history", + "version": "0.1.0", + "history_sources": [ + { + "id": "default", + "provider_key": "dorkos", + "source_id": "default", + "source_format": "dorkos-claude-jsonl-v1", + "enabled": true, + "command": ["ctx-history-source-dorkos", "export"], + "timeout_seconds": 300 + } + ] +} +``` + +`name`, `id`, `provider_key`, and `source_id` must be stable lowercase ASCII +identifiers. `command` is an argv array; ctx does not run it through a shell. + +`enabled: true` means `ctx import --all` may run that source. Explicit imports +can run a discovered source even when it is not enabled. + +## Import + +```bash +ctx import --history-source dorkos +ctx import --plugin dorkos +ctx import --history-source dorkos/default +ctx import --history-source-manifest ./ctx-history-plugin.json +ctx import --plugin-manifest ./ctx-history-plugin.json +ctx import --all +ctx import --history-source hermes --reset-cursor +``` + +Selectors can match plugin name, source id, `plugin/source`, `provider_key`, or +`provider_key/source_id`, but they must resolve to exactly one source before ctx +executes a command. Prefer `plugin/source` when a machine has multiple plugins. + +`--history-source-manifest` is a development path: it adds that manifest for the +current command without installing it. With no selector, ctx imports sources +from the supplied manifest path. + +`--reset-cursor` withholds the previous cursor and sets +`CTX_HISTORY_FULL_RESCAN=1`. The plugin should emit a fresh `source.cursor.after` +checkpoint if the rescan succeeds. + +`ctx setup` and search refresh do not execute plugins in this version. Run +`ctx import --history-source ` or `ctx import --all` to catch up +plugin-backed sources before searching. + +## Runtime Environment + +ctx sets these variables before invoking a plugin command: + +- `CTX_DATA_ROOT` +- `CTX_HISTORY_PLUGIN=1` +- `CTX_HISTORY_PLUGIN_NAME` +- `CTX_HISTORY_PLUGIN_MANIFEST` +- `CTX_HISTORY_SOURCE`, such as `dorkos/default` +- `CTX_HISTORY_SOURCE_ID` +- `CTX_HISTORY_PROVIDER_KEY` +- `CTX_HISTORY_SOURCE_FORMAT` +- `CTX_HISTORY_CURSOR_STREAM` +- `CTX_HISTORY_MACHINE_ID` +- `CTX_HISTORY_FULL_RESCAN`, `1` or `0` +- `CTX_HISTORY_CURSOR`, when a previous cursor exists and is small enough for + inline environment handoff +- `CTX_HISTORY_CURSOR_JSON`, same value as `CTX_HISTORY_CURSOR` when set +- `CTX_HISTORY_CURSOR_FILE`, a temporary file containing the cursor + +Use `CTX_HISTORY_CURSOR_FILE` for large native cursor maps. The file exists only +while the plugin process runs and is the reliable cursor handoff path. + +The plugin must write only `ctx-history-jsonl-v1` JSONL to stdout. Progress and +diagnostics belong on stderr. If the command exits nonzero or stdout is invalid, +ctx imports nothing from that run and does not advance the cursor. + +Plugin commands receive a limited inherited environment by default: `PATH`, +`HOME`, basic locale variables, temporary-directory variables, and XDG data or +config homes. Put provider-specific environment values in the manifest `env` +object instead of relying on the parent shell. + +## Cursor Contract + +The plugin controls the cursor string. It may be a number, an opaque token, or a +JSON string. ctx stores it under a stable custom stream derived from: + +- `provider_key` +- `source_id` +- `source_format` +- local machine id + +On the next import, ctx passes the stored `cursor.after.cursor` value back in +the runtime environment. This keeps native cursor design inside the provider +adapter: + +- file appenders can use byte offsets; +- SQLite stores can use row ids; +- split stores can use JSON maps keyed by session id, file path, or direction. + +Every plugin run should emit a `source` record matching the manifest +`provider_key`, `source_id`, and `source_format`. ctx rejects mismatches before +writing imported rows. + +## Adapter Shapes + +The local research checkouts showed four different storage models, which is why +ctx should not maintain native adapters for them. + +### DorkOS + +DorkOS currently derives history from Claude SDK JSONL files under +`~/.claude/projects//*.jsonl`. A DorkOS plugin should read those files by +byte offset and use a cursor like: + +```json +{"files":{"/home/me/.claude/projects/x/session.jsonl":{"offset":12345,"size":13000,"mtimeMs":1780000000000}}} +``` + +The plugin can enrich events with DorkOS metadata from `~/.dork/dork.db`, but +the transcript source is still the Claude JSONL file. + +### OpenClaw + +OpenClaw currently has session metadata under +`~/.openclaw/agents//sessions/sessions.json` and transcript JSONL +files beside it. A plugin should use OpenClaw's session accessor where possible, +resolve transcript paths, and cursor by byte offset: + +```json +{"backend":"openclaw-file","transcripts":{"/home/me/.openclaw/agents/a/sessions/s.jsonl":{"offset":456,"size":900,"lastRecordId":"rec-2"}}} +``` + +If OpenClaw flips storage to SQLite, the OpenClaw-owned plugin can keep the same +ctx stdout contract while changing its native reader. + +### Hermes + +Hermes Agent stores canonical history in `~/.hermes/state.db`. A Hermes plugin +should read `sessions` and `messages` read-only, order by `messages.id`, and +cursor by the maximum message row id: + +```json +{"message_id":1234} +``` + +Session metadata-only changes may need a second cursor if Hermes exposes a +reliable session update high-water mark. + +### NanoClaw + +NanoClaw uses a central `data/v2.db` plus per-session inbound and outbound +SQLite databases under `data/v2-sessions///`. +Inbound messages use even `seq` values and outbound messages use odd `seq` +values. A generic NanoClaw plugin can cursor by per-session sequence: + +```json +{"sessions":{"sess-abc":42,"sess-def":8}} +``` + +Provider-specific NanoClaw plugins can instead read mounted provider state, such +as Claude JSONL, when they need full internal tool/thinking events. + +## Minimal Plugin Pseudocode + +```python +import json, os, sqlite3, sys + +cursor = json.loads(os.environ.get("CTX_HISTORY_CURSOR_JSON") or "{}") +after_message_id = cursor.get("message_id", 0) +db = sqlite3.connect(os.path.expanduser("~/.hermes/state.db")) + +print(json.dumps({"record_type": "manifest", "schema_version": "ctx-history-jsonl-v1"})) +print(json.dumps({ + "record_type": "source", + "source_id": os.environ["CTX_HISTORY_SOURCE_ID"], + "provider_key": os.environ["CTX_HISTORY_PROVIDER_KEY"], + "source_format": os.environ["CTX_HISTORY_SOURCE_FORMAT"], + "cursor": { + "after": { + "stream": os.environ["CTX_HISTORY_CURSOR_STREAM"], + "cursor": json.dumps({"message_id": after_message_id}), + "observed_at": "2026-07-01T12:00:00Z" + } + } +})) + +for row in db.execute("SELECT id, session_id, role, content, timestamp FROM messages WHERE id > ? ORDER BY id", (after_message_id,)): + # Emit session records as needed, then event records with stable event_index. + pass +``` diff --git a/docs/providers.md b/docs/providers.md index ad2400269..b6d18692b 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -35,10 +35,12 @@ The current CLI imports local history for: These are built-in provider adapters for native local history. The custom history format is separate: `ctx import --format ctx-history-jsonl-v1 --path -` reads an explicit JSONL interchange file from any exporter. It is -stored internally under the bounded provider `custom` while preserving the -exporter's `provider_key`, `source_id`, and `session_id` as metadata and ID -namespace components. It is not auto-discovered by `ctx sources`. +` reads an explicit JSONL interchange file from any exporter, and +history-source plugins can stream the same format from local adapter commands. +Custom history is stored internally under the bounded provider `custom` while +preserving the exporter's `provider_key`, `source_id`, and `session_id` as +metadata and ID namespace components. File imports are not auto-discovered; +local plugin manifests are listed by `ctx sources`. Use `ctx sources` for the truth on the current machine: diff --git a/docs/search.md b/docs/search.md index aecef8532..7aab44a9c 100644 --- a/docs/search.md +++ b/docs/search.md @@ -104,13 +104,14 @@ are intentionally looking for material from the active session tree. `--refresh` defaults to `auto`. `auto` attempts a best-effort pre-search import of discovered native provider sources and serves the existing index if that -refresh fails. On large discovered sources or already-cataloged indexes, `auto` -serves current results without a foreground catch-up scan; use -`--refresh strict` or `ctx import --all` when you need a full catch-up before -querying. `off` skips the pre-search refresh. `strict` fails the search if the -refresh cannot run or import successfully. Preview native sources such as -NanoClaw and AstrBot are searched from the existing index until they are -explicitly imported through a supported path. +refresh fails. Search refresh does not execute history-source plugin commands. +On large discovered sources or already-cataloged indexes, `auto` serves current +results without a foreground catch-up scan; use `--refresh strict` or +`ctx import --all` when you need a full catch-up before querying. `off` skips +the pre-search refresh. `strict` fails the search if the refresh cannot run or +import successfully. Preview native sources such as NanoClaw and AstrBot, plus +search-only sources without native import support, are searched from the +existing index until they are explicitly imported through a supported path. Use `--refresh off` for a strictly read-only search over the existing ctx index. This avoids provider imports and avoids updating the ctx SQLite store. diff --git a/docs/storage.md b/docs/storage.md index 57fd6e542..ac24bf4df 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -76,8 +76,8 @@ analytics marker described under network behavior. | --- | --- | --- | | `ctx setup` | provider transcript files and home path metadata for source discovery | data root, `work.sqlite`, `config.toml`, and SQLite index | | `ctx status` | data root metadata and existing SQLite store | none | -| `ctx sources` | known provider paths under the user's home | none | -| `ctx import` | provider transcript files and path metadata, or the explicit custom history JSONL file passed with `--format ctx-history-jsonl-v1 --path` | data root, `config.toml` if missing, and SQLite index | +| `ctx sources` | known provider paths under the user's home and local history-source plugin manifests | none | +| `ctx import` | provider transcript files and path metadata, the explicit custom history JSONL file passed with `--format ctx-history-jsonl-v1 --path`, or stdout from an explicit history-source plugin command | data root, `config.toml` if missing, and SQLite index | | `ctx show` | SQLite index | selected `--out` path for `show session` when provided | | `ctx locate` | SQLite index and raw source path metadata | none | | `ctx search` | native provider transcript files, path metadata, and SQLite index | SQLite index for newly discovered native provider history | @@ -127,6 +127,7 @@ ctx import --all ctx import --resume ctx import --path ~/.codex/sessions ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl +ctx import --history-source dorkos ``` Current adapters are safe to re-run. They rescan sources idempotently and keep @@ -134,8 +135,10 @@ source paths or cursors when available. Custom history JSONL imports follow the same v1 lifecycle: ctx rescans the explicit file, upserts already-imported records, stores supplied source cursor metadata under ctx-owned custom cursor streams, and preserves event native -cursors. The path is not added to `config.toml` or treated as a fixed provider -location. +cursors. History-source plugins receive the previous stored cursor on each +explicit import and stream the same JSONL format to stdout. Failed plugin runs +do not advance cursors. Explicit file paths and plugin manifests are not added +to `config.toml` or treated as fixed provider homes. ## Upgrade Reindexing From ced549f46dbd2b40351c78cec265901a6e43e66f Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:12:19 -0500 Subject: [PATCH 30/72] Add search refresh for history source plugins --- crates/ctx-cli/src/history_source_plugins.rs | 246 +++++++++++-- crates/ctx-cli/src/main.rs | 58 ++- crates/ctx-cli/tests/cli.rs | 368 ++++++++++++++++++- crates/ctx-history-capture/src/lib.rs | 70 ++-- docs/cli-reference.md | 20 +- docs/first-10-minutes.md | 5 +- docs/history-source-plugins.md | 23 +- docs/product-contract.md | 9 +- docs/search.md | 23 +- 9 files changed, 698 insertions(+), 124 deletions(-) diff --git a/crates/ctx-cli/src/history_source_plugins.rs b/crates/ctx-cli/src/history_source_plugins.rs index 50ed14527..7464138fd 100644 --- a/crates/ctx-cli/src/history_source_plugins.rs +++ b/crates/ctx-cli/src/history_source_plugins.rs @@ -2,15 +2,15 @@ use std::{ collections::{BTreeMap, BTreeSet}, env, fs::{self, OpenOptions}, - io::{Read, Write}, + io::{ErrorKind, Read, Write}, path::{Path, PathBuf}, - process::{Command, Stdio}, + process::{Child, ChildStderr, ChildStdout, Command, ExitStatus, Stdio}, thread, time::{Duration, Instant}, }; #[cfg(unix)] -use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::{fs::OpenOptionsExt, io::AsRawFd}; use anyhow::{anyhow, Context, Result}; use serde::Deserialize; @@ -19,6 +19,8 @@ use uuid::Uuid; const PLUGIN_MANIFEST_FILE: &str = "ctx-history-plugin.json"; const LEGACY_PLUGIN_MANIFEST_FILE: &str = "plugin.json"; const DEFAULT_PLUGIN_TIMEOUT_SECONDS: u64 = 300; +const MAX_PLUGIN_STDOUT_BYTES: usize = 64 * 1024 * 1024; +const MAX_PLUGIN_STDERR_BYTES: usize = 256 * 1024; const MAX_PLUGIN_STDERR_SNIPPET_BYTES: usize = 4096; const MAX_INLINE_CURSOR_ENV_BYTES: usize = 8192; const SAFE_PLUGIN_ENV: &[&str] = &[ @@ -80,19 +82,14 @@ impl HistorySourcePluginSource { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] #[serde(rename_all = "snake_case")] pub enum HistorySourcePluginRefresh { + #[default] Manual, Auto, } -impl Default for HistorySourcePluginRefresh { - fn default() -> Self { - Self::Manual - } -} - #[derive(Debug, Clone)] pub struct HistorySourcePluginRun { pub stdout: Vec, @@ -158,7 +155,7 @@ pub fn discover_history_source_plugins( let mut manifest_sources = read_plugin_manifest(&manifest_path)?; sources.append(&mut manifest_sources); } - sources.sort_by(|left, right| left.label().cmp(&right.label())); + sources.sort_by_key(|source| source.label()); Ok(sources) } @@ -231,21 +228,121 @@ pub fn run_history_source_plugin( }); } }; - let mut stdout = child + let stdout = child .stdout .take() .context("history source plugin stdout was not piped")?; - let mut stderr = child + let stderr = child .stderr .take() .context("history source plugin stderr was not piped")?; + let run_result = collect_child_output_with_timeout( + &mut child, + stdout, + stderr, + source.timeout, + &source.label(), + ); + cleanup_cursor_file(cursor_file.as_ref()); + let (status, stdout, stderr) = run_result?; + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if !status.success() { + let detail = if stderr.is_empty() { + format!("exit status {status}") + } else { + format!("exit status {status}: {}", stderr_snippet(&stderr)) + }; + return Err(anyhow!( + "history source plugin {} failed: {detail}", + source.label() + )); + } + Ok(HistorySourcePluginRun { stdout, stderr }) +} + +#[cfg(unix)] +fn collect_child_output_with_timeout( + child: &mut Child, + mut stdout: ChildStdout, + mut stderr: ChildStderr, + timeout: Duration, + source_label: &str, +) -> Result<(ExitStatus, Vec, Vec)> { + set_nonblocking(stdout.as_raw_fd())?; + set_nonblocking(stderr.as_raw_fd())?; + + let started = Instant::now(); + let mut status = None; + let mut stdout_open = true; + let mut stderr_open = true; + let mut stdout_bytes = Vec::new(); + let mut stderr_bytes = Vec::new(); + loop { + if stdout_open { + read_available_with_limit( + &mut stdout, + &mut stdout_bytes, + &mut stdout_open, + MAX_PLUGIN_STDOUT_BYTES, + "stdout", + source_label, + ) + .inspect_err(|_| { + let _ = child.kill(); + let _ = child.wait(); + })?; + } + if stderr_open { + read_available_with_limit( + &mut stderr, + &mut stderr_bytes, + &mut stderr_open, + MAX_PLUGIN_STDERR_BYTES, + "stderr", + source_label, + ) + .inspect_err(|_| { + let _ = child.kill(); + let _ = child.wait(); + })?; + } + if status.is_none() { + status = child.try_wait()?; + } + if let Some(status) = status { + if !stdout_open && !stderr_open { + return Ok((status, stdout_bytes, stderr_bytes)); + } + } + if started.elapsed() >= timeout { + if status.is_none() { + let _ = child.kill(); + let _ = child.wait(); + } + return Err(anyhow!( + "history source plugin {source_label} timed out after {}s", + timeout.as_secs() + )); + } + thread::sleep(Duration::from_millis(25)); + } +} + +#[cfg(not(unix))] +fn collect_child_output_with_timeout( + child: &mut Child, + stdout: ChildStdout, + stderr: ChildStderr, + timeout: Duration, + source_label: &str, +) -> Result<(ExitStatus, Vec, Vec)> { + let stdout_source = source_label.to_owned(); let stdout_handle = thread::spawn(move || { - let mut bytes = Vec::new(); - stdout.read_to_end(&mut bytes).map(|_| bytes) + read_pipe_with_limit(stdout, MAX_PLUGIN_STDOUT_BYTES, "stdout", &stdout_source) }); + let stderr_source = source_label.to_owned(); let stderr_handle = thread::spawn(move || { - let mut bytes = Vec::new(); - stderr.read_to_end(&mut bytes).map(|_| bytes) + read_pipe_with_limit(stderr, MAX_PLUGIN_STDERR_BYTES, "stderr", &stderr_source) }); let started = Instant::now(); @@ -253,14 +350,12 @@ pub fn run_history_source_plugin( if let Some(status) = child.try_wait()? { break status; } - if started.elapsed() >= source.timeout { + if started.elapsed() >= timeout { let _ = child.kill(); let _ = child.wait(); - cleanup_cursor_file(cursor_file.as_ref()); return Err(anyhow!( - "history source plugin {} timed out after {}s", - source.label(), - source.timeout.as_secs() + "history source plugin {source_label} timed out after {}s", + timeout.as_secs() )); } thread::sleep(Duration::from_millis(25)); @@ -272,20 +367,77 @@ pub fn run_history_source_plugin( let stderr = stderr_handle .join() .map_err(|_| anyhow!("history source plugin stderr reader panicked"))??; - cleanup_cursor_file(cursor_file.as_ref()); - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if !status.success() { - let detail = if stderr.is_empty() { - format!("exit status {status}") - } else { - format!("exit status {status}: {}", stderr_snippet(&stderr)) - }; - return Err(anyhow!( - "history source plugin {} failed: {detail}", - source.label() - )); + Ok((status, stdout, stderr)) +} + +#[cfg(unix)] +fn set_nonblocking(fd: std::os::fd::RawFd) -> Result<()> { + let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if flags < 0 { + return Err(std::io::Error::last_os_error()).context("read plugin pipe flags"); + } + let result = unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) }; + if result < 0 { + return Err(std::io::Error::last_os_error()).context("set plugin pipe nonblocking"); + } + Ok(()) +} + +#[cfg(unix)] +fn read_available_with_limit( + reader: &mut R, + bytes: &mut Vec, + open: &mut bool, + max_bytes: usize, + name: &str, + source_label: &str, +) -> Result<()> { + let mut buffer = [0u8; 8192]; + loop { + match reader.read(&mut buffer) { + Ok(0) => { + *open = false; + return Ok(()); + } + Ok(count) => { + if bytes.len().saturating_add(count) > max_bytes { + return Err(anyhow!( + "history source plugin {source_label} {name} exceeded {max_bytes} byte limit" + )); + } + bytes.extend_from_slice(&buffer[..count]); + } + Err(err) if err.kind() == ErrorKind::WouldBlock => return Ok(()), + Err(err) if err.kind() == ErrorKind::Interrupted => continue, + Err(err) => { + return Err(err) + .with_context(|| format!("read history source plugin {source_label} {name}")) + } + } + } +} + +#[cfg(any(test, not(unix)))] +fn read_pipe_with_limit( + mut reader: R, + max_bytes: usize, + name: &str, + source_label: &str, +) -> Result> { + let mut bytes = Vec::new(); + let mut buffer = [0u8; 8192]; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + return Ok(bytes); + } + if bytes.len().saturating_add(count) > max_bytes { + return Err(anyhow!( + "history source plugin {source_label} {name} exceeded {max_bytes} byte limit" + )); + } + bytes.extend_from_slice(&buffer[..count]); } - Ok(HistorySourcePluginRun { stdout, stderr }) } fn inherit_safe_plugin_env(command: &mut Command) { @@ -504,3 +656,27 @@ fn stderr_snippet(value: &str) -> String { } snippet } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn read_pipe_with_limit_accepts_output_at_limit() { + let bytes = read_pipe_with_limit(Cursor::new(b"abcd"), 4, "stdout", "plugin/default") + .expect("output at limit should pass"); + assert_eq!(bytes, b"abcd"); + } + + #[test] + fn read_pipe_with_limit_rejects_output_over_limit() { + let err = read_pipe_with_limit(Cursor::new(b"abcde"), 4, "stdout", "plugin/default") + .expect_err("output over limit should fail"); + assert!( + err.to_string() + .contains("history source plugin plugin/default stdout exceeded 4 byte limit"), + "{err}" + ); + } +} diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index e16e79d22..1868c652e 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -315,7 +315,7 @@ struct SearchArgs { value_enum, default_value_t = RefreshArg::Auto, help = "Pre-search refresh behavior: auto, off, or strict", - long_help = "Pre-search refresh behavior. auto best-effort refreshes discovered native provider sources and serves the existing index if refresh fails; off searches the existing index only; strict fails if the refresh cannot run or import successfully." + long_help = "Pre-search refresh behavior. auto best-effort refreshes discovered native provider sources and enabled auto history-source plugins, then serves the existing index if refresh fails; off searches the existing index only; strict fails if the refresh cannot run or import successfully." )] refresh: RefreshArg, #[arg( @@ -4424,16 +4424,27 @@ fn refresh_before_search(args: &SearchArgs, data_root: &Path) -> Result sources, + Err(err) if args.refresh == RefreshArg::Auto => { + return Ok(SearchRefreshReport::failed( + RefreshArg::Auto, + sources.len(), + error_summary(&err), + )); + } + Err(err) => return Err(err.context("search refresh failed")), + }; + if sources.is_empty() && plugin_sources.is_empty() { if args.refresh == RefreshArg::Strict { return Err(anyhow!( - "strict search refresh found no supported discovered native provider sources; use --refresh off to search the existing index" + "strict search refresh found no supported discovered native provider or enabled auto history-source plugin sources; use --refresh off to search the existing index" )); } return Ok(SearchRefreshReport::skipped(args.refresh, "no_sources")); } - let source_count = sources.len(); - match refresh_sources_for_search(data_root, sources, args.refresh, args.json) { + let source_count = sources.len().saturating_add(plugin_sources.len()); + match refresh_sources_for_search(data_root, sources, plugin_sources, args.refresh, args.json) { Ok(totals) => Ok(SearchRefreshReport::completed( args.refresh, source_count, @@ -4468,9 +4479,23 @@ fn search_refresh_sources(provider: Option) -> Vec { .collect() } +fn search_refresh_plugin_sources( + data_root: &Path, + provider: Option, +) -> Result> { + if !matches!(provider, None | Some(ProviderArg::Custom)) { + return Ok(Vec::new()); + } + Ok(discover_history_source_plugins(data_root, &[])? + .into_iter() + .filter(|source| source.enabled && source.refresh == HistorySourcePluginRefresh::Auto) + .collect()) +} + fn refresh_sources_for_search( data_root: &Path, sources: Vec, + plugin_sources: Vec, refresh: RefreshArg, json_output: bool, ) -> Result { @@ -4481,7 +4506,7 @@ fn refresh_sources_for_search( .into_iter() .map(|source| (source, SourceStats::default())) .collect::>(); - if planned_sources.is_empty() { + if planned_sources.is_empty() && plugin_sources.is_empty() { return Ok(ImportTotals::default()); } @@ -4566,6 +4591,27 @@ fn refresh_sources_for_search( } } + if !plugin_sources.is_empty() { + let mut store = Store::open(&db_path)?; + for plugin_source in plugin_sources { + progress.message( + "refreshing", + format!("running history source plugin {}", plugin_source.label()), + ); + let (summary, stats) = + import_history_source_plugin(&mut store, &plugin_source, data_root, false) + .with_context(|| { + format!("refresh history source plugin {}", plugin_source.label()) + })?; + totals.add(&summary, &stats); + progress.done( + "refreshing", + format!("refreshed history source plugin {}", plugin_source.label()), + 0, + ); + } + } + Store::open(&db_path)?.checkpoint_wal_truncate_if_larger_than(WAL_TRUNCATE_MIN_BYTES)?; Ok(totals) } diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 874d5bc1a..74737fc21 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -48,10 +48,21 @@ fn write_history_source_plugin( enabled: bool, cursor_log: Option<&Path>, ) -> HistorySourcePluginFixture { - write_history_source_plugin_at( + write_history_source_plugin_with_refresh(temp, provider, enabled, None, cursor_log) +} + +fn write_history_source_plugin_with_refresh( + temp: &TempDir, + provider: &str, + enabled: bool, + refresh: Option<&str>, + cursor_log: Option<&Path>, +) -> HistorySourcePluginFixture { + write_history_source_plugin_at_with_refresh( &temp.path().join("history-plugins"), provider, enabled, + refresh, cursor_log, ) } @@ -61,6 +72,16 @@ fn write_history_source_plugin_at( provider: &str, enabled: bool, cursor_log: Option<&Path>, +) -> HistorySourcePluginFixture { + write_history_source_plugin_at_with_refresh(root, provider, enabled, None, cursor_log) +} + +fn write_history_source_plugin_at_with_refresh( + root: &Path, + provider: &str, + enabled: bool, + refresh: Option<&str>, + cursor_log: Option<&Path>, ) -> HistorySourcePluginFixture { let manifest_dir = root.join(provider); fs::create_dir_all(&manifest_dir).unwrap(); @@ -146,20 +167,24 @@ for record in records: cursor_log_py = cursor_log_py ); fs::write(&script, script_body).unwrap(); + let mut source_manifest = json!({ + "id": "default", + "provider_key": provider, + "source_id": "default", + "source_format": format!("{provider}-history-v1"), + "enabled": enabled, + "command": [python_command(), script.display().to_string(), provider], + "timeout_seconds": 10 + }); + if let Some(refresh) = refresh { + source_manifest["refresh"] = json!(refresh); + } let manifest = json!({ "schema_version": 1, "name": provider, "display_name": format!("{provider} history"), "version": "0.1.0", - "history_sources": [{ - "id": "default", - "provider_key": provider, - "source_id": "default", - "source_format": format!("{provider}-history-v1"), - "enabled": enabled, - "command": [python_command(), script.display().to_string(), provider], - "timeout_seconds": 10 - }] + "history_sources": [source_manifest] }); fs::write( manifest_dir.join("ctx-history-plugin.json"), @@ -180,24 +205,56 @@ fn write_raw_history_source_plugin( temp: &TempDir, provider: &str, script_body: &str, +) -> HistorySourcePluginFixture { + write_raw_history_source_plugin_with_options(temp, provider, script_body, false, None) +} + +fn write_raw_history_source_plugin_with_options( + temp: &TempDir, + provider: &str, + script_body: &str, + enabled: bool, + refresh: Option<&str>, +) -> HistorySourcePluginFixture { + write_raw_history_source_plugin_with_options_and_timeout( + temp, + provider, + script_body, + enabled, + refresh, + 10, + ) +} + +fn write_raw_history_source_plugin_with_options_and_timeout( + temp: &TempDir, + provider: &str, + script_body: &str, + enabled: bool, + refresh: Option<&str>, + timeout_seconds: u64, ) -> HistorySourcePluginFixture { let manifest_dir = temp.path().join("history-plugins").join(provider); fs::create_dir_all(&manifest_dir).unwrap(); let script = manifest_dir.join("export.py"); let run_marker = manifest_dir.join("ran"); fs::write(&script, script_body).unwrap(); + let mut source_manifest = json!({ + "id": "default", + "provider_key": provider, + "source_id": "default", + "source_format": format!("{provider}-history-v1"), + "enabled": enabled, + "command": [python_command(), script.display().to_string()], + "timeout_seconds": timeout_seconds + }); + if let Some(refresh) = refresh { + source_manifest["refresh"] = json!(refresh); + } let manifest = json!({ "schema_version": 1, "name": provider, - "history_sources": [{ - "id": "default", - "provider_key": provider, - "source_id": "default", - "source_format": format!("{provider}-history-v1"), - "enabled": false, - "command": [python_command(), script.display().to_string()], - "timeout_seconds": 10 - }] + "history_sources": [source_manifest] }); fs::write( manifest_dir.join("ctx-history-plugin.json"), @@ -3774,6 +3831,279 @@ fn search_refresh_off_serves_existing_index_without_importing() { assert_search_provider_oracle(&fresh, "codex", "onboarding", 1, "message"); } +#[test] +fn search_refresh_auto_runs_enabled_auto_history_source_plugins_incrementally() { + let temp = tempdir(); + let cursor_log = temp.path().join("cursor-log.txt"); + let plugin = write_history_source_plugin_with_refresh( + &temp, + "hermes", + true, + Some("auto"), + Some(&cursor_log), + ); + + let initial = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "search", + "hermes plugin initial marker", + "--provider", + "custom", + "--json", + ]), + ); + assert_eq!(initial["freshness"]["mode"], "auto"); + assert_eq!(initial["freshness"]["status"], "completed"); + assert_eq!(initial["freshness"]["source_count"], 1); + assert_eq!(initial["freshness"]["totals"]["imported_sources"], 1); + assert_eq!(initial["freshness"]["totals"]["imported_sessions"], 1); + assert_eq!(initial["freshness"]["totals"]["imported_events"], 1); + assert!( + !initial["results"].as_array().unwrap().is_empty(), + "initial plugin refresh was not searchable before query: {initial:#}" + ); + assert!(plugin.run_marker.exists()); + + fs::remove_file(&plugin.run_marker).unwrap(); + let incremental = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "search", + "hermes plugin incremental marker", + "--provider", + "custom", + "--json", + ]), + ); + assert_eq!(incremental["freshness"]["mode"], "auto"); + assert_eq!(incremental["freshness"]["status"], "completed"); + assert_eq!(incremental["freshness"]["source_count"], 1); + assert_eq!(incremental["freshness"]["totals"]["imported_sources"], 1); + assert_eq!(incremental["freshness"]["totals"]["imported_events"], 1); + assert!( + !incremental["results"].as_array().unwrap().is_empty(), + "incremental plugin refresh was not searchable before query: {incremental:#}" + ); + assert!(plugin.run_marker.exists()); + + let cursor_log = fs::read_to_string(cursor_log).unwrap(); + assert!(cursor_log.contains(r#""message_id":7"#), "{cursor_log}"); + assert!(cursor_log.contains("cursor_file="), "{cursor_log}"); +} + +#[test] +fn search_refresh_auto_combines_native_sources_and_auto_history_source_plugins() { + let temp = tempdir(); + let fixture = PathBuf::from(provider_history_fixture("codex-sessions")); + copy_dir_all(&fixture, &temp.path().join(".codex").join("sessions")); + let plugin = + write_history_source_plugin_with_refresh(&temp, "hermes", true, Some("auto"), None); + + let search = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["search", "hermes plugin initial marker", "--json"]), + ); + + assert_eq!(search["freshness"]["mode"], "auto"); + assert_eq!(search["freshness"]["status"], "completed"); + assert_eq!(search["freshness"]["source_count"], 2); + assert!( + search["freshness"]["totals"]["imported_sessions"] + .as_u64() + .unwrap() + >= 3 + ); + assert!( + !search["results"].as_array().unwrap().is_empty(), + "combined refresh did not make plugin history searchable: {search:#}" + ); + assert!(plugin.run_marker.exists()); +} + +#[test] +fn search_refresh_provider_filter_does_not_execute_history_source_plugins() { + let temp = tempdir(); + let fixture = PathBuf::from(provider_history_fixture("codex-sessions")); + copy_dir_all(&fixture, &temp.path().join(".codex").join("sessions")); + let plugin = + write_history_source_plugin_with_refresh(&temp, "hermes", true, Some("auto"), None); + + let search = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["search", "onboarding", "--provider", "codex", "--json"]), + ); + + assert_eq!(search["freshness"]["mode"], "auto"); + assert_eq!(search["freshness"]["status"], "completed"); + assert_eq!(search["freshness"]["source_count"], 1); + assert_search_provider_oracle(&search, "codex", "onboarding", 1, "message"); + assert!(!plugin.run_marker.exists()); +} + +#[test] +fn search_refresh_off_does_not_execute_history_source_plugins() { + let temp = tempdir(); + let plugin = + write_history_source_plugin_with_refresh(&temp, "hermes", true, Some("auto"), None); + + let search = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "search", + "hermes plugin initial marker", + "--provider", + "custom", + "--refresh", + "off", + "--json", + ]), + ); + + assert_eq!(search["freshness"]["mode"], "off"); + assert_eq!(search["freshness"]["status"], "skipped"); + assert!(search["results"].as_array().unwrap().is_empty()); + assert!(!plugin.run_marker.exists()); +} + +#[test] +fn search_refresh_auto_skips_disabled_or_manual_history_source_plugins() { + let temp = tempdir(); + let plugin_root = temp.path().join("history-plugins"); + let manual = write_history_source_plugin_at_with_refresh( + &plugin_root, + "hermes", + true, + Some("manual"), + None, + ); + let disabled = write_history_source_plugin_at_with_refresh( + &plugin_root, + "dorkos", + false, + Some("auto"), + None, + ); + + let search = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) + .args([ + "search", + "plugin initial marker", + "--provider", + "custom", + "--json", + ]), + ); + + assert_eq!(search["freshness"]["mode"], "auto"); + assert_eq!(search["freshness"]["status"], "no_sources"); + assert_eq!(search["freshness"]["source_count"], 0); + assert!(search["results"].as_array().unwrap().is_empty()); + assert!(!manual.run_marker.exists()); + assert!(!disabled.run_marker.exists()); +} + +#[test] +fn search_refresh_strict_fails_on_history_source_plugin_failure() { + let temp = tempdir(); + let script = r#"#!/usr/bin/env python3 +import sys +print("plugin exploded", file=sys.stderr) +sys.exit(23) +"#; + let plugin = write_raw_history_source_plugin_with_options( + &temp, + "badplugin", + script, + true, + Some("auto"), + ); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "search", + "anything", + "--provider", + "custom", + "--refresh", + "strict", + "--json", + ]), + ); + + assert!(stderr.contains("search refresh failed"), "{stderr}"); + assert!( + stderr.contains("history source plugin badplugin/default failed"), + "{stderr}" + ); + assert!(stderr.contains("plugin exploded"), "{stderr}"); +} + +#[test] +fn search_refresh_strict_times_out_when_plugin_helper_keeps_stdout_open() { + let temp = tempdir(); + let script = r#"#!/usr/bin/env python3 +import json +import os +import subprocess + +observed = "2026-07-01T12:00:00Z" +source_id = os.environ["CTX_HISTORY_SOURCE_ID"] +provider_key = os.environ["CTX_HISTORY_PROVIDER_KEY"] +source_format = os.environ["CTX_HISTORY_SOURCE_FORMAT"] +cursor_stream = os.environ["CTX_HISTORY_CURSOR_STREAM"] +records = [ + {"record_type": "manifest", "schema_version": "ctx-history-jsonl-v1"}, + {"record_type": "source", "source_id": source_id, "provider_key": provider_key, "source_format": source_format, "observed_at": observed, "cursor": {"after": {"stream": cursor_stream, "cursor": json.dumps({"seq": 1}), "observed_at": observed}}}, + {"record_type": "session", "source_id": source_id, "session_id": "hanging-session", "started_at": observed, "agent_type": "primary", "is_primary": True, "status": "completed"}, + {"record_type": "event", "source_id": source_id, "session_id": "hanging-session", "event_index": 0, "event_type": "message", "role": "assistant", "occurred_at": observed, "payload": {"text": "hanging plugin marker"}, "preview": "hanging plugin marker"}, +] +for record in records: + print(json.dumps(record, separators=(",", ":")), flush=True) +subprocess.Popen(["sh", "-c", "sleep 5"]) +"#; + let plugin = write_raw_history_source_plugin_with_options_and_timeout( + &temp, + "hanging", + script, + true, + Some("auto"), + 1, + ); + + let started = Instant::now(); + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "search", + "hanging plugin marker", + "--provider", + "custom", + "--refresh", + "strict", + "--json", + ]), + ); + assert!( + started.elapsed() < Duration::from_secs(3), + "plugin timeout did not bound pipe draining: {stderr}" + ); + assert!( + stderr.contains("history source plugin hanging/default timed out after 1s"), + "{stderr}" + ); +} + #[test] fn search_refresh_auto_imports_fresh_work_despite_large_existing_catalog() { let temp = tempdir(); diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index ab6408270..c88108373 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -3899,16 +3899,16 @@ fn normalize_custom_history_jsonl_v1_reader( } } - validate_custom_history_references( - &mut summary, + let reference_index = CustomHistoryReferenceIndex { manifest_line, - &sources, - &sessions, - &events, - &event_keys, - &file_touches, - &edges, - ); + sources: &sources, + sessions: &sessions, + events: &events, + event_keys: &event_keys, + file_touches: &file_touches, + edges: &edges, + }; + validate_custom_history_references(&mut summary, reference_index); if summary.failed > 0 { return Ok(custom_history_failed_normalization(summary)); } @@ -4057,17 +4057,21 @@ fn validate_custom_history_identifier( } } +struct CustomHistoryReferenceIndex<'a> { + manifest_line: Option, + sources: &'a BTreeMap, + sessions: &'a BTreeMap<(String, String), (usize, CtxHistoryJsonlSessionRecord)>, + events: &'a [(usize, CtxHistoryJsonlEventRecord)], + event_keys: &'a BTreeSet<(String, String, u64)>, + file_touches: &'a [(usize, CtxHistoryJsonlFileTouchRecord)], + edges: &'a [(usize, CtxHistoryJsonlEdgeRecord)], +} + fn validate_custom_history_references( summary: &mut ProviderImportSummary, - manifest_line: Option, - sources: &BTreeMap, - sessions: &BTreeMap<(String, String), (usize, CtxHistoryJsonlSessionRecord)>, - events: &[(usize, CtxHistoryJsonlEventRecord)], - event_keys: &BTreeSet<(String, String, u64)>, - file_touches: &[(usize, CtxHistoryJsonlFileTouchRecord)], - edges: &[(usize, CtxHistoryJsonlEdgeRecord)], + references: CustomHistoryReferenceIndex<'_>, ) { - if manifest_line.is_none() { + if references.manifest_line.is_none() { push_provider_import_failure( summary, 0, @@ -4075,8 +4079,8 @@ fn validate_custom_history_references( ); } - for (line_number, session) in sessions.values() { - if !sources.contains_key(&session.source_id) { + for (line_number, session) in references.sessions.values() { + if !references.sources.contains_key(&session.source_id) { push_provider_import_failure( summary, *line_number, @@ -4088,7 +4092,7 @@ fn validate_custom_history_references( } if let Some(parent) = &session.parent_session_id { let key = (session.source_id.clone(), parent.clone()); - if !sessions.contains_key(&key) { + if !references.sessions.contains_key(&key) { push_provider_import_failure( summary, *line_number, @@ -4098,7 +4102,7 @@ fn validate_custom_history_references( } if let Some(root) = &session.root_session_id { let key = (session.source_id.clone(), root.clone()); - if root != &session.session_id && !sessions.contains_key(&key) { + if root != &session.session_id && !references.sessions.contains_key(&key) { push_provider_import_failure( summary, *line_number, @@ -4108,8 +4112,11 @@ fn validate_custom_history_references( } } - for (line_number, event) in events { - if !sessions.contains_key(&(event.source_id.clone(), event.session_id.clone())) { + for (line_number, event) in references.events { + if !references + .sessions + .contains_key(&(event.source_id.clone(), event.session_id.clone())) + { push_provider_import_failure( summary, *line_number, @@ -4121,8 +4128,11 @@ fn validate_custom_history_references( } } - for (line_number, file_touch) in file_touches { - if !sessions.contains_key(&(file_touch.source_id.clone(), file_touch.session_id.clone())) { + for (line_number, file_touch) in references.file_touches { + if !references + .sessions + .contains_key(&(file_touch.source_id.clone(), file_touch.session_id.clone())) + { push_provider_import_failure( summary, *line_number, @@ -4138,7 +4148,7 @@ fn validate_custom_history_references( file_touch.session_id.clone(), event_index, ); - if !event_keys.contains(&key) { + if !references.event_keys.contains(&key) { push_provider_import_failure( summary, *line_number, @@ -4148,10 +4158,10 @@ fn validate_custom_history_references( } } - for (line_number, edge) in edges { + for (line_number, edge) in references.edges { let from_key = (edge.source_id.clone(), edge.from_session_id.clone()); let to_key = (edge.source_id.clone(), edge.to_session_id.clone()); - if !sessions.contains_key(&from_key) { + if !references.sessions.contains_key(&from_key) { push_provider_import_failure( summary, *line_number, @@ -4161,7 +4171,7 @@ fn validate_custom_history_references( ), ); } - if !sessions.contains_key(&to_key) { + if !references.sessions.contains_key(&to_key) { push_provider_import_failure( summary, *line_number, @@ -4172,7 +4182,7 @@ fn validate_custom_history_references( ); } if edge.edge_type == SessionEdgeType::ParentChild { - let Some((_, child)) = sessions.get(&to_key) else { + let Some((_, child)) = references.sessions.get(&to_key) else { continue; }; if let Some(parent) = &child.parent_session_id { diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 4224e261a..a238edf06 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -201,13 +201,12 @@ ctx search "this current task" --include-current-session ``` `search` defaults to `--refresh auto`, which quietly refreshes discovered native -provider sources before querying indexed sessions and events. The refresh is -best-effort and keeps JSON stdout reserved for the search result object. -History-source plugin commands are not executed by search refresh. On large -discovered sources or already-cataloged indexes, `auto` serves current results -without a foreground catch-up scan; use `--refresh strict` or `ctx import --all` -when you need a full catch-up before querying. Use `--refresh off` to search the -existing index without refreshing, or +provider sources and enabled auto history-source plugins before querying indexed +sessions and events. The refresh is best-effort and keeps JSON stdout reserved +for the search result object. On large discovered sources or already-cataloged +indexes, `auto` serves current results without a foreground catch-up scan; use +`--refresh strict` or `ctx import --all` when you need a full catch-up before +querying. Use `--refresh off` to search the existing index without refreshing, or `--refresh strict` to fail when the pre-search refresh cannot run or import successfully. Preview native sources such as NanoClaw and AstrBot are searched from the existing index until they are explicitly imported through a supported @@ -266,9 +265,10 @@ provider IDs in ctx output; multiword IDs may be snake_case, such as `copilot_cli` or `factory_ai_droid`, while compact IDs such as `openclaw`, `nanoclaw`, and `astrbot` stay compact. -`search` reads discovered native provider files for pre-search refresh plus -SQLite, and may write newly discovered native provider history into the local -index before querying. +`search` reads discovered native provider files and runs enabled auto +history-source plugin commands for pre-search refresh, then queries SQLite. It +may write newly discovered provider or plugin history into the local index before +querying. ## SQL diff --git a/docs/first-10-minutes.md b/docs/first-10-minutes.md index 3a34ae0b9..ef656c3e7 100644 --- a/docs/first-10-minutes.md +++ b/docs/first-10-minutes.md @@ -86,8 +86,9 @@ ctx search "build failure" --term checksum --term release --limit 5 ``` `--limit` is capped at `200`. Search defaults to `--refresh auto`, which -best-effort refreshes discovered native provider sources before querying; use -`--refresh off` to search only the existing index. +best-effort refreshes discovered native provider sources and enabled auto +history-source plugins before querying; use `--refresh off` to search only the +existing index. Inside Codex, ctx excludes the active session tree by default when it can identify it, so your current prompt and subagents do not dominate results. Add diff --git a/docs/history-source-plugins.md b/docs/history-source-plugins.md index 66274bb40..398e1c70b 100644 --- a/docs/history-source-plugins.md +++ b/docs/history-source-plugins.md @@ -6,9 +6,10 @@ searchable in ctx without ctx owning their storage schemas. The narrow waist is: 1. A local manifest declares one or more history sources. -2. ctx invokes the declared command only during explicit import. +2. ctx invokes enabled auto-refresh commands during search refresh, or any + selected source during explicit import. 3. The command writes `ctx-history-jsonl-v1` records to stdout. -4. ctx validates and imports that stream atomically. +4. The stream is checked and imported as one batch. 5. ctx passes the previous source cursor back on the next run. Plugins are command-line adapters, not an in-process ABI and not a hosted plugin @@ -41,6 +42,7 @@ Manifest example: "source_id": "default", "source_format": "dorkos-claude-jsonl-v1", "enabled": true, + "refresh": "auto", "command": ["ctx-history-source-dorkos", "export"], "timeout_seconds": 300 } @@ -51,8 +53,10 @@ Manifest example: `name`, `id`, `provider_key`, and `source_id` must be stable lowercase ASCII identifiers. `command` is an argv array; ctx does not run it through a shell. -`enabled: true` means `ctx import --all` may run that source. Explicit imports -can run a discovered source even when it is not enabled. +`enabled: true` means `ctx import --all` may run that source. `refresh: auto` +means `ctx search` may run it during the normal pre-search refresh. Explicit +imports can run a discovered source even when it is not enabled or is marked +`refresh: manual`. ## Import @@ -78,9 +82,11 @@ from the supplied manifest path. `CTX_HISTORY_FULL_RESCAN=1`. The plugin should emit a fresh `source.cursor.after` checkpoint if the rescan succeeds. -`ctx setup` and search refresh do not execute plugins in this version. Run -`ctx import --history-source ` or `ctx import --all` to catch up -plugin-backed sources before searching. +`ctx setup` does not execute plugins. `ctx search` defaults to `--refresh auto` +and runs discovered plugin sources only when they are both `enabled: true` and +`refresh: auto`; `--refresh off` never runs plugins, and `--refresh strict` +fails if an auto plugin refresh fails. Plugin refresh is incremental because ctx +passes the previously stored source cursor before invoking the command. ## Runtime Environment @@ -108,6 +114,9 @@ while the plugin process runs and is the reliable cursor handoff path. The plugin must write only `ctx-history-jsonl-v1` JSONL to stdout. Progress and diagnostics belong on stderr. If the command exits nonzero or stdout is invalid, ctx imports nothing from that run and does not advance the cursor. +stdout is capped at 64 MiB per run and stderr at 256 KiB, so plugins should emit +incremental batches from the supplied cursor instead of full historical dumps +during normal refresh. Plugin commands receive a limited inherited environment by default: `PATH`, `HOME`, basic locale variables, temporary-directory variables, and XDG data or diff --git a/docs/product-contract.md b/docs/product-contract.md index 86d05b78e..421b1f4d3 100644 --- a/docs/product-contract.md +++ b/docs/product-contract.md @@ -14,10 +14,11 @@ product boundary is retrieval, not interpretation. transcript formats. - `ctx sources` reports known local provider history paths, including whether a native source is currently importable. -- `ctx import` indexes supported local transcript formats. -- `ctx search` can refresh discovered native provider sources before returning - ranked local hits from the local index, with event IDs when a hit maps to an - indexed event. +- `ctx import` indexes supported local transcript formats and selected local + history-source plugins. +- `ctx search` can refresh discovered native provider sources and enabled auto + history-source plugins before returning ranked local hits from the local + index, with event IDs when a hit maps to an indexed event. - `ctx show session` and `ctx show event` render transcripts, hits, and context windows using ctx-owned IDs, and `ctx show session --out` writes transcript artifacts. diff --git a/docs/search.md b/docs/search.md index 7aab44a9c..fc1a10cf3 100644 --- a/docs/search.md +++ b/docs/search.md @@ -3,8 +3,8 @@ `ctx search` finds matching indexed history. Default results are session-diverse: ctx shows the strongest matching span from each session, then lets you drill into dense event-level results when needed. By default it first performs a quiet -best-effort refresh of discovered native provider sources, then queries the -local SQLite store. +best-effort refresh of discovered native provider sources and enabled auto +history-source plugins, then queries the local SQLite store. ## Search @@ -103,18 +103,19 @@ work do not dominate history research. Use `--include-current-session` when you are intentionally looking for material from the active session tree. `--refresh` defaults to `auto`. `auto` attempts a best-effort pre-search import -of discovered native provider sources and serves the existing index if that -refresh fails. Search refresh does not execute history-source plugin commands. -On large discovered sources or already-cataloged indexes, `auto` serves current -results without a foreground catch-up scan; use `--refresh strict` or -`ctx import --all` when you need a full catch-up before querying. `off` skips -the pre-search refresh. `strict` fails the search if the refresh cannot run or -import successfully. Preview native sources such as NanoClaw and AstrBot, plus -search-only sources without native import support, are searched from the +of discovered native provider sources and enabled auto history-source plugins, +then serves the existing index if that refresh fails. On large discovered +sources or already-cataloged indexes, `auto` serves current results without a +foreground catch-up scan; use `--refresh strict` or `ctx import --all` when you +need a full catch-up before querying. `off` skips the pre-search refresh and +never runs plugin commands. `strict` fails the search if the refresh cannot run +or import successfully. Preview native sources such as NanoClaw and AstrBot, +plus search-only sources without native import support, are searched from the existing index until they are explicitly imported through a supported path. Use `--refresh off` for a strictly read-only search over the existing ctx index. -This avoids provider imports and avoids updating the ctx SQLite store. +This avoids provider imports, plugin execution, and updates to the ctx SQLite +store. ## History Reports From 78cd4b46a9995447b0973df29cdcd9444a75d1d9 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:11:51 -0500 Subject: [PATCH 31/72] Tighten history source plugin API --- crates/ctx-cli/src/history_source_plugins.rs | 72 ++- crates/ctx-cli/src/main.rs | 359 +++++++++++- crates/ctx-cli/src/mcp.rs | 45 +- crates/ctx-cli/tests/cli.rs | 300 +++++++++- crates/ctx-history-search/src/lib.rs | 577 ++++++++++++++++++- crates/ctx-history-store/src/lib.rs | 89 +++ docs/cli-reference.md | 27 +- docs/custom-history-import-format.md | 9 +- docs/history-source-plugin-design.md | 350 +++++++++++ docs/history-source-plugins.md | 45 +- docs/search.md | 4 + docs/storage.md | 4 +- 12 files changed, 1773 insertions(+), 108 deletions(-) create mode 100644 docs/history-source-plugin-design.md diff --git a/crates/ctx-cli/src/history_source_plugins.rs b/crates/ctx-cli/src/history_source_plugins.rs index 7464138fd..4b97a3f2f 100644 --- a/crates/ctx-cli/src/history_source_plugins.rs +++ b/crates/ctx-cli/src/history_source_plugins.rs @@ -17,7 +17,6 @@ use serde::Deserialize; use uuid::Uuid; const PLUGIN_MANIFEST_FILE: &str = "ctx-history-plugin.json"; -const LEGACY_PLUGIN_MANIFEST_FILE: &str = "plugin.json"; const DEFAULT_PLUGIN_TIMEOUT_SECONDS: u64 = 300; const MAX_PLUGIN_STDOUT_BYTES: usize = 64 * 1024 * 1024; const MAX_PLUGIN_STDERR_BYTES: usize = 256 * 1024; @@ -74,11 +73,7 @@ impl HistorySourcePluginSource { } pub fn matches_selector(&self, selector: &str) -> bool { - selector == self.plugin_name - || selector == self.id - || selector == self.label() - || selector == self.provider_key - || selector == format!("{}/{}", self.provider_key, self.source_id) + selector == self.label() || selector == format!("{}/{}", self.provider_key, self.source_id) } } @@ -105,6 +100,18 @@ pub struct HistorySourcePluginRunOptions<'a> { pub full_rescan: bool, } +#[derive(Debug, Clone, Default)] +pub struct HistorySourcePluginDiscovery { + pub sources: Vec, + pub failures: Vec, +} + +#[derive(Debug, Clone)] +pub struct HistorySourcePluginManifestFailure { + pub manifest_path: PathBuf, + pub error: String, +} + #[derive(Debug, Deserialize)] struct HistorySourcePluginManifest { schema_version: u32, @@ -144,11 +151,23 @@ pub fn discover_history_source_plugins( data_root: &Path, extra_manifests: &[PathBuf], ) -> Result> { + let discovery = discover_history_source_plugins_with_diagnostics(data_root, extra_manifests)?; + Ok(discovery.sources) +} + +pub fn discover_history_source_plugins_with_diagnostics( + data_root: &Path, + extra_manifests: &[PathBuf], +) -> Result { let mut sources = Vec::new(); + let mut failures = Vec::new(); for manifest_path in plugin_manifest_paths(data_root) { match read_plugin_manifest(&manifest_path) { Ok(mut manifest_sources) => sources.append(&mut manifest_sources), - Err(_) => continue, + Err(error) => failures.push(HistorySourcePluginManifestFailure { + manifest_path, + error: error.to_string(), + }), } } for manifest_path in explicit_plugin_manifest_paths(extra_manifests)? { @@ -156,7 +175,7 @@ pub fn discover_history_source_plugins( sources.append(&mut manifest_sources); } sources.sort_by_key(|source| source.label()); - Ok(sources) + Ok(HistorySourcePluginDiscovery { sources, failures }) } pub fn run_history_source_plugin( @@ -202,16 +221,13 @@ pub fn run_history_source_plugin( })?; if cursor.len() <= MAX_INLINE_CURSOR_ENV_BYTES { command.env("CTX_HISTORY_CURSOR", cursor); - command.env("CTX_HISTORY_CURSOR_JSON", cursor); } else { command.env_remove("CTX_HISTORY_CURSOR"); - command.env_remove("CTX_HISTORY_CURSOR_JSON"); } command.env("CTX_HISTORY_CURSOR_FILE", &path); Some(path) } else { command.env_remove("CTX_HISTORY_CURSOR"); - command.env_remove("CTX_HISTORY_CURSOR_JSON"); command.env_remove("CTX_HISTORY_CURSOR_FILE"); None }; @@ -450,7 +466,7 @@ fn inherit_safe_plugin_env(command: &mut Command) { fn write_private_temp_file(prefix: &str, contents: &str) -> Result { for _ in 0..16 { - let path = env::temp_dir().join(format!("{prefix}-{}.json", Uuid::new_v4())); + let path = env::temp_dir().join(format!("{prefix}-{}.cursor", Uuid::new_v4())); let mut options = OpenOptions::new(); options.write(true).create_new(true); #[cfg(unix)] @@ -499,13 +515,14 @@ fn read_plugin_manifest(path: &Path) -> Result> { let provider_key = source.provider_key.unwrap_or_else(|| manifest.name.clone()); validate_plugin_id("provider_key", &provider_key)?; let source_id = source.source_id.unwrap_or_else(|| source.id.clone()); - if source.source_format.trim().is_empty() { - return Err(anyhow!( - "history source plugin manifest {} source {} has empty source_format", + validate_plugin_id("source_id", &source_id)?; + validate_source_format(&source.source_format).with_context(|| { + format!( + "history source plugin manifest {} source {} has invalid source_format", path.display(), source.id - )); - } + ) + })?; if source.command.is_empty() || source.command.iter().any(|part| part.trim().is_empty()) { return Err(anyhow!( "history source plugin manifest {} source {} has empty command", @@ -548,11 +565,6 @@ fn plugin_manifest_paths(data_root: &Path) -> Vec { collect_manifest_path_candidates(&path, &mut candidates); } } - if let Some(paths) = env::var_os("CTX_PLUGIN_PATH") { - for path in env::split_paths(&paths) { - collect_manifest_path_candidates(&path, &mut candidates); - } - } candidates.into_iter().collect() } @@ -584,10 +596,6 @@ fn collect_manifest_path_candidates(path: &Path, candidates: &mut BTreeSet Result<()> { + let valid = + !value.trim().is_empty() && value.len() <= 512 && !value.chars().any(char::is_control); + if valid { + Ok(()) + } else { + Err(anyhow!( + "source_format must be non-empty, at most 512 bytes, and contain no control characters" + )) + } +} + fn validate_plugin_id(label: &str, value: &str) -> Result<()> { let valid = !value.is_empty() && value.len() <= 128 diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 1868c652e..5f1815530 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -58,7 +58,8 @@ use ctx_history_store::{ RAW_SQL_MAX_TIMEOUT, }; use history_source_plugins::{ - discover_history_source_plugins, run_history_source_plugin, HistorySourcePluginRefresh, + discover_history_source_plugins, discover_history_source_plugins_with_diagnostics, + run_history_source_plugin, HistorySourcePluginManifestFailure, HistorySourcePluginRefresh, HistorySourcePluginRunOptions, HistorySourcePluginSource, }; @@ -134,15 +135,10 @@ struct ImportArgs { provider: Option, #[arg(long)] path: Option, - #[arg( - long = "history-source", - alias = "plugin", - conflicts_with_all = ["provider", "path", "format", "all"] - )] + #[arg(long = "history-source", conflicts_with_all = ["provider", "path", "format", "all"])] history_source: Option, #[arg( long = "history-source-manifest", - alias = "plugin-manifest", conflicts_with_all = ["provider", "path", "format"] )] history_source_manifest: Vec, @@ -269,6 +265,26 @@ struct SearchArgs { limit: usize, #[arg(long, help = "Search only one provider")] provider: Option, + #[arg( + long = "history-source", + help = "Filter custom history imports by plugin/source or provider_key/source_id" + )] + history_source: Option, + #[arg( + long = "provider-key", + help = "Filter custom history imports by provider_key" + )] + provider_key: Option, + #[arg( + long = "source-id", + help = "Filter custom history imports by source_id" + )] + source_id: Option, + #[arg( + long = "source-format", + help = "Filter custom history imports by source_format" + )] + source_format: Option, #[arg( long, help = "Filter by stored workspace, cwd, source path, or repo-name text" @@ -373,6 +389,7 @@ impl SqlArgs { pub(crate) struct SearchFilterInput { session: Option, provider: Option, + source_identity: SourceIdentityFilterArgs, workspace: Option, since: Option, primary_only: bool, @@ -382,6 +399,66 @@ pub(crate) struct SearchFilterInput { include_current_session: bool, } +#[derive(Debug, Clone, Default)] +pub(crate) struct SourceIdentityFilterArgs { + history_source: Option, + provider_key: Option, + source_id: Option, + source_format: Option, +} + +#[derive(Debug, Clone, Default)] +struct SourceIdentityFilters { + history_source: Option, + provider_key: Option, + source_id: Option, + source_format: Option, +} + +impl SourceIdentityFilters { + fn is_empty(&self) -> bool { + self.history_source.is_none() + && self.provider_key.is_none() + && self.source_id.is_none() + && self.source_format.is_none() + } + + fn matches_plugin_source(&self, source: &HistorySourcePluginSource) -> bool { + if let Some(selector) = &self.history_source { + if !source.matches_selector(selector) { + return false; + } + } + if let Some(provider_key) = &self.provider_key { + if source.provider_key != *provider_key { + return false; + } + } + if let Some(source_id) = &self.source_id { + if source.source_id != *source_id { + return false; + } + } + if let Some(source_format) = &self.source_format { + if source.source_format != *source_format { + return false; + } + } + true + } +} + +impl From<&SearchArgs> for SourceIdentityFilterArgs { + fn from(args: &SearchArgs) -> Self { + Self { + history_source: args.history_source.clone(), + provider_key: args.provider_key.clone(), + source_id: args.source_id.clone(), + source_format: args.source_format.clone(), + } + } +} + impl CommandRoot { fn name(&self) -> &'static str { match self { @@ -1765,7 +1842,9 @@ fn run_sources( analytics_properties: &mut AnalyticsProperties, ) -> Result<()> { let sources = discovered_sources(); - let plugin_sources = discover_history_source_plugins(&data_root, &[])?; + let plugin_discovery = discover_history_source_plugins_with_diagnostics(&data_root, &[])?; + let plugin_sources = plugin_discovery.sources; + let plugin_failures = plugin_discovery.failures; let existing = sources.iter().filter(|source| source.exists).count(); let importable = sources .iter() @@ -1778,7 +1857,10 @@ fn run_sources( analytics::insert_count_bucket( analytics_properties, "providers_detected_bucket", - sources.len().saturating_add(plugin_sources.len()) as u64, + sources + .len() + .saturating_add(plugin_sources.len()) + .saturating_add(plugin_failures.len()) as u64, ); analytics::insert_count_bucket( analytics_properties, @@ -1793,6 +1875,7 @@ fn run_sources( if args.json { let mut source_values = sources_json(&sources); source_values.extend(plugin_sources_json(&plugin_sources)); + source_values.extend(plugin_manifest_failures_json(&plugin_failures)); print_json(json!({ "schema_version": 1, "sources": source_values, @@ -1807,6 +1890,13 @@ fn run_sources( source.source_format ); } + for failure in plugin_failures { + println!( + "custom history-source-plugin invalid: {}: {}", + failure.manifest_path.display(), + failure.error + ); + } for source in plugin_sources { println!( "custom {} available (history-source-plugin:{})", @@ -1818,6 +1908,13 @@ fn run_sources( Ok(()) } +pub(crate) fn discovered_plugin_sources_json(data_root: &Path) -> Result> { + let plugin_discovery = discover_history_source_plugins_with_diagnostics(data_root, &[])?; + let mut values = plugin_sources_json(&plugin_discovery.sources); + values.extend(plugin_manifest_failures_json(&plugin_discovery.failures)); + Ok(values) +} + fn catalog_available_sources( store: &Store, sources: &[SourceInfo], @@ -1975,7 +2072,7 @@ fn run_import_internal( &mut store, &plugin_source, &data_root, - args.resume || args.reset_cursor, + args.reset_cursor, ) { Ok((summary, stats)) => { totals.add(&summary, &stats); @@ -2020,6 +2117,7 @@ fn run_import_internal( } } + let native_import_requested = !planned_sources.is_empty(); if should_parallelize_import(&planned_sources) { let final_refresh_required = store.event_search_projection_needs_backfill()? || planned_sources @@ -2302,7 +2400,7 @@ fn run_import_internal( return Err(anyhow!("all import sources failed{detail}")); } Ok(ImportReport { - resume: args.resume, + resume: args.resume && native_import_requested, totals, sources: imported_sources, }) @@ -3643,6 +3741,11 @@ impl SearchDto { .then_some(result.more_matches_in_session), "provider": result.provider, "provider_session_id": result.provider_session_id, + "history_source": result.history_source, + "history_source_plugin": result.history_source_plugin, + "provider_key": result.provider_key, + "source_id": result.source_id, + "source_format": result.source_format, "timestamp": result.timestamp, "cwd": result.cwd, "source_path": result.raw_source_path, @@ -4198,6 +4301,7 @@ fn run_search( insert_db_size_bucket(analytics_properties, &db_path); let store = Store::open(&db_path)?; insert_store_analytics_counts(analytics_properties, &store)?; + let source_identity = SourceIdentityFilterArgs::from(&args); let query = args.query.unwrap_or_default(); let query_term_count = query .split_whitespace() @@ -4226,6 +4330,7 @@ fn run_search( SearchFilterInput { session: args.session, provider: args.provider, + source_identity, workspace: args.workspace.clone(), since: args.since.clone(), primary_only: args.primary_only, @@ -4356,6 +4461,18 @@ fn print_search_result_verbose( if let Some(provider_session_id) = &result.provider_session_id { println!(" provider_session_id: {provider_session_id}"); } + if let Some(history_source) = &result.history_source { + println!(" history_source: {history_source}"); + } + if let Some(provider_key) = &result.provider_key { + println!(" provider_key: {provider_key}"); + } + if let Some(source_id) = &result.source_id { + println!(" source_id: {source_id}"); + } + if let Some(source_format) = &result.source_format { + println!(" source_format: {source_format}"); + } println!(" {}", result.snippet); println!(" rank: {:.2}", result.rank); if result.result_scope == ctx_history_search::SearchResultScope::Session { @@ -4387,6 +4504,12 @@ fn search_result_summary(result: &ctx_history_search::SearchPacketResult) -> Vec if let Some(provider) = result.provider { summary.push(provider.as_str().to_owned()); } + if let Some(history_source) = &result.history_source { + summary.push(history_source.clone()); + } else if let (Some(provider_key), Some(source_id)) = (&result.provider_key, &result.source_id) + { + summary.push(format!("{provider_key}/{source_id}")); + } if result.result_scope == ctx_history_search::SearchResultScope::Session { summary.push(format!("importance {:.2}", result.session_importance)); } else { @@ -4423,18 +4546,33 @@ fn refresh_before_search(args: &SearchArgs, data_root: &Path) -> Result sources, - Err(err) if args.refresh == RefreshArg::Auto => { - return Ok(SearchRefreshReport::failed( - RefreshArg::Auto, - sources.len(), - error_summary(&err), - )); - } - Err(err) => return Err(err.context("search refresh failed")), + let source_identity = normalize_source_identity_filters(SourceIdentityFilterArgs::from(args))?; + if !source_identity.is_empty() + && args + .provider + .is_some_and(|provider| !matches!(provider, ProviderArg::Custom)) + { + return Err(anyhow!( + "custom history source filters can only be combined with --provider custom" + )); + } + let sources = if source_identity.is_empty() { + search_refresh_sources(args.provider) + } else { + Vec::new() }; + let plugin_sources = + match search_refresh_plugin_sources(data_root, args.provider, &source_identity) { + Ok(sources) => sources, + Err(err) if args.refresh == RefreshArg::Auto => { + return Ok(SearchRefreshReport::failed( + RefreshArg::Auto, + sources.len(), + error_summary(&err), + )); + } + Err(err) => return Err(err.context("search refresh failed")), + }; if sources.is_empty() && plugin_sources.is_empty() { if args.refresh == RefreshArg::Strict { return Err(anyhow!( @@ -4482,13 +4620,18 @@ fn search_refresh_sources(provider: Option) -> Vec { fn search_refresh_plugin_sources( data_root: &Path, provider: Option, + source_identity: &SourceIdentityFilters, ) -> Result> { if !matches!(provider, None | Some(ProviderArg::Custom)) { return Ok(Vec::new()); } Ok(discover_history_source_plugins(data_root, &[])? .into_iter() - .filter(|source| source.enabled && source.refresh == HistorySourcePluginRefresh::Auto) + .filter(|source| { + source.enabled + && source.refresh == HistorySourcePluginRefresh::Auto + && source_identity.matches_plugin_source(source) + }) .collect()) } @@ -4851,19 +4994,20 @@ fn import_history_source_plugin( }, )?; let _plugin_stderr = &run.stderr; - validate_history_source_plugin_output(source, &run.stdout, &machine_id)?; - let validation = validate_custom_history_jsonl_v1_reader(Cursor::new(run.stdout.as_slice())) + validate_history_source_plugin_output(source, &run.stdout, &machine_id, full_rescan)?; + let stdout = annotate_history_source_plugin_output(source, &run.stdout)?; + let validation = validate_custom_history_jsonl_v1_reader(Cursor::new(stdout.as_slice())) .map_err(anyhow::Error::from)?; if validation.failed > 0 { return Err(history_source_plugin_import_failure(source, &validation)); } let stats = SourceStats { files: 1, - bytes: run.stdout.len() as u64, + bytes: stdout.len() as u64, }; store.upsert_record(&record)?; let summary = import_custom_history_jsonl_v1_reader( - Cursor::new(run.stdout), + Cursor::new(stdout), store, CustomHistoryJsonlV1ImportOptions { machine_id, @@ -4880,10 +5024,70 @@ fn import_history_source_plugin( Ok((summary, stats)) } +fn annotate_history_source_plugin_output( + source: &HistorySourcePluginSource, + stdout: &[u8], +) -> Result> { + let text = std::str::from_utf8(stdout).with_context(|| { + format!( + "history source plugin {} emitted non-UTF-8 ctx-history-jsonl-v1 output", + source.label() + ) + })?; + let mut out = Vec::with_capacity(stdout.len()); + for (index, line) in text.lines().enumerate() { + let line_number = index + 1; + if line.trim().is_empty() { + continue; + } + let mut record: CtxHistoryJsonlRecord = serde_json::from_str(line).with_context(|| { + format!( + "history source plugin {} emitted invalid ctx-history-jsonl-v1 at line {line_number}", + source.label() + ) + })?; + if let CtxHistoryJsonlRecord::Source(source_record) = &mut record { + let mut metadata = match std::mem::take(&mut source_record.metadata) { + Value::Object(map) => map, + Value::Null => serde_json::Map::new(), + other => { + let mut map = serde_json::Map::new(); + map.insert("metadata".to_owned(), other); + map + } + }; + metadata.insert( + "ctx_history_plugin".to_owned(), + json!({ + "plugin_name": source.plugin_name, + "plugin_source_id": source.id, + "history_source": source.label(), + "plugin_display_name": source.plugin_display_name, + "plugin_version": source.plugin_version, + "manifest_path": source.manifest_path, + "provider_key": source.provider_key, + "source_id": source.source_id, + "source_format": source.source_format, + }), + ); + source_record.metadata = Value::Object(metadata); + } + serde_json::to_writer(&mut out, &record).with_context(|| { + format!( + "serialize annotated history source plugin {} record at line {line_number}", + source.label() + ) + })?; + out.push(b'\n'); + } + Ok(out) +} + fn validate_history_source_plugin_output( source: &HistorySourcePluginSource, stdout: &[u8], machine_id: &str, + require_after_cursor: bool, ) -> Result<()> { let text = std::str::from_utf8(stdout).with_context(|| { format!( @@ -4892,6 +5096,7 @@ fn validate_history_source_plugin_output( ) })?; let mut saw_source = false; + let mut saw_after_cursor = false; for (index, line) in text.lines().enumerate() { let line_number = index + 1; if line.trim().is_empty() { @@ -4907,6 +5112,14 @@ fn validate_history_source_plugin_output( continue; }; saw_source = true; + if source_record + .cursor + .as_ref() + .and_then(|cursor| cursor.after.as_ref()) + .is_some() + { + saw_after_cursor = true; + } if source_record.provider_key != source.provider_key || source_record.source_id != source.source_id || source_record.source_format != source.source_format @@ -4937,6 +5150,12 @@ fn validate_history_source_plugin_output( source.label() )); } + if require_after_cursor && !saw_after_cursor { + return Err(anyhow!( + "history source plugin {} was reset but emitted no source.cursor.after checkpoint; emit a fresh cursor after a full rescan", + source.label() + )); + } Ok(()) } @@ -5913,6 +6132,37 @@ fn plugin_sources_json(sources: &[HistorySourcePluginSource]) -> Vec { .collect() } +fn plugin_manifest_failures_json(failures: &[HistorySourcePluginManifestFailure]) -> Vec { + failures + .iter() + .map(|failure| { + json!({ + "provider": CaptureProvider::Custom.as_str(), + "kind": "history_source_plugin", + "plugin": null, + "plugin_display_name": null, + "plugin_version": null, + "history_source": null, + "history_source_id": null, + "display_name": null, + "provider_key": null, + "source_id": null, + "source_format": null, + "manifest_path": failure.manifest_path, + "enabled": false, + "refresh": null, + "status": "invalid", + "import_support": "history_source_plugin", + "native_import": false, + "importable": false, + "raw_retention": "metadata_only", + "unsupported_reason": failure.error, + "error": failure.error, + }) + }) + .collect() +} + fn history_source_plugin_refresh_json(refresh: HistorySourcePluginRefresh) -> &'static str { match refresh { HistorySourcePluginRefresh::Manual => "manual", @@ -5942,6 +6192,21 @@ fn search_filters( input: SearchFilterInput, store: Option<&Store>, ) -> Result { + let source_identity = normalize_source_identity_filters(input.source_identity)?; + if !source_identity.is_empty() + && input + .provider + .is_some_and(|provider| !matches!(provider, ProviderArg::Custom)) + { + return Err(anyhow!( + "custom history source filters can only be combined with --provider custom" + )); + } + let provider = if !source_identity.is_empty() { + Some(CaptureProvider::Custom) + } else { + input.provider.map(ProviderArg::capture_provider) + }; let session = input .session .as_deref() @@ -5959,7 +6224,11 @@ fn search_filters( }; Ok(ctx_history_search::SearchFilters { session, - provider: input.provider.map(ProviderArg::capture_provider), + provider, + history_source: source_identity.history_source, + provider_key: source_identity.provider_key, + source_id: source_identity.source_id, + source_format: source_identity.source_format, repo: input.workspace, since: input.since.as_deref().map(parse_since_filter).transpose()?, primary_only: input.primary_only, @@ -5975,6 +6244,40 @@ fn search_filters( }) } +fn normalize_source_identity_filters( + input: SourceIdentityFilterArgs, +) -> Result { + let history_source = normalize_source_identity_filter("history-source", input.history_source)?; + if history_source + .as_deref() + .is_some_and(|value| !value.contains('/')) + { + return Err(anyhow!( + "--history-source expects plugin/source or provider_key/source_id" + )); + } + Ok(SourceIdentityFilters { + history_source, + provider_key: normalize_source_identity_filter("provider-key", input.provider_key)?, + source_id: normalize_source_identity_filter("source-id", input.source_id)?, + source_format: normalize_source_identity_filter("source-format", input.source_format)?, + }) +} + +fn normalize_source_identity_filter(label: &str, value: Option) -> Result> { + let Some(value) = value else { + return Ok(None); + }; + let value = value.trim(); + if value.is_empty() { + return Err(anyhow!("--{label} cannot be empty")); + } + if value.chars().any(char::is_control) { + return Err(anyhow!("--{label} cannot contain control characters")); + } + Ok(Some(value.to_owned())) +} + fn current_codex_provider_session_filter( store: Option<&Store>, ) -> Option { diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index 2bd15a76c..02232830b 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -17,10 +17,11 @@ use serde_json::{json, Value}; use uuid::Uuid; use super::{ - compact_json, config::CONFIG_FILE, discovered_sources, event_window, event_window_json, - indexed_history_item_count, mark_share_safe, raw_sql_result_json, search_filters, - session_transcript_json, sources_json, OutputFormat, ProviderArg, RefreshArg, SearchDto, - SearchFilterInput, SearchRefreshReport, TranscriptMode, MAX_SEARCH_LIMIT, + compact_json, config::CONFIG_FILE, discovered_plugin_sources_json, discovered_sources, + event_window, event_window_json, indexed_history_item_count, mark_share_safe, + raw_sql_result_json, search_filters, session_transcript_json, sources_json, OutputFormat, + ProviderArg, RefreshArg, SearchDto, SearchFilterInput, SearchRefreshReport, + SourceIdentityFilterArgs, TranscriptMode, MAX_SEARCH_LIMIT, }; const MCP_PROTOCOL_VERSION: &str = "2025-11-25"; @@ -198,7 +199,7 @@ fn handle_tools_call(params: Value, data_root: &Path) -> Result { } "sources" => { validate_argument_keys(&arguments, &[])?; - tool_sources() + tool_sources(data_root) } "search" => { validate_argument_keys( @@ -207,6 +208,10 @@ fn handle_tools_call(params: Value, data_root: &Path) -> Result { "query", "limit", "provider", + "history_source", + "provider_key", + "source_id", + "source_format", "workspace", "since", "primary_only", @@ -303,11 +308,13 @@ fn tool_status(data_root: &Path) -> Result { })) } -fn tool_sources() -> Result { +fn tool_sources(data_root: &Path) -> Result { let sources = discovered_sources(); + let mut source_values = sources_json(&sources); + source_values.extend(discovered_plugin_sources_json(data_root)?); Ok(json!({ "schema_version": 1, - "sources": sources_json(&sources), + "sources": source_values, "read_only": true, })) } @@ -320,6 +327,10 @@ fn tool_search(arguments: &Value, data_root: &Path) -> Result { return Err(anyhow!("limit must be between 1 and {MAX_SEARCH_LIMIT}")); } let provider = optional_provider(arguments, "provider")?; + let history_source = optional_string(arguments, "history_source")?; + let provider_key = optional_string(arguments, "provider_key")?; + let source_id = optional_string(arguments, "source_id")?; + let source_format = optional_string(arguments, "source_format")?; let session = optional_string(arguments, "session")?; let workspace = optional_string(arguments, "workspace")?; let since = optional_string(arguments, "since")?; @@ -337,6 +348,12 @@ fn tool_search(arguments: &Value, data_root: &Path) -> Result { SearchFilterInput { session, provider, + source_identity: SourceIdentityFilterArgs { + history_source, + provider_key, + source_id, + source_format, + }, workspace, since, primary_only, @@ -493,6 +510,10 @@ fn tool_definitions() -> Vec { "query": { "type": "string" }, "limit": { "type": "integer", "minimum": 1, "maximum": MAX_SEARCH_LIMIT, "default": 20 }, "provider": { "type": "string", "enum": provider_names() }, + "history_source": { "type": "string", "description": "Custom history source selector as plugin/source or provider_key/source_id." }, + "provider_key": { "type": "string", "description": "Custom history provider_key." }, + "source_id": { "type": "string", "description": "Custom history source_id." }, + "source_format": { "type": "string", "description": "Custom history source_format." }, "workspace": { "type": "string", "description": "Workspace path or name text." }, "since": { "type": "string", "description": "RFC3339 timestamp or day window such as 30d." }, "include_subagents": { "type": "boolean", "default": false, "description": "Include subagent sessions in addition to primary-agent sessions." }, @@ -570,6 +591,11 @@ fn provider_names() -> Vec<&'static str> { "copilot_cli", ProviderArg::FactoryAiDroid.cli_name(), "factory_ai_droid", + ProviderArg::OpenClaw.cli_name(), + ProviderArg::Hermes.cli_name(), + ProviderArg::NanoClaw.cli_name(), + ProviderArg::AstrBot.cli_name(), + ProviderArg::Custom.cli_name(), ]; names.sort_unstable(); names @@ -650,6 +676,11 @@ fn optional_provider(arguments: &Value, key: &str) -> Result "cursor" => Ok(Some(ProviderArg::Cursor)), "copilot-cli" | "copilot_cli" => Ok(Some(ProviderArg::CopilotCli)), "factory-ai-droid" | "factory_ai_droid" => Ok(Some(ProviderArg::FactoryAiDroid)), + "openclaw" => Ok(Some(ProviderArg::OpenClaw)), + "hermes" => Ok(Some(ProviderArg::Hermes)), + "nanoclaw" => Ok(Some(ProviderArg::NanoClaw)), + "astrbot" => Ok(Some(ProviderArg::AstrBot)), + "custom" => Ok(Some(ProviderArg::Custom)), _ => Err(anyhow!( "provider must be one of {}", provider_names().join(", ") diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 74737fc21..f65422ad5 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -106,11 +106,11 @@ source_id = os.environ["CTX_HISTORY_SOURCE_ID"] provider_key = os.environ["CTX_HISTORY_PROVIDER_KEY"] source_format = os.environ["CTX_HISTORY_SOURCE_FORMAT"] cursor_stream = os.environ["CTX_HISTORY_CURSOR_STREAM"] -cursor_json = os.environ.get("CTX_HISTORY_CURSOR_JSON") +cursor_inline = os.environ.get("CTX_HISTORY_CURSOR") cursor_file = os.environ.get("CTX_HISTORY_CURSOR_FILE") pathlib.Path({run_marker_json}).write_text("ran\n") cursor_log = {cursor_log_py} -cursor_text = cursor_json +cursor_text = cursor_inline if not cursor_text and cursor_file: cursor_text = pathlib.Path(cursor_file).read_text() if cursor_log and cursor_text: @@ -1110,6 +1110,110 @@ fn history_source_plugins_are_listed_without_running() { assert!(!plugin.run_marker.exists()); } +#[test] +fn invalid_installed_history_source_plugin_is_listed_as_invalid() { + let temp = tempdir(); + let plugin_root = temp.path().join("history-plugins"); + let bad_dir = plugin_root.join("bad"); + fs::create_dir_all(&bad_dir).unwrap(); + fs::write(bad_dir.join("ctx-history-plugin.json"), "{not-json").unwrap(); + + let sources = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) + .args(["sources", "--json"]), + ); + let invalid = sources["sources"] + .as_array() + .unwrap() + .iter() + .find(|source| source["kind"] == "history_source_plugin" && source["status"] == "invalid") + .unwrap(); + assert_eq!(invalid["importable"], false); + assert_eq!(invalid["enabled"], false); + assert!(invalid["error"] + .as_str() + .unwrap() + .contains("parse history source plugin manifest")); +} + +#[test] +fn invalid_installed_history_source_plugin_does_not_block_valid_import() { + let temp = tempdir(); + let plugin_root = temp.path().join("history-plugins"); + let good = write_history_source_plugin_at(&plugin_root, "dorkos", false, None); + let bad_dir = plugin_root.join("bad"); + fs::create_dir_all(&bad_dir).unwrap(); + fs::write(bad_dir.join("ctx-history-plugin.json"), "{not-json").unwrap(); + + let imported = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) + .args([ + "import", + "--history-source", + "dorkos/default", + "--json", + "--progress", + "none", + ]), + ); + + assert_eq!(imported["totals"]["imported_sources"], 1); + assert!(good.run_marker.exists()); +} + +#[test] +fn removed_history_source_plugin_aliases_and_legacy_discovery_are_ignored() { + let temp = tempdir(); + let plugin = write_history_source_plugin(&temp, "dorkos", false, None); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["import", "--plugin", "dorkos/default"]), + ); + assert!(stderr.contains("--plugin"), "{stderr}"); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["import", "--plugin-manifest", "ctx-history-plugin.json"]), + ); + assert!(stderr.contains("--plugin-manifest"), "{stderr}"); + + let sources = json_output( + ctx(&temp) + .env_remove("CTX_HISTORY_PLUGIN_PATH") + .env("CTX_PLUGIN_PATH", &plugin.manifest_dir) + .args(["sources", "--json"]), + ); + assert!(!sources["sources"] + .as_array() + .unwrap() + .iter() + .any(|source| source["history_source"] == "dorkos/default")); + + let legacy_dir = temp.path().join("legacy-plugin"); + fs::create_dir_all(&legacy_dir).unwrap(); + fs::copy( + plugin.manifest_dir.join("ctx-history-plugin.json"), + legacy_dir.join("plugin.json"), + ) + .unwrap(); + let sources = json_output( + ctx(&temp) + .env_remove("CTX_PLUGIN_PATH") + .env("CTX_HISTORY_PLUGIN_PATH", &legacy_dir) + .args(["sources", "--json"]), + ); + assert!(!sources["sources"] + .as_array() + .unwrap() + .iter() + .any(|source| source["history_source"] == "dorkos/default")); +} + #[test] fn setup_does_not_execute_enabled_history_source_plugins() { let temp = tempdir(); @@ -1125,7 +1229,7 @@ fn setup_does_not_execute_enabled_history_source_plugins() { } #[test] -fn ambiguous_history_source_plugin_selector_fails_before_execution() { +fn bare_history_source_plugin_selector_fails_before_execution() { let temp = tempdir(); let plugin_root = temp.path().join("history-plugins"); let dorkos = write_history_source_plugin_at(&plugin_root, "dorkos", false, None); @@ -1134,16 +1238,13 @@ fn ambiguous_history_source_plugin_selector_fails_before_execution() { let stderr = failure_stderr( ctx(&temp) .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) - .args([ - "import", - "--history-source", - "default", - "--progress", - "none", - ]), + .args(["import", "--history-source", "dorkos", "--progress", "none"]), ); - assert!(stderr.contains("matched multiple sources"), "{stderr}"); + assert!( + stderr.contains("no history source plugin matched"), + "{stderr}" + ); assert!(!dorkos.run_marker.exists()); assert!(!hermes.run_marker.exists()); } @@ -1241,6 +1342,42 @@ for record in records: ); } +#[test] +fn history_source_plugin_reset_requires_fresh_after_cursor() { + let temp = tempdir(); + let script = r#"#!/usr/bin/env python3 +import json +records = [ + {"record_type":"manifest","schema_version":"ctx-history-jsonl-v1"}, + {"record_type":"source","source_id":"default","provider_key":"nocursor","source_format":"nocursor-history-v1"}, + {"record_type":"session","source_id":"default","session_id":"run","started_at":"2026-07-01T12:00:00Z"}, +] +for record in records: + print(json.dumps(record)) +"#; + let plugin = write_raw_history_source_plugin(&temp, "nocursor", script); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "nocursor/default", + "--reset-cursor", + "--progress", + "none", + ]), + ); + + assert!(stderr.contains("source.cursor.after"), "{stderr}"); + let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); + assert_eq!( + sqlite_count(&conn, "SELECT COUNT(*) FROM history_records"), + 0 + ); +} + #[test] fn large_history_source_plugin_cursor_uses_cursor_file_without_inline_env() { let temp = tempdir(); @@ -1253,7 +1390,7 @@ import os import pathlib cursor_file = os.environ.get("CTX_HISTORY_CURSOR_FILE") -inline = os.environ.get("CTX_HISTORY_CURSOR_JSON") +inline = os.environ.get("CTX_HISTORY_CURSOR") cursor_text = pathlib.Path(cursor_file).read_text() if cursor_file else inline if cursor_text: with open({log_json}, "a", encoding="utf-8") as handle: @@ -1315,7 +1452,8 @@ fn import_history_source_plugin_is_searchable_and_receives_cursor() { .args([ "import", "--history-source", - "hermes", + "hermes/default", + "--resume", "--json", "--progress", "none", @@ -1338,6 +1476,22 @@ fn import_history_source_plugin_is_searchable_and_receives_cursor() { !initial["results"].as_array().unwrap().is_empty(), "initial plugin import was not searchable: {initial:#}" ); + let initial_by_history_source = json_output(ctx(&temp).args([ + "search", + "hermes plugin initial marker", + "--history-source", + "hermes/default", + "--refresh", + "off", + "--json", + ])); + let source_filtered_result = &initial_by_history_source["results"][0]; + assert_eq!(source_filtered_result["provider"], "custom"); + assert_eq!(source_filtered_result["history_source"], "hermes/default"); + assert_eq!(source_filtered_result["history_source_plugin"], "hermes"); + assert_eq!(source_filtered_result["provider_key"], "hermes"); + assert_eq!(source_filtered_result["source_id"], "default"); + assert_eq!(source_filtered_result["source_format"], "hermes-history-v1"); let second = json_output( ctx(&temp) @@ -1345,7 +1499,7 @@ fn import_history_source_plugin_is_searchable_and_receives_cursor() { .args([ "import", "--history-source", - "hermes", + "hermes/default", "--json", "--progress", "none", @@ -1353,6 +1507,8 @@ fn import_history_source_plugin_is_searchable_and_receives_cursor() { ); assert_eq!(second["totals"]["imported_sessions"], 0); assert_eq!(second["totals"]["imported_events"], 1); + assert_eq!(second["resume"], false); + assert_eq!(second["resume_mode"], "normal_scan"); let incremental = json_output(ctx(&temp).args([ "search", @@ -3561,6 +3717,79 @@ fn mcp_search_and_show_tools_return_structured_json_without_refresh() { assert!(!event["events"].as_array().unwrap().is_empty()); } +#[test] +fn mcp_sources_and_search_support_history_source_plugins() { + let temp = tempdir(); + let plugin = write_history_source_plugin(&temp, "hermes", false, None); + json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "hermes/default", + "--json", + "--progress", + "none", + ]), + ); + + let responses = mcp_roundtrip_with_env( + &temp, + &[ + json!({ + "jsonrpc": "2.0", + "id": "init", + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { "name": "ctx-test", "version": "0" } + } + }), + json!({ + "jsonrpc": "2.0", + "id": "sources", + "method": "tools/call", + "params": { + "name": "sources", + "arguments": {} + } + }), + json!({ + "jsonrpc": "2.0", + "id": "search", + "method": "tools/call", + "params": { + "name": "search", + "arguments": { + "query": "hermes plugin initial marker", + "provider": "custom", + "history_source": "hermes/default", + "limit": 5 + } + } + }), + ], + &[( + "CTX_HISTORY_PLUGIN_PATH", + plugin.manifest_dir.to_str().unwrap(), + )], + ); + + let sources = responses[1]["result"]["structuredContent"]["sources"] + .as_array() + .unwrap(); + assert!(sources + .iter() + .any(|source| source["history_source"] == "hermes/default")); + + let search = &responses[2]["result"]["structuredContent"]; + assert_eq!(search["filters"]["provider"], "custom"); + assert_eq!(search["filters"]["history_source"], "hermes/default"); + assert_eq!(search["results"][0]["history_source"], "hermes/default"); +} + #[test] fn mcp_search_excludes_active_codex_session_by_default_when_available() { let temp = tempdir(); @@ -3894,6 +4123,49 @@ fn search_refresh_auto_runs_enabled_auto_history_source_plugins_incrementally() assert!(cursor_log.contains("cursor_file="), "{cursor_log}"); } +#[test] +fn search_refresh_history_source_filter_runs_only_matching_auto_plugin() { + let temp = tempdir(); + let plugin_root = temp.path().join("history-plugins"); + let dorkos = write_history_source_plugin_at_with_refresh( + &plugin_root, + "dorkos", + true, + Some("auto"), + None, + ); + let hermes = write_history_source_plugin_at_with_refresh( + &plugin_root, + "hermes", + true, + Some("auto"), + None, + ); + + let search = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) + .args([ + "search", + "dorkos plugin initial marker", + "--history-source", + "dorkos/default", + "--json", + ]), + ); + + assert_eq!(search["filters"]["provider"], "custom"); + assert_eq!(search["filters"]["history_source"], "dorkos/default"); + assert_eq!(search["freshness"]["status"], "completed"); + assert_eq!(search["freshness"]["source_count"], 1); + assert!(dorkos.run_marker.exists()); + assert!(!hermes.run_marker.exists()); + assert!( + !search["results"].as_array().unwrap().is_empty(), + "source-filtered refresh did not import matching plugin: {search:#}" + ); +} + #[test] fn search_refresh_auto_combines_native_sources_and_auto_history_source_plugins() { let temp = tempdir(); diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index a932e08fb..cb71fdcc2 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -62,6 +62,14 @@ pub struct SearchFilters { pub session: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub provider: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub history_source: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_format: Option, #[serde(default, rename = "workspace", skip_serializing_if = "Option::is_none")] pub repo: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -120,6 +128,16 @@ pub struct SearchPacketResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub provider_session_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub history_source: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub history_source_plugin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_format: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub timestamp: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub cwd: Option, @@ -195,6 +213,11 @@ struct HitMetadata { time: chrono::DateTime, provider: Option, provider_session_id: Option, + history_source: Option, + history_source_plugin: Option, + provider_key: Option, + source_id: Option, + source_format: Option, session_id: Option, parent_session_id: Option, root_session_id: Option, @@ -365,9 +388,34 @@ fn merge_search_result(existing: &mut SearchPacketResult, incoming: SearchPacket existing.event_seq = incoming.event_seq; existing.timestamp = incoming.timestamp; existing.cwd = incoming.cwd.clone(); + existing.provider = incoming.provider; + existing.provider_session_id = incoming.provider_session_id.clone(); + existing.history_source = incoming.history_source.clone(); + existing.history_source_plugin = incoming.history_source_plugin.clone(); + existing.provider_key = incoming.provider_key.clone(); + existing.source_id = incoming.source_id.clone(); + existing.source_format = incoming.source_format.clone(); existing.raw_source_path = incoming.raw_source_path.clone(); existing.raw_source_exists = incoming.raw_source_exists; existing.cursor = incoming.cursor.clone(); + } else { + existing.history_source = existing + .history_source + .clone() + .or(incoming.history_source.clone()); + existing.history_source_plugin = existing + .history_source_plugin + .clone() + .or(incoming.history_source_plugin.clone()); + existing.provider_key = existing + .provider_key + .clone() + .or(incoming.provider_key.clone()); + existing.source_id = existing.source_id.clone().or(incoming.source_id.clone()); + existing.source_format = existing + .source_format + .clone() + .or(incoming.source_format.clone()); } existing.rank = existing_rank.max(incoming_rank) + 0.08; existing.more_matches_in_session = existing @@ -471,6 +519,19 @@ fn candidate_search_result( provider_session_id: display_hit .as_ref() .and_then(|hit| hit.provider_session_id.clone()), + history_source: display_hit + .as_ref() + .and_then(|hit| hit.history_source.clone()), + history_source_plugin: display_hit + .as_ref() + .and_then(|hit| hit.history_source_plugin.clone()), + provider_key: display_hit + .as_ref() + .and_then(|hit| hit.provider_key.clone()), + source_id: display_hit.as_ref().and_then(|hit| hit.source_id.clone()), + source_format: display_hit + .as_ref() + .and_then(|hit| hit.source_format.clone()), timestamp: display_hit.as_ref().map(|hit| hit.time), cwd: display_hit.as_ref().and_then(|hit| hit.cwd.clone()), raw_source_path: display_hit @@ -499,6 +560,7 @@ fn candidate_display_hit(candidate: &Candidate, filters: &SearchFilters) -> Opti && filters .session .map_or(true, |id| hit.session_id == Some(id)) + && hit_matches_history_source_filter(&hit, filters) }) { return Some(event_hit(event, &candidate.context)); } @@ -516,6 +578,10 @@ fn candidate_display_hit(candidate: &Candidate, filters: &SearchFilters) -> Opti .provider .map_or(true, |provider| session.provider == provider) && filters.session.map_or(true, |id| session.id == id) + && hit_matches_history_source_filter( + &session_hit(session, &candidate.context), + filters, + ) }) .or_else(|| candidate.context.sessions.first()) .map(|session| session_hit(session, &candidate.context)) @@ -530,6 +596,9 @@ fn fast_event_search_packet( if query.trim().is_empty() { return Ok(None); } + if has_history_source_filter(&options.filters) { + return Ok(None); + } if !store.has_at_least_events(LARGE_EVENT_CORPUS_THRESHOLD)? { return Ok(None); } @@ -861,6 +930,11 @@ fn event_search_result( session_importance: 0.0, provider: hit.provider, provider_session_id: hit.session_external_session_id.clone(), + history_source: hit.history_source.clone(), + history_source_plugin: hit.history_source_plugin.clone(), + provider_key: hit.provider_key.clone(), + source_id: hit.source_id.clone(), + source_format: hit.source_format.clone(), timestamp: Some(hit.occurred_at), cwd: hit.cwd.clone(), raw_source_path: hit.raw_source_path.clone(), @@ -1317,7 +1391,9 @@ fn search_sections( } } for session in &context.sessions { - if !session_matches_agent_scope(session, filters) { + if !session_matches_agent_scope(session, filters) + || !source_id_matches_history_source_filter(session.capture_source_id, context, filters) + { continue; } let hit = session_hit(session, context); @@ -1536,6 +1612,9 @@ fn event_hit_matches_agent_scope(hit: &EventSearchHit, filters: &SearchFilters) } fn record_text_matches_agent_scope(context: &RecordContext, filters: &SearchFilters) -> bool { + if has_history_source_filter(filters) { + return false; + } context .sessions .iter() @@ -1548,11 +1627,32 @@ fn item_matches_agent_scope( context: &RecordContext, filters: &SearchFilters, ) -> bool { + let item_source_id = source_id.or_else(|| { + session_id + .and_then(|id| context.sessions.iter().find(|session| session.id == id)) + .and_then(|session| session.capture_source_id) + }); + if !source_id_matches_history_source_filter(item_source_id, context, filters) { + return false; + } associated_session(session_id, source_id, context) .map(|session| session_matches_agent_scope(session, filters)) .unwrap_or(true) } +fn source_id_matches_history_source_filter( + source_id: Option, + context: &RecordContext, + filters: &SearchFilters, +) -> bool { + if !has_history_source_filter(filters) { + return true; + } + source_id + .and_then(|id| context.sources.get(&id)) + .is_some_and(|source| source_matches_history_source_filter(source, filters)) +} + fn associated_session( session_id: Option, source_id: Option, @@ -1591,6 +1691,7 @@ fn record_context_display_hit( .provider .map_or(true, |provider| session.provider == provider) && filters.session.map_or(true, |id| session.id == id) + && hit_matches_history_source_filter(&session_hit(session, context), filters) }) .or_else(|| { context @@ -1639,6 +1740,11 @@ fn empty_hit(time: chrono::DateTime) -> HitMetadata { time, provider: None, provider_session_id: None, + history_source: None, + history_source_plugin: None, + provider_key: None, + source_id: None, + source_format: None, session_id: None, parent_session_id: None, root_session_id: None, @@ -1744,10 +1850,16 @@ fn source_hit( return empty_hit(time); }; let raw_source_path = source.descriptor.raw_source_path.clone(); + let identity = source_history_identity(source); let mut hit = HitMetadata { time, provider: Some(source.descriptor.provider), provider_session_id: source.descriptor.external_session_id.clone(), + history_source: identity.history_source, + history_source_plugin: identity.history_source_plugin, + provider_key: identity.provider_key, + source_id: identity.source_id, + source_format: identity.source_format, session_id: None, parent_session_id: None, root_session_id: None, @@ -1788,6 +1900,175 @@ fn source_cursor(source: &ctx_history_core::CaptureSource) -> Option { .map(str::to_owned) } +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct SourceHistoryIdentity { + history_source: Option, + history_source_plugin: Option, + provider_key: Option, + source_id: Option, + source_format: Option, +} + +fn source_history_identity(source: &ctx_history_core::CaptureSource) -> SourceHistoryIdentity { + let metadata = &source.sync.metadata; + let source_metadata = metadata + .get("source_metadata") + .and_then(serde_json::Value::as_object); + let plugin = source_metadata + .and_then(|metadata| metadata.get("ctx_history_plugin")) + .or_else(|| metadata.get("ctx_history_plugin")) + .and_then(serde_json::Value::as_object); + let custom = source_metadata + .and_then(|metadata| metadata.get("ctx_history_jsonl_v1")) + .or_else(|| metadata.get("ctx_history_jsonl_v1")) + .and_then(serde_json::Value::as_object); + let plugin_name = plugin + .and_then(|plugin| plugin.get("plugin_name")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let plugin_source_id = plugin + .and_then(|plugin| plugin.get("plugin_source_id")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let history_source = plugin + .and_then(|plugin| plugin.get("history_source")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .or_else(|| { + plugin_name + .as_deref() + .zip(plugin_source_id.as_deref()) + .map(|(plugin_name, source_id)| format!("{plugin_name}/{source_id}")) + }); + let provider_key = custom + .and_then(|custom| custom.get("provider_key")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let source_id = custom + .and_then(|custom| custom.get("source_id")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let source_format = custom + .and_then(|custom| custom.get("source_format")) + .and_then(serde_json::Value::as_str) + .or_else(|| { + source_metadata + .and_then(|metadata| metadata.get("source_format")) + .and_then(serde_json::Value::as_str) + }) + .or_else(|| { + metadata + .get("source_format") + .and_then(serde_json::Value::as_str) + }) + .map(str::to_owned); + SourceHistoryIdentity { + history_source, + history_source_plugin: plugin_name, + provider_key, + source_id, + source_format, + } +} + +fn has_history_source_filter(filters: &SearchFilters) -> bool { + filters + .history_source + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || filters + .provider_key + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || filters + .source_id + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || filters + .source_format + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) +} + +fn source_matches_history_source_filter( + source: &ctx_history_core::CaptureSource, + filters: &SearchFilters, +) -> bool { + let identity = source_history_identity(source); + source_identity_matches_history_source_filter(&identity, filters) +} + +fn hit_matches_history_source_filter(hit: &HitMetadata, filters: &SearchFilters) -> bool { + if !has_history_source_filter(filters) { + return true; + } + source_identity_matches_history_source_filter( + &SourceHistoryIdentity { + history_source: hit.history_source.clone(), + history_source_plugin: hit.history_source_plugin.clone(), + provider_key: hit.provider_key.clone(), + source_id: hit.source_id.clone(), + source_format: hit.source_format.clone(), + }, + filters, + ) +} + +fn source_identity_matches_history_source_filter( + identity: &SourceHistoryIdentity, + filters: &SearchFilters, +) -> bool { + if let Some(selector) = filters + .history_source + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let plugin_match = identity.history_source.as_deref() == Some(selector); + let provider_source_match = identity + .provider_key + .as_deref() + .zip(identity.source_id.as_deref()) + .is_some_and(|(provider_key, source_id)| { + selector == format!("{provider_key}/{source_id}") + }); + if !plugin_match && !provider_source_match { + return false; + } + } + if let Some(provider_key) = filters + .provider_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + if identity.provider_key.as_deref() != Some(provider_key) { + return false; + } + } + if let Some(source_id) = filters + .source_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + if identity.source_id.as_deref() != Some(source_id) { + return false; + } + } + if let Some(source_format) = filters + .source_format + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + if identity.source_format.as_deref() != Some(source_format) { + return false; + } + } + true +} + fn event_cursor(event: &Event) -> Option { event .payload @@ -1953,6 +2234,7 @@ fn has_filters(filters: &SearchFilters) -> bool { .as_ref() .is_some_and(|value| !value.trim().is_empty()) || filters.exclude_provider_session.is_some() + || has_history_source_filter(filters) } fn record_matches_filters( @@ -2014,6 +2296,15 @@ fn record_matches_filters( } } + if has_history_source_filter(filters) + && !context + .sources + .values() + .any(|source| source_matches_history_source_filter(source, filters)) + { + return false; + } + if let Some(since) = filters.since { let has_recent_event = context .events @@ -2354,6 +2645,11 @@ mod tests { score: 1.0, provider: Some(CaptureProvider::Codex), session_external_session_id: Some("provider-session-1".into()), + history_source: None, + history_source_plugin: None, + provider_key: None, + source_id: None, + source_format: None, agent_type: Some(AgentType::Primary), session_is_primary: Some(true), cwd: None, @@ -2405,6 +2701,11 @@ mod tests { score: 1.0, provider: None, session_external_session_id: None, + history_source: None, + history_source_plugin: None, + provider_key: None, + source_id: None, + source_format: None, agent_type: Some(AgentType::Subagent), session_is_primary: Some(false), cwd: None, @@ -3236,6 +3537,280 @@ mod tests { assert!(wrong_provider.results.is_empty()); } + #[test] + fn search_filters_custom_history_source_identity() { + let (_temp, store) = test_store(); + let record = HistoryRecord::new( + "Custom plugin import", + "ordinary body", + Vec::new(), + "session", + Some("/workspace/custom".into()), + ); + store.insert_record(&record).unwrap(); + + let source_id = Uuid::parse_str("018f45d0-0000-7000-8000-000000000451").unwrap(); + let source = CaptureSource { + id: source_id, + descriptor: CaptureSourceDescriptor { + kind: CaptureSourceKind::ProviderImport, + provider: CaptureProvider::Custom, + machine_id: "machine-1".into(), + process_id: None, + cwd: Some("/workspace/custom".into()), + raw_source_path: Some("/tmp/dorkos-plugin/ctx-history-plugin.json".into()), + external_session_id: Some("ctx-history-jsonl-v1-session".into()), + }, + started_at: fixed_time(), + ended_at: None, + sync: SyncMetadata { + metadata: serde_json::json!({ + "ctx_history_plugin": { + "plugin_name": "dorkos", + "plugin_source_id": "default", + "history_source": "dorkos/default" + }, + "ctx_history_jsonl_v1": { + "provider_key": "dorkos", + "source_id": "default", + "source_format": "dorkos-history-v1" + } + }), + ..sync_metadata() + }, + }; + store.upsert_capture_source(&source).unwrap(); + + let session = Session { + id: Uuid::parse_str("018f45d0-0000-7000-8000-000000000452").unwrap(), + history_record_id: Some(record.id), + parent_session_id: None, + root_session_id: None, + capture_source_id: Some(source_id), + provider: CaptureProvider::Custom, + external_session_id: Some("ctx-history-jsonl-v1-session".into()), + external_agent_id: None, + agent_type: AgentType::Primary, + role_hint: Some("primary".into()), + is_primary: true, + status: SessionStatus::Imported, + transcript_blob_id: None, + started_at: fixed_time(), + ended_at: None, + timestamps: timestamps(), + sync: sync_metadata(), + }; + store.upsert_session(&session).unwrap(); + + let event = Event { + id: Uuid::parse_str("018f45d0-0000-7000-8000-000000000453").unwrap(), + seq: 451, + history_record_id: Some(record.id), + session_id: Some(session.id), + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::Assistant), + occurred_at: fixed_time(), + capture_source_id: Some(source_id), + payload: serde_json::json!({ + "body": { + "text": "dorkos-source-filter-needle" + } + }), + payload_blob_id: None, + dedupe_key: Some("custom-history-source-filter-event".into()), + redaction_state: RedactionState::SafePreview, + sync: sync_metadata(), + }; + store.upsert_event(&event).unwrap(); + store.upsert_record(&record).unwrap(); + + let packet = search_packet( + &store, + "dorkos-source-filter-needle", + &PacketOptions { + limit: 10, + filters: SearchFilters { + provider: Some(CaptureProvider::Custom), + history_source: Some("dorkos/default".into()), + ..SearchFilters::default() + }, + ..PacketOptions::default() + }, + ) + .unwrap(); + + assert_eq!(packet.results.len(), 1); + let result = &packet.results[0]; + assert_eq!(result.provider, Some(CaptureProvider::Custom)); + assert_eq!(result.history_source.as_deref(), Some("dorkos/default")); + assert_eq!(result.history_source_plugin.as_deref(), Some("dorkos")); + assert_eq!(result.provider_key.as_deref(), Some("dorkos")); + assert_eq!(result.source_id.as_deref(), Some("default")); + assert_eq!(result.source_format.as_deref(), Some("dorkos-history-v1")); + + let provider_source_packet = search_packet( + &store, + "dorkos-source-filter-needle", + &PacketOptions { + limit: 10, + filters: SearchFilters { + provider: Some(CaptureProvider::Custom), + provider_key: Some("dorkos".into()), + source_id: Some("default".into()), + source_format: Some("dorkos-history-v1".into()), + ..SearchFilters::default() + }, + ..PacketOptions::default() + }, + ) + .unwrap(); + assert_eq!(provider_source_packet.results.len(), 1); + + let wrong_source = search_packet( + &store, + "dorkos-source-filter-needle", + &PacketOptions { + limit: 10, + filters: SearchFilters { + provider: Some(CaptureProvider::Custom), + history_source: Some("openclaw/default".into()), + ..SearchFilters::default() + }, + ..PacketOptions::default() + }, + ) + .unwrap(); + assert!(wrong_source.results.is_empty()); + } + + #[test] + fn fast_event_search_exposes_custom_history_source_identity() { + let (_temp, store) = test_store(); + let record = HistoryRecord::new( + "Large custom plugin import", + "ordinary body", + Vec::new(), + "agent_history", + Some("/workspace/custom".into()), + ); + store.insert_record(&record).unwrap(); + + let source_id = Uuid::parse_str("018f45d0-0000-7000-8000-000000000481").unwrap(); + store + .upsert_capture_source(&CaptureSource { + id: source_id, + descriptor: CaptureSourceDescriptor { + kind: CaptureSourceKind::ProviderImport, + provider: CaptureProvider::Custom, + machine_id: "machine-1".into(), + process_id: None, + cwd: Some("/workspace/custom".into()), + raw_source_path: Some("/tmp/large-dorkos/ctx-history-plugin.json".into()), + external_session_id: Some("ctx-history-jsonl-v1-large".into()), + }, + started_at: fixed_time(), + ended_at: None, + sync: SyncMetadata { + metadata: serde_json::json!({ + "source_metadata": { + "ctx_history_plugin": { + "plugin_name": "dorkos", + "plugin_source_id": "default", + "history_source": "dorkos/default" + }, + "ctx_history_jsonl_v1": { + "provider_key": "dorkos", + "source_id": "default", + "source_format": "dorkos-history-v1" + } + } + }), + ..sync_metadata() + }, + }) + .unwrap(); + + let session = Session { + id: Uuid::parse_str("018f45d0-0000-7000-8000-000000000482").unwrap(), + history_record_id: Some(record.id), + parent_session_id: None, + root_session_id: None, + capture_source_id: Some(source_id), + provider: CaptureProvider::Custom, + external_session_id: Some("ctx-history-jsonl-v1-large".into()), + external_agent_id: None, + agent_type: AgentType::Primary, + role_hint: Some("primary".into()), + is_primary: true, + status: SessionStatus::Imported, + transcript_blob_id: None, + started_at: fixed_time(), + ended_at: None, + timestamps: timestamps(), + sync: sync_metadata(), + }; + store.upsert_session(&session).unwrap(); + + let target_event_id = Uuid::parse_str("018f45d0-0000-7000-8000-000000000483").unwrap(); + for index in 0..=(LARGE_EVENT_CORPUS_THRESHOLD as u64) { + let event_id = if index == LARGE_EVENT_CORPUS_THRESHOLD as u64 { + target_event_id + } else { + Uuid::parse_str(&format!("018f45d0-0000-7000-8000-0000002{index:05x}")).unwrap() + }; + let text = if event_id == target_event_id { + "large-custom-source-identity-needle" + } else { + "ordinary large custom event" + }; + store + .upsert_event(&Event { + id: event_id, + seq: 40_000 + index, + history_record_id: Some(record.id), + session_id: Some(session.id), + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::Assistant), + occurred_at: fixed_time() + chrono::Duration::milliseconds(index as i64), + capture_source_id: Some(source_id), + payload: serde_json::json!({ + "body": { "text": text } + }), + payload_blob_id: None, + dedupe_key: Some(format!("large-custom-source-identity-{index}")), + redaction_state: RedactionState::SafePreview, + sync: sync_metadata(), + }) + .unwrap(); + } + store.refresh_search_index().unwrap(); + + let packet = search_packet( + &store, + "large-custom-source-identity-needle", + &PacketOptions { + limit: 5, + filters: SearchFilters { + provider: Some(CaptureProvider::Custom), + ..SearchFilters::default() + }, + ..PacketOptions::default() + }, + ) + .unwrap(); + + assert_eq!(packet.results.len(), 1); + let result = &packet.results[0]; + assert_eq!(result.event_id, Some(target_event_id)); + assert_eq!(result.history_source.as_deref(), Some("dorkos/default")); + assert_eq!(result.history_source_plugin.as_deref(), Some("dorkos")); + assert_eq!(result.provider_key.as_deref(), Some("dorkos")); + assert_eq!(result.source_id.as_deref(), Some("default")); + assert_eq!(result.source_format.as_deref(), Some("dorkos-history-v1")); + } + #[test] fn filtered_search_pages_past_fts_decoys() { let (_temp, store) = test_store(); diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index cc7c0211c..311d26980 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -324,6 +324,11 @@ pub struct EventSearchHit { pub score: f64, pub provider: Option, pub session_external_session_id: Option, + pub history_source: Option, + pub history_source_plugin: Option, + pub provider_key: Option, + pub source_id: Option, + pub source_format: Option, pub agent_type: Option, pub session_is_primary: Option, pub cwd: Option, @@ -3716,6 +3721,8 @@ impl Store { |row| { let payload_json = row.get::<_, String>(18)?; let source_metadata_json = row.get::<_, Option>(19)?; + let source_identity = + event_search_source_identity(source_metadata_json.as_deref())?; Ok(EventSearchHit { event_id: parse_uuid(row.get::<_, String>(0)?)?, history_record_id: parse_optional_uuid(row.get(1)?)?, @@ -3729,6 +3736,11 @@ impl Store { score: row.get(9)?, provider: parse_optional_text_enum::(row.get(10)?)?, session_external_session_id: row.get(11)?, + history_source: source_identity.history_source, + history_source_plugin: source_identity.history_source_plugin, + provider_key: source_identity.provider_key, + source_id: source_identity.source_id, + source_format: source_identity.source_format, session_parent_session_id: parse_optional_uuid(row.get(12)?)?, session_root_session_id: parse_optional_uuid(row.get(13)?)?, agent_type: parse_optional_text_enum::(row.get(14)?)?, @@ -7388,6 +7400,83 @@ fn event_search_cursor( .map(str::to_owned)) } +#[derive(Default)] +struct EventSearchSourceIdentity { + history_source: Option, + history_source_plugin: Option, + provider_key: Option, + source_id: Option, + source_format: Option, +} + +fn event_search_source_identity( + source_metadata_json: Option<&str>, +) -> rusqlite::Result { + let Some(source_metadata_json) = source_metadata_json else { + return Ok(EventSearchSourceIdentity::default()); + }; + let metadata: serde_json::Value = serde_json::from_str(source_metadata_json) + .map_err(|err| rusqlite::Error::ToSqlConversionFailure(Box::new(err)))?; + let source_metadata = metadata + .get("source_metadata") + .and_then(serde_json::Value::as_object); + let plugin = source_metadata + .and_then(|metadata| metadata.get("ctx_history_plugin")) + .or_else(|| metadata.get("ctx_history_plugin")) + .and_then(serde_json::Value::as_object); + let custom = source_metadata + .and_then(|metadata| metadata.get("ctx_history_jsonl_v1")) + .or_else(|| metadata.get("ctx_history_jsonl_v1")) + .and_then(serde_json::Value::as_object); + let plugin_name = plugin + .and_then(|plugin| plugin.get("plugin_name")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let plugin_source_id = plugin + .and_then(|plugin| plugin.get("plugin_source_id")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let history_source = plugin + .and_then(|plugin| plugin.get("history_source")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .or_else(|| { + plugin_name + .as_deref() + .zip(plugin_source_id.as_deref()) + .map(|(plugin_name, source_id)| format!("{plugin_name}/{source_id}")) + }); + let provider_key = custom + .and_then(|custom| custom.get("provider_key")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let source_id = custom + .and_then(|custom| custom.get("source_id")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + let source_format = custom + .and_then(|custom| custom.get("source_format")) + .and_then(serde_json::Value::as_str) + .or_else(|| { + source_metadata + .and_then(|metadata| metadata.get("source_format")) + .and_then(serde_json::Value::as_str) + }) + .or_else(|| { + metadata + .get("source_format") + .and_then(serde_json::Value::as_str) + }) + .map(str::to_owned); + Ok(EventSearchSourceIdentity { + history_source, + history_source_plugin: plugin_name, + provider_key, + source_id, + source_format, + }) +} + fn collect_rows( rows: rusqlite::MappedRows<'_, impl FnMut(&rusqlite::Row<'_>) -> rusqlite::Result>, ) -> Result> { diff --git a/docs/cli-reference.md b/docs/cli-reference.md index a238edf06..cd0e1a76a 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -69,9 +69,11 @@ Native JSON rows include `provider`, `path`, `exists`, `source_format`, `status`, `import_support`, `native_import`, `importable`, `raw_retention`, and any `unsupported_reason`. Plugin JSON rows use `kind: "history_source_plugin"` and include `plugin`, `history_source`, -`provider_key`, `source_id`, `manifest_path`, and `enabled`. `sources` reads -path metadata and plugin manifests, writes nothing to provider files or source -repositories, and does not execute plugin commands. +`provider_key`, `source_id`, `manifest_path`, and `enabled`. Invalid installed +plugin manifests appear as non-importable plugin rows with `status: "invalid"` +and an `error`. `sources` reads path metadata and plugin manifests, writes +nothing to provider files or source repositories, and does not execute plugin +commands. ## Import @@ -94,11 +96,9 @@ ctx import --provider factory-ai-droid ctx import --path ~/.codex/sessions ctx import --provider pi --path ~/.pi/sessions.jsonl ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl -ctx import --history-source dorkos -ctx import --plugin dorkos/default +ctx import --history-source dorkos/default ctx import --history-source-manifest ./ctx-history-plugin.json -ctx import --plugin-manifest ./ctx-history-plugin.json -ctx import --history-source hermes --reset-cursor +ctx import --history-source hermes/default --reset-cursor ctx import --resume ctx import --json ctx import --progress json --json @@ -120,9 +120,10 @@ the schema and incremental semantics. History-source plugins are local command adapters that stream `ctx-history-jsonl-v1` to stdout. Use `--history-source ` for an explicit plugin import, or `--history-source-manifest ` to test a manifest -without installing it. `--plugin` and `--plugin-manifest` are aliases. -`--reset-cursor` withholds the previous plugin cursor for that run and asks the -plugin to perform a full rescan. See `docs/history-source-plugins.md`. +without installing it. Selectors are exact `plugin/source` or +`provider_key/source_id` values. `--reset-cursor` withholds the previous plugin +cursor for that run and asks the plugin to perform a full rescan. See +`docs/history-source-plugins.md`. Import selection rules: @@ -198,6 +199,8 @@ ctx search "token budget" --limit 5 ctx search "token budget" --session ctx search "review findings" --include-subagents ctx search "this current task" --include-current-session +ctx search "release notes" --history-source dorkos/default +ctx search "release notes" --provider-key dorkos --source-id default ``` `search` defaults to `--refresh auto`, which quietly refreshes discovered native @@ -222,6 +225,10 @@ for dense event-level results across sessions. Repeat `--term ` when you want to broaden a search across several related words or phrases and merge the ranked results; `--term` is OR-style broadening, not a must-include filter. +Custom history imports can be filtered by `--history-source` using +`plugin/source` or `provider_key/source_id`, or by exact `--provider-key`, +`--source-id`, and `--source-format` values. These filters imply +`--provider custom` and cannot be combined with another provider. Default search excludes subagent sessions so primary human-agent intent and decisions stay prominent. Use `--include-subagents` when implementation details, code review notes, test output, or failure analysis from subagent sessions diff --git a/docs/custom-history-import-format.md b/docs/custom-history-import-format.md index 704209db8..10266f6a0 100644 --- a/docs/custom-history-import-format.md +++ b/docs/custom-history-import-format.md @@ -14,7 +14,7 @@ ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl or from a local history-source plugin command: ```bash -ctx import --history-source my-agent +ctx import --history-source my-agent/default ``` ctx does not discover a fixed storage location for this format. File imports @@ -221,9 +221,10 @@ exporter-supplied cursor object in source metadata. Event `native_cursor` values are also preserved. For plugin imports, ctx passes the previously stored source cursor to the next -command through `CTX_HISTORY_CURSOR_JSON` and `CTX_HISTORY_CURSOR_FILE`. The -cursor string remains exporter-owned, so it can encode byte offsets, SQLite row -ids, session sequence maps, or another native high-water mark. +command through `CTX_HISTORY_CURSOR` for small cursors and always through +`CTX_HISTORY_CURSOR_FILE` when a previous cursor exists. The cursor string +remains exporter-owned, so it can encode byte offsets, SQLite row ids, session +sequence maps, or another native high-water mark. If an import is interrupted, run the same command again. File imports perform another idempotent rescan. Plugin imports receive the last successfully stored diff --git a/docs/history-source-plugin-design.md b/docs/history-source-plugin-design.md new file mode 100644 index 000000000..7a7bec6a6 --- /dev/null +++ b/docs/history-source-plugin-design.md @@ -0,0 +1,350 @@ +# History Source Plugin Design + +This document describes the history source plugin architecture as implemented +on `codex/history-source-plugins`. + +## Problem + +ctx has first-party local history adapters for common agent tools, but the agent +ecosystem changes quickly and many tools use custom local storage. Maintaining a +native adapter for every tool would couple ctx to unstable schemas owned by other +projects. + +The goal is to let unsupported agents make their history searchable in ctx +without ctx learning their native storage shape. + +The integration must: + +- work fully locally; +- support incremental refresh before `ctx search`; +- avoid a hosted plugin store or in-process extension ABI; +- keep adapter ownership with the third-party tool or user; +- reuse the existing ctx capture, store, and search pipeline; +- provide a batch escape hatch for tools that only write files. + +## Non-Goals + +This design does not add: + +- an in-process plugin ABI; +- a remote marketplace; +- background daemon scheduling; +- plugin installation management; +- native adapters for every third-party agent; +- a guarantee that plugin commands are sandboxed from the local user account. + +Plugins are local commands. A user or local tool that installs a plugin is +choosing to run that command with the user's normal local permissions. + +## User Model + +There are two supported paths. + +The preferred path for ongoing integrations is a history source plugin: + +1. A manifest declares one or more local history sources. +2. ctx discovers the manifest. +3. ctx runs the source command during explicit import or search refresh. +4. The command writes `ctx-history-jsonl-v1` records to stdout. +5. ctx imports the stream and stores the latest cursor. +6. On the next run, ctx passes that cursor back to the command. + +The optional batch path is a file import: + +1. A tool writes `ctx-history-jsonl-v1` records to a file. +2. The user or tool runs `ctx import --format ctx-history-jsonl-v1 --path ...`. +3. ctx imports the file idempotently. + +The file path is useful for simple exporters, debugging, and one-time imports. +It is not the best path for day-to-day refresh because ctx cannot discover, +invoke, or cursor an arbitrary file writer by itself. + +## Public Contracts + +The architecture has two public contracts. + +### Manifest Contract + +A plugin manifest is JSON at `ctx-history-plugin.json`. + +Manifests can be discovered from: + +- `$CTX_DATA_ROOT/plugins//ctx-history-plugin.json`; +- entries in `CTX_HISTORY_PLUGIN_PATH`. + +The implemented schema is: + +```json +{ + "schema_version": 1, + "name": "example-agent", + "display_name": "Example Agent", + "version": "0.1.0", + "history_sources": [ + { + "id": "default", + "display_name": "Example local history", + "provider_key": "example-agent", + "source_id": "default", + "source_format": "example-agent-sqlite-v1", + "enabled": true, + "refresh": "auto", + "command": ["example-agent-to-ctx", "export"], + "working_dir": ".", + "env": { + "EXAMPLE_AGENT_PROFILE": "default" + }, + "timeout_seconds": 300 + } + ] +} +``` + +`schema_version`, `name`, `history_sources[].id`, `source_format`, and +`command` are required. + +`provider_key` defaults to the manifest `name`. `source_id` defaults to the +source `id`. `enabled` defaults to `false`. `refresh` defaults to `manual`. +`timeout_seconds` defaults to 300 seconds and is clamped to at least 1 second. + +Identifiers must be stable lowercase ASCII values with digits, `.`, `_`, or +`-`. They must start with a lowercase ASCII letter or digit and be no more than +128 bytes. + +`command` is an argv array. ctx does not execute it through a shell. + +### Stream Contract + +Plugin commands and batch files emit `ctx-history-jsonl-v1`. + +Each line is a JSON object with one of these `record_type` values: + +- `manifest`; +- `source`; +- `session`; +- `event`; +- `file_touch`; +- `edge`. + +The stream contract intentionally mirrors the normalized shape ctx already +stores: + +- source metadata identifies the exporter, native format, cursor, machine, and + raw input; +- sessions represent conversations, tasks, runs, branches, or subagents; +- events represent ordered messages and tool events; +- file touches connect history to code search and audit workflows; +- edges preserve parent-child, spawned, forked, resumed, or related sessions. + +ctx stores these imports under the bounded internal provider `custom`, while +preserving exporter-owned `provider_key`, `source_id`, `source_format`, +`session_id`, and native metadata. + +## Runtime Contract + +Before running a plugin command, ctx sets: + +- `CTX_DATA_ROOT`; +- `CTX_HISTORY_PLUGIN=1`; +- `CTX_HISTORY_PLUGIN_NAME`; +- `CTX_HISTORY_PLUGIN_MANIFEST`; +- `CTX_HISTORY_SOURCE`, such as `example-agent/default`; +- `CTX_HISTORY_SOURCE_ID`; +- `CTX_HISTORY_PROVIDER_KEY`; +- `CTX_HISTORY_SOURCE_FORMAT`; +- `CTX_HISTORY_CURSOR_STREAM`; +- `CTX_HISTORY_MACHINE_ID`; +- `CTX_HISTORY_FULL_RESCAN`, `1` or `0`; +- `CTX_HISTORY_CURSOR`, when a previous cursor exists and is small enough; +- `CTX_HISTORY_CURSOR_FILE`, a temporary file containing the previous cursor. + +Plugins should read `CTX_HISTORY_CURSOR_FILE` first. Inline cursor environment +variables are only a convenience for small cursors. + +Plugins must write only `ctx-history-jsonl-v1` to stdout. Progress and warnings +belong on stderr. + +ctx clears the inherited environment and re-adds a small allowlist: + +- `PATH`; +- `HOME`; +- user and locale variables; +- temporary-directory variables; +- XDG data, config, cache, and state roots. + +Manifest `env` entries are then added. This avoids accidental dependence on the +parent shell while still allowing plugin authors to pass explicit configuration. + +## Incremental Semantics + +The plugin owns cursor meaning. ctx treats the cursor as an opaque string. + +Examples: + +- append-only files can use byte offsets; +- SQLite stores can use row ids; +- split stores can use JSON maps keyed by session id or file path; +- API-backed local tools can use an opaque sync token. + +On a successful import, ctx stores the cursor emitted by the plugin's `source` +record. Failed runs do not advance the cursor. + +`ctx import --history-source ... --reset-cursor` withholds the previous cursor +and sets `CTX_HISTORY_FULL_RESCAN=1`. A reset plugin run must emit a fresh +`source.cursor.after` checkpoint; otherwise ctx rejects the run so an old stored +cursor cannot be reused accidentally. + +`ctx search` uses the same pre-search refresh model as native provider sources: + +- `--refresh auto` best-effort refreshes enabled auto plugins and then searches + the current index; +- `--refresh strict` fails if refresh cannot complete; +- `--refresh off` never executes plugin commands. + +Provider-filtered search only runs plugin refresh when the provider filter is +`custom` or absent. + +## Import And Discovery Behavior + +`ctx sources` lists plugin sources without executing plugin commands. + +`ctx import --history-source ` runs exactly one matching source. +Selectors can match: + +- `plugin/source`; +- `provider_key/source_id`. + +The selector must resolve to one source before ctx runs anything. + +`ctx import --history-source-manifest ` adds a manifest for the current +command without installing it. + +`ctx import --all` includes enabled plugin sources, plus discovered native +provider sources. + +`ctx setup` does not execute plugin commands. + +## Failure Model + +Plugin runs fail closed for that run: + +- nonzero exit status fails the run; +- invalid stdout fails the run; +- stdout over 64 MiB fails the run; +- stderr over 256 KiB fails the run; +- timeout fails the run; +- source identity mismatches fail before records are imported. + +For explicit single-source imports, failures are returned to the user. For +`ctx import --all`, plugin failures can be reported as source failures without +discarding successful imports from other sources. For `ctx search --refresh +auto`, failures are recorded as refresh failures and search continues against +the existing index. + +The cursor only advances after a successful source import, so the usual recovery +path is to fix the plugin and run the same command again. + +## Security And Trust + +This architecture reduces ctx's native schema maintenance burden, but it does +not make third-party code harmless. A plugin command is local code. It can read +whatever the current user can read unless the operating system or user wraps it +in additional isolation. + +The implemented mitigations are practical guardrails: + +- commands are argv arrays, not shell strings; +- ctx clears the environment and re-adds only a small allowlist; +- stdin is closed; +- stdout, stderr, and runtime are bounded; +- cursor files are private temporary files on Unix; +- plugin discovery never executes commands; +- invalid installed manifests are reported by `ctx sources` as non-importable + rows; +- selectors fail before execution unless they identify exactly one source. + +The product should describe plugins as local adapters, not as trusted apps from +ctx. + +## Why This Is Smaller Than Native Adapter Expansion + +Adding a native adapter requires ctx to own: + +- discovery paths; +- native schema parsing; +- incremental logic; +- storage migrations or upstream compatibility breaks; +- tests and fixtures for that provider forever. + +The plugin model keeps ctx's owned surface to: + +- one manifest schema; +- one stream schema; +- one command runtime; +- one cursor handoff; +- common validation and import behavior. + +That is still a public API commitment, but it is a narrower and more durable +commitment than chasing every custom agent database. + +## Why Keep Batch File Import + +The batch importer uses the same stream parser as plugins. Keeping it provides a +low-friction path for: + +- one-off migration; +- local debugging; +- agents that can write a file but cannot easily be invoked by ctx; +- support reproduction cases; +- tests for the stream contract independent of process execution. + +The UX distinction should stay clear: + +- use a plugin for ongoing search-time refresh; +- use a file for explicit batch import. + +## Current Implementation + +The branch adds: + +- `crates/ctx-history-core/src/history_jsonl.rs` for typed + `ctx-history-jsonl-v1` records; +- custom-history normalization and import in `ctx-history-capture`; +- `crates/ctx-cli/src/history_source_plugins.rs` for manifest discovery, + command execution, cursor environment, timeout, and output limits; +- CLI support for `--history-source` and `--history-source-manifest`; +- search refresh support for enabled auto plugin sources; +- source listing for valid and invalid installed plugin manifests; +- source-aware search filters for `--history-source`, `--provider-key`, + `--source-id`, and `--source-format`; +- plugin identity metadata on imported custom sources; +- docs for the stream format and plugin manifest; +- tests for schema round trips, malformed streams, idempotency, cursors, + discovery, explicit imports, `import --all`, search refresh, failures, and + timeouts. + +## Open Questions Before Shipping + +The implementation is mergeable, but these are the product/API questions worth +settling before a stable release: + +- Should `ctx-history-jsonl-v1` be documented as stable immediately, or marked + preview while plugin feedback is collected? +- Should direct file import stay in public CLI help, or be documented as a + batch/debug path behind the plugin story? +- Should the manifest support a semver range for stream schema versions before + v2 exists, or is `schema_version: 1` enough for now? +- Should plugin commands receive `CTX_HISTORY_CURSOR_FILE` even when no cursor + exists, containing a well-known empty value, or is absence simpler? +- Should auto-refresh plugin failures appear more visibly in normal human + `ctx search` output, or is current best-effort behavior enough? + +## Recommendation + +Ship the plugin architecture after final API wording review. It solves a real +integration problem with a small local contract, keeps ctx out of third-party +storage schemas, and preserves the native-provider experience of incremental +refresh before search. + +Keep the batch file importer, but position it as an optional explicit path. The +preferred ongoing integration should remain manifest plus command stdout. diff --git a/docs/history-source-plugins.md b/docs/history-source-plugins.md index 398e1c70b..fd6773d61 100644 --- a/docs/history-source-plugins.md +++ b/docs/history-source-plugins.md @@ -21,11 +21,11 @@ manifest, cursor handoff, validation, import, and search index. Put a manifest at one of: - `$CTX_DATA_ROOT/plugins//ctx-history-plugin.json`; -- any directory or manifest file listed in `CTX_HISTORY_PLUGIN_PATH`; -- any directory or manifest file listed in `CTX_PLUGIN_PATH`. +- any directory or manifest file listed in `CTX_HISTORY_PLUGIN_PATH`. `ctx sources` and `ctx sources --json` list plugin sources without executing -their commands. +their commands. Invalid installed manifests are listed as non-importable +`history_source_plugin` rows so authors can diagnose broken local config. Manifest example: @@ -61,18 +61,16 @@ imports can run a discovered source even when it is not enabled or is marked ## Import ```bash -ctx import --history-source dorkos -ctx import --plugin dorkos ctx import --history-source dorkos/default ctx import --history-source-manifest ./ctx-history-plugin.json -ctx import --plugin-manifest ./ctx-history-plugin.json ctx import --all -ctx import --history-source hermes --reset-cursor +ctx import --history-source hermes/default --reset-cursor ``` -Selectors can match plugin name, source id, `plugin/source`, `provider_key`, or -`provider_key/source_id`, but they must resolve to exactly one source before ctx -executes a command. Prefer `plugin/source` when a machine has multiple plugins. +Selectors match `plugin/source` or `provider_key/source_id`, and must resolve +to exactly one source before ctx executes a command. ctx does not accept bare +plugin names, bare source ids, or bare provider keys because many integrations +use ids like `default`. `--history-source-manifest` is a development path: it adds that manifest for the current command without installing it. With no selector, ctx imports sources @@ -80,7 +78,8 @@ from the supplied manifest path. `--reset-cursor` withholds the previous cursor and sets `CTX_HISTORY_FULL_RESCAN=1`. The plugin should emit a fresh `source.cursor.after` -checkpoint if the rescan succeeds. +checkpoint if the rescan succeeds; ctx rejects reset runs that do not emit a +new after checkpoint so an old cursor cannot be reused accidentally. `ctx setup` does not execute plugins. `ctx search` defaults to `--refresh auto` and runs discovered plugin sources only when they are both `enabled: true` and @@ -88,6 +87,17 @@ and runs discovered plugin sources only when they are both `enabled: true` and fails if an auto plugin refresh fails. Plugin refresh is incremental because ctx passes the previously stored source cursor before invoking the command. +Search can be limited to a custom history source after import: + +```bash +ctx search "release notes" --history-source dorkos/default +ctx search "release notes" --provider-key dorkos --source-id default +ctx search "release notes" --source-format dorkos-claude-jsonl-v1 +``` + +These filters imply `--provider custom`; combining them with another provider is +an error. + ## Runtime Environment ctx sets these variables before invoking a plugin command: @@ -105,7 +115,6 @@ ctx sets these variables before invoking a plugin command: - `CTX_HISTORY_FULL_RESCAN`, `1` or `0` - `CTX_HISTORY_CURSOR`, when a previous cursor exists and is small enough for inline environment handoff -- `CTX_HISTORY_CURSOR_JSON`, same value as `CTX_HISTORY_CURSOR` when set - `CTX_HISTORY_CURSOR_FILE`, a temporary file containing the cursor Use `CTX_HISTORY_CURSOR_FILE` for large native cursor maps. The file exists only @@ -147,8 +156,9 @@ writing imported rows. ## Adapter Shapes -The local research checkouts showed four different storage models, which is why -ctx should not maintain native adapters for them. +The examples below are illustrative shapes for plugin authors. Some of these +providers also have native ctx support; plugins are still useful for custom +forks, private variants, or newer schemas that ctx does not support yet. ### DorkOS @@ -207,9 +217,12 @@ as Claude JSONL, when they need full internal tool/thinking events. ## Minimal Plugin Pseudocode ```python -import json, os, sqlite3, sys +import json, os, pathlib, sqlite3, sys -cursor = json.loads(os.environ.get("CTX_HISTORY_CURSOR_JSON") or "{}") +cursor_text = os.environ.get("CTX_HISTORY_CURSOR") +if not cursor_text and os.environ.get("CTX_HISTORY_CURSOR_FILE"): + cursor_text = pathlib.Path(os.environ["CTX_HISTORY_CURSOR_FILE"]).read_text() +cursor = json.loads(cursor_text or "{}") after_message_id = cursor.get("message_id", 0) db = sqlite3.connect(os.path.expanduser("~/.hermes/state.db")) diff --git a/docs/search.md b/docs/search.md index fc1a10cf3..4e9754826 100644 --- a/docs/search.md +++ b/docs/search.md @@ -55,6 +55,10 @@ that support it. Search filters narrow both human output and JSON: - `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; +- `--history-source `, for custom + history imports; +- `--provider-key `, `--source-id `, and + `--source-format `, for exact custom history source filters; - `--workspace `, substring match over stored workspace, cwd, source path, or repository-name text; - `--since d`; diff --git a/docs/storage.md b/docs/storage.md index ac24bf4df..85230f6c7 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -80,7 +80,7 @@ analytics marker described under network behavior. | `ctx import` | provider transcript files and path metadata, the explicit custom history JSONL file passed with `--format ctx-history-jsonl-v1 --path`, or stdout from an explicit history-source plugin command | data root, `config.toml` if missing, and SQLite index | | `ctx show` | SQLite index | selected `--out` path for `show session` when provided | | `ctx locate` | SQLite index and raw source path metadata | none | -| `ctx search` | native provider transcript files, path metadata, and SQLite index | SQLite index for newly discovered native provider history | +| `ctx search` | native provider transcript files, path metadata, enabled auto history-source plugin stdout, and SQLite index | SQLite index for newly discovered native provider or plugin history | | `ctx sql` | existing SQLite index only | none | | `ctx docs` | embedded documentation in the binary | selected topic `--out` path for `ctx docs show --out` or selected `--out` directory for `ctx docs man --out` | | `ctx upgrade` | signed release metadata and installed binary/sidecar metadata | installed binary for manual upgrade, install sidecar, `upgrade-state.json`, `upgrade.lock`, and `logs/upgrade.log` | @@ -127,7 +127,7 @@ ctx import --all ctx import --resume ctx import --path ~/.codex/sessions ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl -ctx import --history-source dorkos +ctx import --history-source dorkos/default ``` Current adapters are safe to re-run. They rescan sources idempotently and keep From 4dde2d962dc8eea435218f102d64d5fc1e490595 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:31:57 -0500 Subject: [PATCH 32/72] Link history source plugin docs from README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1f3e4abab..00a8a184e 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ ctx keeps retrieval tied to sessions and events, so another agent can inspect th | [Install the ctx skill](https://ctx.rs/skill) | Install the agent-history search skill with the open skills installer. | | [Agent plugin installs](docs/agent-skill-install.md) | Install the ctx skill through Codex, Claude Code, Cursor, or a raw skill folder. | | [SDKs](docs/sdks.md) | Use ctx agent history search from TypeScript, Python, Rust, Go, JVM, Swift, or .NET code. | +| [Custom history plugins](docs/history-source-plugins.md) | Build an advanced local adapter for unsupported agent history formats. | | [Cursor](https://ctx.rs/agents/cursor) | Import Cursor agent transcripts and ask Cursor to cite retrieved local history before editing. | | [How it works](https://ctx.rs/concepts/how-it-works) | Understand discovery, import, SQLite storage, search refresh, and cited retrieval. | | [Supported agents](https://ctx.rs/concepts/supported-agents) | See which agent histories ctx can discover, import, and search today. | From 10cfd7113483a3c177d237bc878298efcb948390 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:21:21 -0500 Subject: [PATCH 33/72] Tighten history source plugin docs --- README.md | 2 +- docs/cli-reference.md | 16 ++--- docs/custom-history-import-format.md | 4 +- docs/history-source-plugin-design.md | 25 ++++--- docs/history-source-plugins.md | 100 +++++++++++++-------------- docs/storage.md | 2 +- 6 files changed, 71 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 00a8a184e..bcd3cc684 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ ctx keeps retrieval tied to sessions and events, so another agent can inspect th | [Install the ctx skill](https://ctx.rs/skill) | Install the agent-history search skill with the open skills installer. | | [Agent plugin installs](docs/agent-skill-install.md) | Install the ctx skill through Codex, Claude Code, Cursor, or a raw skill folder. | | [SDKs](docs/sdks.md) | Use ctx agent history search from TypeScript, Python, Rust, Go, JVM, Swift, or .NET code. | -| [Custom history plugins](docs/history-source-plugins.md) | Build an advanced local adapter for unsupported agent history formats. | +| [Custom history plugins](docs/history-source-plugins.md) | Build an advanced local adapter for agent formats ctx does not support natively. | | [Cursor](https://ctx.rs/agents/cursor) | Import Cursor agent transcripts and ask Cursor to cite retrieved local history before editing. | | [How it works](https://ctx.rs/concepts/how-it-works) | Understand discovery, import, SQLite storage, search refresh, and cited retrieval. | | [Supported agents](https://ctx.rs/concepts/supported-agents) | See which agent histories ctx can discover, import, and search today. | diff --git a/docs/cli-reference.md b/docs/cli-reference.md index cd0e1a76a..c8c23d98a 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -96,9 +96,9 @@ ctx import --provider factory-ai-droid ctx import --path ~/.codex/sessions ctx import --provider pi --path ~/.pi/sessions.jsonl ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl -ctx import --history-source dorkos/default +ctx import --history-source example-agent/default ctx import --history-source-manifest ./ctx-history-plugin.json -ctx import --history-source hermes/default --reset-cursor +ctx import --history-source example-agent/default --reset-cursor ctx import --resume ctx import --json ctx import --progress json --json @@ -117,10 +117,10 @@ Custom history can be imported from an explicit JSONL file with remembered as a provider home; see `docs/custom-history-import-format.md` for the schema and incremental semantics. -History-source plugins are local command adapters that stream -`ctx-history-jsonl-v1` to stdout. Use `--history-source ` for an -explicit plugin import, or `--history-source-manifest ` to test a manifest -without installing it. Selectors are exact `plugin/source` or +History-source plugins are local commands that stream `ctx-history-jsonl-v1` to +stdout. Use `--history-source ` for an explicit plugin import, or +`--history-source-manifest ` to test a manifest without installing it. +Selectors are exact `plugin/source` or `provider_key/source_id` values. `--reset-cursor` withholds the previous plugin cursor for that run and asks the plugin to perform a full rescan. See `docs/history-source-plugins.md`. @@ -199,8 +199,8 @@ ctx search "token budget" --limit 5 ctx search "token budget" --session ctx search "review findings" --include-subagents ctx search "this current task" --include-current-session -ctx search "release notes" --history-source dorkos/default -ctx search "release notes" --provider-key dorkos --source-id default +ctx search "release notes" --history-source example-agent/default +ctx search "release notes" --provider-key example-agent --source-id default ``` `search` defaults to `--refresh auto`, which quietly refreshes discovered native diff --git a/docs/custom-history-import-format.md b/docs/custom-history-import-format.md index 10266f6a0..29047965a 100644 --- a/docs/custom-history-import-format.md +++ b/docs/custom-history-import-format.md @@ -18,8 +18,8 @@ ctx import --history-source my-agent/default ``` ctx does not discover a fixed storage location for this format. File imports -are explicit paths. Plugin imports are explicit local command adapters declared -by a local manifest; see `docs/history-source-plugins.md`. +are explicit paths. Plugin imports run local commands declared by a local +manifest; see `docs/history-source-plugins.md`. Each line is one JSON object. Every object has a `record_type` field with one of: diff --git a/docs/history-source-plugin-design.md b/docs/history-source-plugin-design.md index 7a7bec6a6..7f2d9ce60 100644 --- a/docs/history-source-plugin-design.md +++ b/docs/history-source-plugin-design.md @@ -1,7 +1,7 @@ # History Source Plugin Design -This document describes the history source plugin architecture as implemented -on `codex/history-source-plugins`. +This document describes history source plugins as implemented on +`codex/history-source-plugins`. ## Problem @@ -10,8 +10,8 @@ ecosystem changes quickly and many tools use custom local storage. Maintaining a native adapter for every tool would couple ctx to unstable schemas owned by other projects. -The goal is to let unsupported agents make their history searchable in ctx -without ctx learning their native storage shape. +Plugins let tools that ctx does not support natively make their history +searchable without ctx learning their storage shape. The integration must: @@ -61,7 +61,7 @@ invoke, or cursor an arbitrary file writer by itself. ## Public Contracts -The architecture has two public contracts. +The feature has two public contracts. ### Manifest Contract @@ -246,10 +246,10 @@ path is to fix the plugin and run the same command again. ## Security And Trust -This architecture reduces ctx's native schema maintenance burden, but it does -not make third-party code harmless. A plugin command is local code. It can read -whatever the current user can read unless the operating system or user wraps it -in additional isolation. +This reduces ctx's native schema maintenance burden. It does not make +third-party code harmless. A plugin command is local code. It can read whatever +the current user can read unless the operating system or user wraps it in +additional isolation. The implemented mitigations are practical guardrails: @@ -341,10 +341,9 @@ settling before a stable release: ## Recommendation -Ship the plugin architecture after final API wording review. It solves a real -integration problem with a small local contract, keeps ctx out of third-party -storage schemas, and preserves the native-provider experience of incremental -refresh before search. +Ship this after final API wording review. It solves a real integration problem +with a small local contract, keeps ctx out of third-party storage schemas, and +preserves the native-provider experience of incremental refresh before search. Keep the batch file importer, but position it as an optional explicit path. The preferred ongoing integration should remain manifest plus command stdout. diff --git a/docs/history-source-plugins.md b/docs/history-source-plugins.md index fd6773d61..7d8f8a2b4 100644 --- a/docs/history-source-plugins.md +++ b/docs/history-source-plugins.md @@ -1,9 +1,9 @@ # History Source Plugins -History source plugins let third-party tools make their local histories -searchable in ctx without ctx owning their storage schemas. +History source plugins let local tools make their histories searchable in ctx +without ctx owning their storage schemas. -The narrow waist is: +A plugin integration works like this: 1. A local manifest declares one or more history sources. 2. ctx invokes enabled auto-refresh commands during search refresh, or any @@ -12,9 +12,10 @@ The narrow waist is: 4. The stream is checked and imported as one batch. 5. ctx passes the previous source cursor back on the next run. -Plugins are command-line adapters, not an in-process ABI and not a hosted plugin -store. Plugin authors own their native JSONL, SQLite, or API reads. ctx owns the -manifest, cursor handoff, validation, import, and search index. +Plugins run as local commands. ctx does not load plugin code in-process or +operate a plugin store. Plugin authors own their native JSONL, SQLite, or API +reads. ctx owns the manifest, cursor handoff, validation, import, and search +index. ## Install And Discover @@ -32,18 +33,18 @@ Manifest example: ```json { "schema_version": 1, - "name": "dorkos", - "display_name": "DorkOS history", + "name": "example-agent", + "display_name": "Example Agent history", "version": "0.1.0", "history_sources": [ { "id": "default", - "provider_key": "dorkos", + "provider_key": "example-agent", "source_id": "default", - "source_format": "dorkos-claude-jsonl-v1", + "source_format": "example-agent-sqlite-v1", "enabled": true, "refresh": "auto", - "command": ["ctx-history-source-dorkos", "export"], + "command": ["example-agent-to-ctx", "export"], "timeout_seconds": 300 } ] @@ -61,10 +62,10 @@ imports can run a discovered source even when it is not enabled or is marked ## Import ```bash -ctx import --history-source dorkos/default +ctx import --history-source example-agent/default ctx import --history-source-manifest ./ctx-history-plugin.json ctx import --all -ctx import --history-source hermes/default --reset-cursor +ctx import --history-source example-agent/default --reset-cursor ``` Selectors match `plugin/source` or `provider_key/source_id`, and must resolve @@ -90,9 +91,9 @@ passes the previously stored source cursor before invoking the command. Search can be limited to a custom history source after import: ```bash -ctx search "release notes" --history-source dorkos/default -ctx search "release notes" --provider-key dorkos --source-id default -ctx search "release notes" --source-format dorkos-claude-jsonl-v1 +ctx search "release notes" --history-source example-agent/default +ctx search "release notes" --provider-key example-agent --source-id default +ctx search "release notes" --source-format example-agent-sqlite-v1 ``` These filters imply `--provider custom`; combining them with another provider is @@ -106,7 +107,7 @@ ctx sets these variables before invoking a plugin command: - `CTX_HISTORY_PLUGIN=1` - `CTX_HISTORY_PLUGIN_NAME` - `CTX_HISTORY_PLUGIN_MANIFEST` -- `CTX_HISTORY_SOURCE`, such as `dorkos/default` +- `CTX_HISTORY_SOURCE`, such as `example-agent/default` - `CTX_HISTORY_SOURCE_ID` - `CTX_HISTORY_PROVIDER_KEY` - `CTX_HISTORY_SOURCE_FORMAT` @@ -154,66 +155,59 @@ Every plugin run should emit a `source` record matching the manifest `provider_key`, `source_id`, and `source_format`. ctx rejects mismatches before writing imported rows. -## Adapter Shapes +## Common Storage Shapes -The examples below are illustrative shapes for plugin authors. Some of these -providers also have native ctx support; plugins are still useful for custom -forks, private variants, or newer schemas that ctx does not support yet. +Use the cursor format that matches your native storage. ctx treats it as an +opaque string and passes it back on the next run. -### DorkOS +### Append-Only Files -DorkOS currently derives history from Claude SDK JSONL files under -`~/.claude/projects//*.jsonl`. A DorkOS plugin should read those files by -byte offset and use a cursor like: +For one JSONL transcript per session, read each file from the last imported byte +offset and store a cursor keyed by path: ```json -{"files":{"/home/me/.claude/projects/x/session.jsonl":{"offset":12345,"size":13000,"mtimeMs":1780000000000}}} +{"files":{"/home/me/.example-agent/sessions/a.jsonl":{"offset":12345,"size":13000,"mtimeMs":1780000000000}}} ``` -The plugin can enrich events with DorkOS metadata from `~/.dork/dork.db`, but -the transcript source is still the Claude JSONL file. +If a file shrinks or its fingerprint changes, rescan that file from the +beginning and emit the same stable session and event IDs. -### OpenClaw +### SQLite -OpenClaw currently has session metadata under -`~/.openclaw/agents//sessions/sessions.json` and transcript JSONL -files beside it. A plugin should use OpenClaw's session accessor where possible, -resolve transcript paths, and cursor by byte offset: +For a local database with monotonic message IDs, read rows above the previous +high-water mark and advance the cursor to the largest imported ID: ```json -{"backend":"openclaw-file","transcripts":{"/home/me/.openclaw/agents/a/sessions/s.jsonl":{"offset":456,"size":900,"lastRecordId":"rec-2"}}} +{"message_id":1234} ``` -If OpenClaw flips storage to SQLite, the OpenClaw-owned plugin can keep the same -ctx stdout contract while changing its native reader. +Use a second field if session metadata has its own reliable update marker: + +```json +{"message_id":1234,"session_updated_at":"2026-07-01T12:00:00Z"} +``` -### Hermes +### Split Stores -Hermes Agent stores canonical history in `~/.hermes/state.db`. A Hermes plugin -should read `sessions` and `messages` read-only, order by `messages.id`, and -cursor by the maximum message row id: +Some tools keep session metadata in one place and transcripts somewhere else. +Use a cursor map for each moving part: ```json -{"message_id":1234} +{"sessions_version":17,"transcripts":{"/home/me/.example-agent/transcripts/a.jsonl":{"offset":456,"size":900}}} ``` -Session metadata-only changes may need a second cursor if Hermes exposes a -reliable session update high-water mark. +The plugin can change how it reads native storage later without changing the ctx +manifest or stdout contract. -### NanoClaw +### Local APIs Or Commands -NanoClaw uses a central `data/v2.db` plus per-session inbound and outbound -SQLite databases under `data/v2-sessions///`. -Inbound messages use even `seq` values and outbound messages use odd `seq` -values. A generic NanoClaw plugin can cursor by per-session sequence: +If the tool already has an export command or local API, call that API and store +its sync token: ```json -{"sessions":{"sess-abc":42,"sess-def":8}} +{"sync_token":"opaque-provider-token"} ``` -Provider-specific NanoClaw plugins can instead read mounted provider state, such -as Claude JSONL, when they need full internal tool/thinking events. - ## Minimal Plugin Pseudocode ```python @@ -224,7 +218,7 @@ if not cursor_text and os.environ.get("CTX_HISTORY_CURSOR_FILE"): cursor_text = pathlib.Path(os.environ["CTX_HISTORY_CURSOR_FILE"]).read_text() cursor = json.loads(cursor_text or "{}") after_message_id = cursor.get("message_id", 0) -db = sqlite3.connect(os.path.expanduser("~/.hermes/state.db")) +db = sqlite3.connect(os.path.expanduser("~/.example-agent/state.db")) print(json.dumps({"record_type": "manifest", "schema_version": "ctx-history-jsonl-v1"})) print(json.dumps({ diff --git a/docs/storage.md b/docs/storage.md index 85230f6c7..2cd5c93db 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -127,7 +127,7 @@ ctx import --all ctx import --resume ctx import --path ~/.codex/sessions ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl -ctx import --history-source dorkos/default +ctx import --history-source example-agent/default ``` Current adapters are safe to re-run. They rescan sources idempotently and keep From f78a0973f0b7fd971af0f2d690ac2e31dca25af0 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:57:38 -0500 Subject: [PATCH 34/72] Release ctx CLI 0.16.0 --- Cargo.lock | 10 +-- crates/ctx-cli/Cargo.toml | 2 +- crates/ctx-cli/src/history_source_plugins.rs | 77 +++++++++++++++++--- crates/ctx-history-capture/Cargo.toml | 2 +- crates/ctx-history-core/Cargo.toml | 2 +- crates/ctx-history-search/Cargo.toml | 2 +- crates/ctx-history-store/Cargo.toml | 2 +- docs/history-source-plugins.md | 4 +- scripts/build-public-cli-artifact.sh | 10 +-- 9 files changed, 84 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db382477d..c73a7ae42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,7 +256,7 @@ dependencies = [ [[package]] name = "ctx" -version = "0.15.0" +version = "0.16.0" dependencies = [ "anyhow", "assert_cmd", @@ -282,7 +282,7 @@ dependencies = [ [[package]] name = "ctx-history-capture" -version = "0.15.0" +version = "0.16.0" dependencies = [ "chrono", "ctx-history-core", @@ -297,7 +297,7 @@ dependencies = [ [[package]] name = "ctx-history-core" -version = "0.15.0" +version = "0.16.0" dependencies = [ "chrono", "directories", @@ -310,7 +310,7 @@ dependencies = [ [[package]] name = "ctx-history-search" -version = "0.15.0" +version = "0.16.0" dependencies = [ "chrono", "ctx-history-core", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "ctx-history-store" -version = "0.15.0" +version = "0.16.0" dependencies = [ "chrono", "ctx-history-core", diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index d4f55aa96..f78231cc2 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx" -version = "0.15.0" +version = "0.16.0" description = "Local CLI for indexing and searching agent session history" edition.workspace = true autobins = false diff --git a/crates/ctx-cli/src/history_source_plugins.rs b/crates/ctx-cli/src/history_source_plugins.rs index 4b97a3f2f..a3c438490 100644 --- a/crates/ctx-cli/src/history_source_plugins.rs +++ b/crates/ctx-cli/src/history_source_plugins.rs @@ -2,7 +2,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, env, fs::{self, OpenOptions}, - io::{ErrorKind, Read, Write}, + io::{Read, Write}, path::{Path, PathBuf}, process::{Child, ChildStderr, ChildStdout, Command, ExitStatus, Stdio}, thread, @@ -11,6 +11,8 @@ use std::{ #[cfg(unix)] use std::os::unix::{fs::OpenOptionsExt, io::AsRawFd}; +#[cfg(not(unix))] +use std::sync::mpsc; use anyhow::{anyhow, Context, Result}; use serde::Deserialize; @@ -352,13 +354,28 @@ fn collect_child_output_with_timeout( timeout: Duration, source_label: &str, ) -> Result<(ExitStatus, Vec, Vec)> { + #[derive(Clone, Copy)] + enum PipeKind { + Stdout, + Stderr, + } + + let (tx, rx) = mpsc::channel(); let stdout_source = source_label.to_owned(); + let stdout_tx = tx.clone(); let stdout_handle = thread::spawn(move || { - read_pipe_with_limit(stdout, MAX_PLUGIN_STDOUT_BYTES, "stdout", &stdout_source) + let _ = stdout_tx.send(( + PipeKind::Stdout, + read_pipe_with_limit(stdout, MAX_PLUGIN_STDOUT_BYTES, "stdout", &stdout_source), + )); }); let stderr_source = source_label.to_owned(); + let stderr_tx = tx; let stderr_handle = thread::spawn(move || { - read_pipe_with_limit(stderr, MAX_PLUGIN_STDERR_BYTES, "stderr", &stderr_source) + let _ = stderr_tx.send(( + PipeKind::Stderr, + read_pipe_with_limit(stderr, MAX_PLUGIN_STDERR_BYTES, "stderr", &stderr_source), + )); }); let started = Instant::now(); @@ -377,12 +394,50 @@ fn collect_child_output_with_timeout( thread::sleep(Duration::from_millis(25)); }; - let stdout = stdout_handle - .join() - .map_err(|_| anyhow!("history source plugin stdout reader panicked"))??; - let stderr = stderr_handle - .join() - .map_err(|_| anyhow!("history source plugin stderr reader panicked"))??; + let mut stdout = None; + let mut stderr = None; + while stdout.is_none() || stderr.is_none() { + let Some(remaining) = timeout.checked_sub(started.elapsed()) else { + return Err(anyhow!( + "history source plugin {source_label} timed out after {}s", + timeout.as_secs() + )); + }; + if remaining == Duration::ZERO { + return Err(anyhow!( + "history source plugin {source_label} timed out after {}s", + timeout.as_secs() + )); + } + match rx.recv_timeout(remaining) { + Ok((PipeKind::Stdout, result)) => { + stdout = Some(result?); + } + Ok((PipeKind::Stderr, result)) => { + stderr = Some(result?); + } + Err(mpsc::RecvTimeoutError::Timeout) => { + return Err(anyhow!( + "history source plugin {source_label} timed out after {}s", + timeout.as_secs() + )); + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(anyhow!( + "history source plugin {source_label} output reader stopped before pipes were drained" + )); + } + } + } + + if stdout_handle.join().is_err() { + return Err(anyhow!("history source plugin stdout reader panicked")); + } + if stderr_handle.join().is_err() { + return Err(anyhow!("history source plugin stderr reader panicked")); + } + let stdout = stdout.expect("stdout reader result"); + let stderr = stderr.expect("stderr reader result"); Ok((status, stdout, stderr)) } @@ -423,8 +478,8 @@ fn read_available_with_limit( } bytes.extend_from_slice(&buffer[..count]); } - Err(err) if err.kind() == ErrorKind::WouldBlock => return Ok(()), - Err(err) if err.kind() == ErrorKind::Interrupted => continue, + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => return Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, Err(err) => { return Err(err) .with_context(|| format!("read history source plugin {source_label} {name}")) diff --git a/crates/ctx-history-capture/Cargo.toml b/crates/ctx-history-capture/Cargo.toml index 35751e600..1619eef1d 100644 --- a/crates/ctx-history-capture/Cargo.toml +++ b/crates/ctx-history-capture/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-capture" -version = "0.15.0" +version = "0.16.0" description = "Internal provider import adapters for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-core/Cargo.toml b/crates/ctx-history-core/Cargo.toml index e6ffa89f5..117957db5 100644 --- a/crates/ctx-history-core/Cargo.toml +++ b/crates/ctx-history-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-core" -version = "0.15.0" +version = "0.16.0" description = "Internal core types for ctx local agent history indexing" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-search/Cargo.toml b/crates/ctx-history-search/Cargo.toml index 0e4d93b80..db4fabc39 100644 --- a/crates/ctx-history-search/Cargo.toml +++ b/crates/ctx-history-search/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-search" -version = "0.15.0" +version = "0.16.0" description = "Internal search projection and ranking helpers for ctx" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-store/Cargo.toml b/crates/ctx-history-store/Cargo.toml index 02d1155a7..00b301ab7 100644 --- a/crates/ctx-history-store/Cargo.toml +++ b/crates/ctx-history-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-store" -version = "0.15.0" +version = "0.16.0" description = "Internal SQLite storage layer for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/docs/history-source-plugins.md b/docs/history-source-plugins.md index 7d8f8a2b4..1af347a03 100644 --- a/docs/history-source-plugins.md +++ b/docs/history-source-plugins.md @@ -141,7 +141,9 @@ JSON string. ctx stores it under a stable custom stream derived from: - `provider_key` - `source_id` - `source_format` -- local machine id + +The local machine id is stored separately with the cursor so multiple machines +can import the same custom source without overwriting each other's progress. On the next import, ctx passes the stored `cursor.after.cursor` value back in the runtime environment. This keeps native cursor design inside the provider diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index 0e7b6cb91..194fe3817 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -113,8 +113,8 @@ ensure_darwin_cross_tools() { } version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; data=json.load(sys.stdin); print(next(pkg["version"] for pkg in data["packages"] if pkg["name"] == "ctx"))')" -if [[ "${version}" != "0.15.0" ]]; then - echo "error: ctx package version must be 0.15.0 for this release, got ${version}" >&2 +if [[ "${version}" != "0.16.0" ]]; then + echo "error: ctx package version must be 0.16.0 for this release, got ${version}" >&2 exit 1 fi @@ -156,12 +156,12 @@ fi case "${platform}" in linux-x64) "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.15.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.16.0" "${staged}.version" >/dev/null ;; macos-arm64) if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.15.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.16.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi @@ -169,7 +169,7 @@ case "${platform}" in macos-x64) if [[ "$(uname -s)" == "Darwin" ]] && /usr/bin/arch -x86_64 /usr/bin/true >/dev/null 2>&1; then /usr/bin/arch -x86_64 "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.15.0" "${staged}.version" >/dev/null + grep -Fx "ctx 0.16.0" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi From b1084572cd4a8784edf1837437f74de5e2f04563 Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 18:58:47 -0500 Subject: [PATCH 35/72] Tighten search intent contract Co-authored-by: luca-ctx <216224554+luca-ctx@users.noreply.github.com> --- crates/ctx-cli/src/main.rs | 75 ++++++++++++++++- crates/ctx-cli/src/mcp.rs | 21 +++-- crates/ctx-cli/tests/cli.rs | 120 ++++++++++++++++++++++++++- crates/ctx-history-search/src/lib.rs | 118 +++++++++++++++++++++++--- crates/ctx-history-store/src/lib.rs | 21 ++++- docs/cli-reference.md | 7 +- docs/contracts/json.md | 2 +- docs/limitations.md | 3 +- docs/search.md | 5 ++ sdks/dotnet/README.md | 2 +- sdks/python/README.md | 2 +- sdks/swift/README.md | 2 +- sdks/typescript/README.md | 3 +- 13 files changed, 347 insertions(+), 34 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 5f1815530..716bcbd0a 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -459,6 +459,51 @@ impl From<&SearchArgs> for SourceIdentityFilterArgs { } } +pub(crate) struct SearchIntentInput<'a> { + query: Option<&'a str>, + terms: &'a [String], + file: Option<&'a Path>, +} + +pub(crate) fn search_has_intent(input: SearchIntentInput<'_>) -> bool { + input.query.is_some_and(has_search_token) + || input.terms.iter().any(|term| has_search_token(term)) + || input + .file + .and_then(|path| path.to_str()) + .is_some_and(|file| !file.trim().is_empty()) +} + +fn has_search_token(value: &str) -> bool { + value.split_whitespace().any(|term| { + term.trim_matches(|ch: char| !ch.is_alphanumeric() && ch != '_' && ch != '-') + .chars() + .any(char::is_alphanumeric) + }) +} + +pub(crate) fn missing_search_intent_error() -> anyhow::Error { + anyhow!( + "search needs a query, --term, or --file\n\nTry:\n ctx search \"failed migration\"\n ctx search --term \"failed migration\" --term rollback\n ctx search --file crates/foo/src/lib.rs" + ) +} + +fn search_no_results_target(query: &str, terms: &[String]) -> String { + if !query.trim().is_empty() { + return shell_quote_arg(query); + } + let rendered_terms = terms + .iter() + .filter(|term| !term.trim().is_empty()) + .map(|term| format!("--term {}", shell_quote_arg(term))) + .collect::>(); + if rendered_terms.is_empty() { + "search".to_owned() + } else { + rendered_terms.join(" ") + } +} + impl CommandRoot { fn name(&self) -> &'static str { match self { @@ -4275,6 +4320,14 @@ fn run_search( data_root: PathBuf, analytics_properties: &mut AnalyticsProperties, ) -> Result<()> { + if !search_has_intent(SearchIntentInput { + query: args.query.as_deref(), + terms: &args.term, + file: args.file.as_deref(), + }) { + return Err(missing_search_intent_error()); + } + let refresh_started = Instant::now(); let refresh = refresh_before_search(&args, &data_root)?; analytics::insert_duration( @@ -4395,10 +4448,26 @@ fn run_search( } } if packet.results.is_empty() { - println!("no results"); - if query.trim().is_empty() && !uses_composed_terms { - println!("next: ctx search \"what changed recently\" --limit 20"); + if let Some(file) = args + .file + .as_deref() + .filter(|_| query.trim().is_empty() && !uses_composed_terms) + { + println!("no indexed events touched {}", file.display()); + let indexed_items = indexed_history_item_count(&store)?; + if indexed_items == 0 { + println!("next: ctx import --all"); + } else { + println!( + "next: ctx search {}", + shell_quote_arg(&file.display().to_string()) + ); + } } else { + println!( + "no results for {}", + search_no_results_target(&query, &args.term) + ); let indexed_items = indexed_history_item_count(&store)?; if indexed_items == 0 { println!("next: ctx import --all"); diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index 02232830b..60b807dc9 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -19,9 +19,9 @@ use uuid::Uuid; use super::{ compact_json, config::CONFIG_FILE, discovered_plugin_sources_json, discovered_sources, event_window, event_window_json, indexed_history_item_count, mark_share_safe, - raw_sql_result_json, search_filters, session_transcript_json, sources_json, OutputFormat, - ProviderArg, RefreshArg, SearchDto, SearchFilterInput, SearchRefreshReport, - SourceIdentityFilterArgs, TranscriptMode, MAX_SEARCH_LIMIT, + raw_sql_result_json, search_filters, search_has_intent, session_transcript_json, sources_json, + OutputFormat, ProviderArg, RefreshArg, SearchDto, SearchFilterInput, SearchIntentInput, + SearchRefreshReport, SourceIdentityFilterArgs, TranscriptMode, MAX_SEARCH_LIMIT, }; const MCP_PROTOCOL_VERSION: &str = "2025-11-25"; @@ -320,7 +320,6 @@ fn tool_sources(data_root: &Path) -> Result { } fn tool_search(arguments: &Value, data_root: &Path) -> Result { - let store = open_existing_store(data_root)?; let query = optional_string(arguments, "query")?.unwrap_or_default(); let limit = optional_usize(arguments, "limit")?.unwrap_or(20); if !(1..=MAX_SEARCH_LIMIT).contains(&limit) { @@ -338,6 +337,14 @@ fn tool_search(arguments: &Value, data_root: &Path) -> Result { let include_subagents = optional_bool(arguments, "include_subagents")?.unwrap_or(false); let event_type = optional_string(arguments, "event_type")?; let file = optional_string(arguments, "file")?.map(PathBuf::from); + if !search_has_intent(SearchIntentInput { + query: Some(&query), + terms: &[], + file: file.as_deref(), + }) { + return Err(anyhow!("search needs a query or file")); + } + let store = open_existing_store(data_root)?; let events = optional_bool(arguments, "events")?.unwrap_or(false) || session.is_some(); let include_current_session = optional_bool(arguments, "include_current_session")?.unwrap_or(false); @@ -505,9 +512,9 @@ fn tool_definitions() -> Vec { json!({ "name": "search", "title": "Search", - "description": "Search the existing local ctx index. This does not refresh or import provider history.", + "description": "Search the existing local ctx index by query text or touched-file path. This does not refresh or import provider history.", "inputSchema": object_schema(json!({ - "query": { "type": "string" }, + "query": { "type": "string", "description": "Non-empty text query. Required unless file is provided." }, "limit": { "type": "integer", "minimum": 1, "maximum": MAX_SEARCH_LIMIT, "default": 20 }, "provider": { "type": "string", "enum": provider_names() }, "history_source": { "type": "string", "description": "Custom history source selector as plugin/source or provider_key/source_id." }, @@ -518,7 +525,7 @@ fn tool_definitions() -> Vec { "since": { "type": "string", "description": "RFC3339 timestamp or day window such as 30d." }, "include_subagents": { "type": "boolean", "default": false, "description": "Include subagent sessions in addition to primary-agent sessions." }, "event_type": { "type": "string", "enum": event_type_names() }, - "file": { "type": "string" }, + "file": { "type": "string", "description": "Indexed touched-file path. Required unless query is provided." }, "session": { "type": "string", "description": "ctx session id." }, "events": { "type": "boolean", "default": false }, "include_current_session": { "type": "boolean", "default": false, "description": "Include the active Codex session tree when CODEX_THREAD_ID is set." } diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index f65422ad5..b365f1096 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -3717,6 +3717,49 @@ fn mcp_search_and_show_tools_return_structured_json_without_refresh() { assert!(!event["events"].as_array().unwrap().is_empty()); } +#[test] +fn mcp_search_requires_query_term_or_file_without_opening_store() { + let temp = tempdir(); + let responses = mcp_roundtrip( + &temp, + &[ + json!({ + "jsonrpc": "2.0", + "id": "init", + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { "name": "ctx-test", "version": "0" } + } + }), + json!({ + "jsonrpc": "2.0", + "id": "search", + "method": "tools/call", + "params": { + "name": "search", + "arguments": { + "provider": "codex", + "limit": 5 + } + } + }), + ], + ); + + let result = &responses[1]["result"]; + assert_eq!(result["isError"], true); + assert!(result["structuredContent"]["error"] + .as_str() + .unwrap() + .contains("search needs a query or file")); + assert!( + !temp.path().join("work.sqlite").exists(), + "invalid MCP search should fail before opening the ctx store" + ); +} + #[test] fn mcp_sources_and_search_support_history_source_plugins() { let temp = tempdir(); @@ -5937,7 +5980,7 @@ fn human_search_reports_no_results() { .stdout .clone(); let fresh = String::from_utf8(fresh).unwrap(); - assert!(fresh.contains("no results")); + assert!(fresh.contains("no results for definitely-no-results-here")); assert!(fresh.contains("next: ctx import --all")); let fixture = provider_history_fixture("codex-sessions"); @@ -5961,8 +6004,81 @@ fn human_search_reports_no_results() { .stdout .clone(); let indexed = String::from_utf8(indexed).unwrap(); - assert!(indexed.contains("no results")); + assert!(indexed.contains("no results for definitely-no-results-here")); assert!(indexed.contains("next: try broader terms with ctx search --term")); + + let term_only = ctx(&temp) + .args(["search", "--term", "term-only-no-results"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let term_only = String::from_utf8(term_only).unwrap(); + assert!(term_only.contains("no results for --term term-only-no-results")); +} + +#[test] +fn search_requires_query_term_or_file_before_refreshing() { + let temp = tempdir(); + let stderr = failure_stderr(ctx(&temp).args(["search", "--provider", "codex"])); + assert!( + stderr.contains("search needs a query, --term, or --file"), + "{stderr}" + ); + assert!( + stderr.contains("ctx search \"failed migration\""), + "{stderr}" + ); + assert!( + !temp.path().join("work.sqlite").exists(), + "invalid search should fail before creating the ctx store" + ); + + let punctuation = failure_stderr(ctx(&temp).args(["search", "!!!"])); + assert!( + punctuation.contains("search needs a query, --term, or --file"), + "{punctuation}" + ); + let hyphen_only = failure_stderr(ctx(&temp).args(["search", "--", "---"])); + assert!( + hyphen_only.contains("search needs a query, --term, or --file"), + "{hyphen_only}" + ); + let underscore_term = failure_stderr(ctx(&temp).args(["search", "--term", "___"])); + assert!( + underscore_term.contains("search needs a query, --term, or --file"), + "{underscore_term}" + ); +} + +#[test] +fn file_only_search_returns_touched_file_matches() { + let temp = tempdir(); + let fixture = provider_history_fixture("codex-rich-sessions"); + json_output(ctx(&temp).args([ + "import", + "--provider", + "codex", + "--path", + &fixture, + "--json", + ])); + + let search = json_output(ctx(&temp).args(["search", "--file", "src/main.rs", "--json"])); + assert_eq!(search["query"], ""); + let results = search["results"].as_array().unwrap(); + assert_eq!(results.len(), 1); + assert!(results[0]["why_matched"] + .as_array() + .unwrap() + .iter() + .any(|reason| reason == "file_touched")); + assert!(results[0]["citations"] + .as_array() + .unwrap() + .iter() + .any(|citation| citation["item_type"] == "file" && citation["label"] == "file touched")); } #[test] diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index cb71fdcc2..896ebb6fb 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -1055,6 +1055,13 @@ fn ranked_candidates( let mut candidates = Vec::new(); let mut seen = BTreeSet::::new(); let mut scan_budget_exhausted = false; + let file_only = terms.is_empty() && file_scope.is_some(); + if terms.is_empty() && !file_only { + return Ok(CandidateSearch { + candidates, + scan_budget_exhausted, + }); + } if filtered { let page_size = FILTERED_SEARCH_PAGE_SIZE.max(target_candidates); @@ -1062,11 +1069,15 @@ fn ranked_candidates( let mut pages_scanned = 0_usize; loop { pages_scanned = pages_scanned.saturating_add(1); - let records = match query { - Some(query) if !query.trim().is_empty() => { - store.search_records_page(query, page_size, offset)? + let records = if file_only { + store.list_records_page(page_size, offset)? + } else { + match query { + Some(query) if !query.trim().is_empty() => { + store.search_records_page(query, page_size, offset)? + } + _ => Vec::new(), } - _ => store.list_records_page(page_size, offset)?, }; let page_len = records.len(); @@ -1103,9 +1114,15 @@ fn ranked_candidates( } } else { let fetch_limit = target_candidates; - let records = match query { - Some(query) if !query.trim().is_empty() => store.search_records(query, fetch_limit)?, - _ => store.list_records(fetch_limit)?, + let records = if file_only { + store.list_records(fetch_limit)? + } else { + match query { + Some(query) if !query.trim().is_empty() => { + store.search_records(query, fetch_limit)? + } + _ => Vec::new(), + } }; for record in records { if !seen.insert(record.id) { @@ -1256,6 +1273,37 @@ fn analyze_record( let mut citations = Vec::new(); if terms.is_empty() { + if filters + .file + .as_ref() + .is_some_and(|file| !file.trim().is_empty()) + { + let mut primary_hit = None; + for section in search_sections(record, context, filters) + .into_iter() + .filter(|section| section.reason == "file_touched") + { + if primary_hit.is_none() { + primary_hit = Some(section.hit.clone()); + } + score += section.weight; + add_match( + &mut why, + &mut citations, + section.reason, + section.citation, + §ion.hit, + ); + } + if !why.is_empty() { + return MatchAnalysis { + score, + why_matched: why, + citations, + primary_hit, + }; + } + } add_match( &mut why, &mut citations, @@ -2209,7 +2257,7 @@ fn query_terms(query: &str) -> Vec { .split(|ch: char| !ch.is_alphanumeric() && ch != '_' && ch != '-') .filter_map(|term| { let term = term.trim().to_lowercase(); - if term.is_empty() { + if term.is_empty() || !term.chars().any(char::is_alphanumeric) { None } else { Some(term) @@ -3521,6 +3569,33 @@ mod tests { && citation.cursor.as_deref() == Some("line:8") })); + let file_only = search_packet( + &store, + "", + &PacketOptions { + limit: 10, + filters: SearchFilters { + provider: Some(CaptureProvider::Codex), + file: Some("source_filter.rs".into()), + ..SearchFilters::default() + }, + ..PacketOptions::default() + }, + ) + .unwrap(); + assert_eq!(file_only.results.len(), 1); + assert!(file_only.results[0] + .why_matched + .iter() + .any(|reason| reason == "file_touched")); + assert!(!file_only.results[0] + .why_matched + .iter() + .any(|reason| reason == "recent_activity")); + assert!(file_only.results[0].citations.iter().any(|citation| { + citation.citation_type == ContextCitationType::File && citation.id == file.id + })); + let wrong_provider = search_packet( &store, "source-filter-needle", @@ -4164,7 +4239,7 @@ mod tests { } #[test] - fn empty_query_filtered_search_stops_at_scan_budget() { + fn empty_query_filtered_search_returns_empty_without_scanning() { let (_temp, store) = test_store(); let mut records = Vec::new(); for index in 0..=(FILTERED_SEARCH_PAGE_SIZE * FILTERED_SEARCH_MAX_PAGES) { @@ -4197,8 +4272,29 @@ mod tests { .unwrap(); assert!(packet.results.is_empty()); - assert!(packet.truncation.truncated); - assert_eq!(packet.truncation.reason.as_deref(), Some("scan_budget")); + assert!(!packet.truncation.truncated); + assert_eq!(packet.truncation.reason.as_deref(), None); + } + + #[test] + fn no_token_query_returns_empty_without_recent_activity() { + let (_temp, store) = test_store(); + let record = HistoryRecord::new( + "No-token query decoy", + "This record should not be returned for punctuation-only search.", + Vec::new(), + "task", + Some("/workspace/punctuation".into()), + ); + store.upsert_record(&record).unwrap(); + + for query in ["!!!", "---", "___"] { + let packet = + search_packet(&store, query, &PacketOptions::default()).expect("search packet"); + + assert!(packet.results.is_empty(), "{query}"); + assert!(!packet.truncation.truncated, "{query}"); + } } #[test] diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 311d26980..4e649327e 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -3534,6 +3534,9 @@ impl Store { limit: usize, offset: usize, ) -> Result> { + if fts_match_query(query).is_none() { + return Ok(Vec::new()); + } if let Some(records) = self.search_records_fts(query, limit, offset)? { return Ok(records); } @@ -3558,7 +3561,7 @@ impl Store { return Ok(None); } let Some(match_query) = fts_match_query(query) else { - return Ok(Some(self.list_records_page(limit, offset)?)); + return Ok(Some(Vec::new())); }; let has_event_search = table_exists(&self.conn, "event_search")?; let has_artifact_search = table_exists(&self.conn, "artifact_search")?; @@ -5396,7 +5399,7 @@ fn fts_match_query(query: &str) -> Option { let terms = query .split_whitespace() .map(|term| term.trim_matches(|ch: char| !ch.is_alphanumeric() && ch != '_' && ch != '-')) - .filter(|term| !term.is_empty()) + .filter(|term| term.chars().any(char::is_alphanumeric)) .map(|term| format!("\"{}\"", term.replace('"', "\"\""))) .collect::>(); if terms.is_empty() { @@ -7640,6 +7643,20 @@ mod search_order_tests { assert_search_order(&reopened, &expected); } + #[test] + fn search_records_empty_or_no_token_query_returns_empty() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let record = stable_tie_record(1); + store.insert_record(&record).unwrap(); + + assert!(store.search_records("", 10).unwrap().is_empty()); + assert!(store.search_records("!!!", 10).unwrap().is_empty()); + assert!(store.search_records("---", 10).unwrap().is_empty()); + assert!(store.search_records("___", 10).unwrap().is_empty()); + assert!(store.search_records_page("", 10, 0).unwrap().is_empty()); + } + #[test] fn upsert_record_updates_record_search_without_rebuilding_event_search() { let temp = tempdir(); diff --git a/docs/cli-reference.md b/docs/cli-reference.md index c8c23d98a..a0e30958d 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -213,8 +213,9 @@ querying. Use `--refresh off` to search the existing index without refreshing, o `--refresh strict` to fail when the pre-search refresh cannot run or import successfully. Preview native sources such as NanoClaw and AstrBot are searched from the existing index until they are explicitly imported through a supported -path. The query argument is optional so file or metadata filters can drive a -search. Default results are session-diverse: ctx +path. Search requires a non-empty query, at least one non-empty `--term`, or +`--file `; provider, workspace, time, session, event, source, and result +flags only narrow an actual search. Default results are session-diverse: ctx returns the strongest matching span from each session, plus `more_matches_in_session` and `session_importance` when more indexed events from that session also matched. Use `--session ` after a default @@ -433,7 +434,7 @@ ctx show session --format json ctx show event --format json ctx locate session --format json ctx locate event --format json -ctx search [query] --json +ctx search |--term |--file --json ctx sql "SELECT COUNT(*) FROM ctx_sessions" --json ctx docs list --json ctx docs search --json diff --git a/docs/contracts/json.md b/docs/contracts/json.md index 8f5d929a0..ccb55203e 100644 --- a/docs/contracts/json.md +++ b/docs/contracts/json.md @@ -199,7 +199,7 @@ artifact. JSON and JSONL artifact rows use the same ctx-owned ID fields as ## Search ```bash -ctx search [query] --json +ctx search |--term |--file --json ``` Returns: diff --git a/docs/limitations.md b/docs/limitations.md index e7726f192..3d2796e5a 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -33,7 +33,8 @@ shipped. - Large outputs may be represented as bounded previews. - Ranking is deterministic for the same local database and options, but it is not a claim of semantic understanding. -- Empty or very broad queries can return metadata-driven matches. +- Empty or punctuation-only search is invalid. Broad valid queries can still + return metadata-driven matches. ## Retrieval Semantics diff --git a/docs/search.md b/docs/search.md index 4e9754826..685ac062a 100644 --- a/docs/search.md +++ b/docs/search.md @@ -90,6 +90,11 @@ for a file, or combine it with query terms to find sessions that both mention a topic and touched that path. It searches paths recorded during import; it does not inspect the current filesystem. +Search requires a non-empty query, at least one non-empty `--term`, or +`--file `. Provider, workspace, time, session, event, source, and result +flags only narrow an actual search; by themselves they do not browse recent +history. + The default searches primary-agent sessions so human intent and decisions stay prominent. Use `--include-subagents` when you want implementation details, code review notes, test output, or failure analysis from subagent sessions too. diff --git a/sdks/dotnet/README.md b/sdks/dotnet/README.md index d74a9c025..ea92ff6e3 100644 --- a/sdks/dotnet/README.md +++ b/sdks/dotnet/README.md @@ -55,7 +55,7 @@ Console.WriteLine(results.ToJsonObject().ToJsonString()); - `SourcesAsync()` - `ImportHistoryAsync(ImportOptions?)` - `SyncAsync(ImportOptions?)` -- `SearchAsync(SearchOptions?)` +- `SearchAsync(SearchOptions)` with a query, term, or file option - `ShowEventAsync(string, ShowEventOptions?)` - `ShowSessionAsync(string, ShowSessionOptions?)` - `ShowSessionAsync(ShowSessionOptions)` diff --git a/sdks/python/README.md b/sdks/python/README.md index 3e106844f..94441ae06 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -39,7 +39,7 @@ The public methods mirror the agent-history-v1 client surface: - `init()` for `ctx setup --json` - `sources()` - `import_()` and `sync()` (`import` is a reserved Python keyword) -- `search()` +- `search()` with a query, term, or file option - `show_event()` / `showEvent()` - `show_session()` / `showSession()` - `locate_event()` / `locateEvent()` diff --git a/sdks/swift/README.md b/sdks/swift/README.md index af40ae4fe..40ff51203 100644 --- a/sdks/swift/README.md +++ b/sdks/swift/README.md @@ -45,7 +45,7 @@ The public client mirrors the `agent-history-v1` operations: - `sources()` - `importHistory()` - `sync()` -- `search()` +- `search()` with a query, term, or file option - `showEvent()` - `showSession()` - `locateEvent()` diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index e6233f579..de205a15f 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -21,7 +21,8 @@ const results = await client.search("sqlite storage", { refresh: "off" }); - `sources()` wraps `ctx sources --json`. - `import(options)` wraps `ctx import --json`. - `sync(options)` is an alias for `import(options)`. -- `search(query, options)` and `search(options)` wrap `ctx search --json`. +- `search(query, options)` and file/term-based `search(options)` wrap + `ctx search --json`. - `showEvent(id, { before, after, window })` wraps `ctx show event --format json`. - `showSession(id, { mode })` wraps `ctx show session --format json`. - `showSession({ provider, providerSession, mode })` looks up by provider-owned session ID. From 9e2ed1fdfa61931028616741acfc5b754c29b04c Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 19:12:22 -0500 Subject: [PATCH 36/72] Harden read-only search and doctor paths Co-authored-by: luca-ctx <216224554+luca-ctx@users.noreply.github.com> --- crates/ctx-cli/src/main.rs | 67 +++++++++++++------ crates/ctx-cli/tests/cli.rs | 130 ++++++++++++++++++++++++++++++++++-- 2 files changed, 170 insertions(+), 27 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 716bcbd0a..660061eef 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -4025,27 +4025,28 @@ fn prune_null_json(value: &mut Value) { } } -fn run_sql(args: SqlArgs, data_root: PathBuf) -> Result<()> { - let sql = read_sql_input(&args)?; - let db_path = database_path(data_root); +fn open_existing_store_read_only(db_path: &Path, command: &str) -> Result { if !db_path.exists() { return Err(anyhow!( "ctx store is not initialized at {}; run `ctx setup` or `ctx import` first", db_path.display() )); } - let store = match Store::open_read_only(&db_path) { - Ok(store) => store, - Err(StoreError::UnsupportedSchemaVersion(version)) => { - return Err(anyhow!( - "ctx store schema version {version} is not supported by this ctx binary; run `ctx status` once to migrate before using `ctx sql`" - )); - } + match Store::open_read_only(db_path) { + Ok(store) => Ok(store), + Err(StoreError::UnsupportedSchemaVersion(version)) => Err(anyhow!( + "ctx store schema version {version} is not supported by this ctx binary; run `ctx status` once to migrate before using `{command}`" + )), Err(err) => { - return Err(err) - .with_context(|| format!("open read-only ctx store {}", db_path.display())); + Err(err).with_context(|| format!("open read-only ctx store {}", db_path.display())) } - }; + } +} + +fn run_sql(args: SqlArgs, data_root: PathBuf) -> Result<()> { + let sql = read_sql_input(&args)?; + let db_path = database_path(data_root); + let store = open_existing_store_read_only(&db_path, "ctx sql")?; let result = store.raw_sql_query( &sql, RawSqlOptions { @@ -4328,6 +4329,8 @@ fn run_search( return Err(missing_search_intent_error()); } + let db_path = database_path(data_root.clone()); + let had_existing_store = db_path.exists(); let refresh_started = Instant::now(); let refresh = refresh_before_search(&args, &data_root)?; analytics::insert_duration( @@ -4350,9 +4353,22 @@ fn run_search( "search_refresh_source_count_bucket", refresh.source_count as u64, ); - let db_path = database_path(data_root); insert_db_size_bucket(analytics_properties, &db_path); - let store = Store::open(&db_path)?; + if refresh.status == "failed" && args.refresh == RefreshArg::Auto && !had_existing_store { + return Err(anyhow!( + "search refresh failed and no existing ctx index is available; run `ctx import` first or retry with `--refresh strict`: {}", + refresh.error.as_deref().unwrap_or("unknown refresh error") + )); + } + let store = if args.refresh == RefreshArg::Off + || refresh.status == "failed" + || refresh.status == "completed" + || had_existing_store + { + open_existing_store_read_only(&db_path, "ctx search")? + } else { + Store::open(&db_path)? + }; insert_store_analytics_counts(analytics_properties, &store)?; let source_identity = SourceIdentityFilterArgs::from(&args); let query = args.query.unwrap_or_default(); @@ -4835,15 +4851,24 @@ fn run_doctor( ) -> Result<()> { let progress = ProgressReporter::new(args.progress, args.json, "doctor", 0); progress.message("opening", "opening ctx store"); - let store = Store::open(database_path(data_root.clone()))?; - progress.message( - "checking", - "running sqlite integrity and foreign key checks", - ); - let mut findings = store.validate()?; + let db_path = database_path(data_root.clone()); + let mut findings = Vec::new(); if !data_root.exists() { findings.push(format!("data root does not exist: {}", data_root.display())); } + if !db_path.exists() { + findings.push(format!( + "ctx store is not initialized at {}; run `ctx setup` or `ctx import` first", + db_path.display() + )); + } else { + let store = open_existing_store_read_only(&db_path, "ctx doctor")?; + progress.message( + "checking", + "running sqlite integrity and foreign key checks", + ); + findings.extend(store.validate()?); + } analytics::insert_count_bucket( analytics_properties, "finding_count_bucket", diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index b365f1096..cd7aa25ca 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -3463,6 +3463,30 @@ fn fresh_home_search_mvp_flow() { assert!(doctor_progress.contains(r#""phase":"checking""#)); } +#[test] +fn doctor_reports_missing_store_without_creating_it() { + let temp = tempdir(); + + let doctor = json_output(ctx(&temp).args(["doctor", "--json"])); + + assert_eq!(doctor["schema_version"], 1); + assert_eq!(doctor["ok"], false); + assert!(doctor["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| { + finding + .as_str() + .unwrap() + .contains("ctx store is not initialized") + })); + assert!( + !temp.path().join("work.sqlite").exists(), + "doctor should not create the ctx store" + ); +} + #[test] fn mcp_status_and_tools_list_are_read_only_without_initialized_store() { let temp = tempdir(); @@ -4077,13 +4101,22 @@ fn search_refreshes_discovered_codex_sessions_before_query() { #[test] fn search_refresh_off_serves_existing_index_without_importing() { let temp = tempdir(); - let fixture = PathBuf::from(provider_history_fixture("codex-sessions")); + let indexed_fixture = provider_history_fixture("codex-sessions"); + json_output(ctx(&temp).args([ + "import", + "--provider", + "codex", + "--path", + &indexed_fixture, + "--json", + ])); + let discovered_fixture = provider_history_fixture("codex-rich-sessions"); let discovered = temp.path().join(".codex").join("sessions"); - copy_dir_all(&fixture, &discovered); + copy_dir_all(&PathBuf::from(discovered_fixture), &discovered); let stale = json_output(ctx(&temp).args([ "search", - "onboarding", + "redacted sample app", "--provider", "codex", "--refresh", @@ -4095,8 +4128,8 @@ fn search_refresh_off_serves_existing_index_without_importing() { assert!(stale["results"].as_array().unwrap().is_empty()); let status = json_output(ctx(&temp).args(["status", "--json"])); - assert_eq!(status["cataloged_sessions"], 0); - assert_eq!(status["indexed_catalog_sessions"], 0); + assert_eq!(status["cataloged_sessions"], 2); + assert_eq!(status["indexed_catalog_sessions"], 2); let fresh = json_output(ctx(&temp).args(["search", "onboarding", "--provider", "codex", "--json"])); @@ -4263,6 +4296,7 @@ fn search_refresh_provider_filter_does_not_execute_history_source_plugins() { #[test] fn search_refresh_off_does_not_execute_history_source_plugins() { let temp = tempdir(); + json_output(ctx(&temp).args(["setup", "--json"])); let plugin = write_history_source_plugin_with_refresh(&temp, "hermes", true, Some("auto"), None); @@ -4363,6 +4397,78 @@ sys.exit(23) assert!(stderr.contains("plugin exploded"), "{stderr}"); } +#[test] +fn search_refresh_auto_failure_without_prior_store_fails_instead_of_serving_empty_index() { + let temp = tempdir(); + let script = r#"#!/usr/bin/env python3 +import sys +print("plugin exploded", file=sys.stderr) +sys.exit(23) +"#; + let plugin = write_raw_history_source_plugin_with_options( + &temp, + "badplugin", + script, + true, + Some("auto"), + ); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["search", "anything", "--provider", "custom", "--json"]), + ); + + assert!( + stderr.contains("search refresh failed and no existing ctx index is available"), + "{stderr}" + ); + assert!( + stderr.contains("history source plugin badplugin/default failed"), + "{stderr}" + ); + assert!(stderr.contains("plugin exploded"), "{stderr}"); +} + +#[test] +fn search_refresh_auto_failure_serves_prior_index() { + let temp = tempdir(); + let fixture = provider_history_fixture("codex-sessions"); + let script = r#"#!/usr/bin/env python3 +import sys +print("plugin exploded", file=sys.stderr) +sys.exit(23) +"#; + let plugin = write_raw_history_source_plugin_with_options( + &temp, + "badplugin", + script, + true, + Some("auto"), + ); + json_output(ctx(&temp).args([ + "import", + "--provider", + "codex", + "--path", + &fixture, + "--json", + ])); + + let search = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args(["search", "onboarding", "--json"]), + ); + + assert_eq!(search["freshness"]["status"], "failed"); + assert!(search["freshness"]["error"] + .as_str() + .unwrap() + .contains("history source plugin badplugin/default failed")); + assert!(!search["results"].as_array().unwrap().is_empty()); +} + #[test] fn search_refresh_strict_times_out_when_plugin_helper_keeps_stdout_open() { let temp = tempdir(); @@ -4423,9 +4529,9 @@ subprocess.Popen(["sh", "-c", "sleep 5"]) fn search_refresh_auto_imports_fresh_work_despite_large_existing_catalog() { let temp = tempdir(); let fixture = PathBuf::from(provider_history_fixture("codex-sessions")); + let _ = json_output(ctx(&temp).args(["setup", "--json"])); let discovered = temp.path().join(".codex").join("sessions"); copy_dir_all(&fixture, &discovered); - let _ = json_output(ctx(&temp).args(["search", "anything", "--refresh", "off", "--json"])); let mut conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); let tx = conn.transaction().unwrap(); @@ -6052,6 +6158,18 @@ fn search_requires_query_term_or_file_before_refreshing() { ); } +#[test] +fn search_refresh_off_requires_existing_store_without_creating_one() { + let temp = tempdir(); + let stderr = failure_stderr(ctx(&temp).args(["search", "anything", "--refresh", "off"])); + + assert!(stderr.contains("ctx store is not initialized"), "{stderr}"); + assert!( + !temp.path().join("work.sqlite").exists(), + "refresh-off search should not create the ctx store" + ); +} + #[test] fn file_only_search_returns_touched_file_matches() { let temp = tempdir(); From 66497887fc5f8a09c2f1b037a57c3ed55d42aefd Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 19:28:09 -0500 Subject: [PATCH 37/72] Require providers for native path imports Co-authored-by: luca-ctx <216224554+luca-ctx@users.noreply.github.com> --- crates/ctx-cli/src/main.rs | 17 +- crates/ctx-cli/tests/cli.rs | 18 +- .../src/provider_sources.rs | 278 ++++++++++++++---- docs/cli-reference.md | 6 +- docs/first-10-minutes.md | 3 +- docs/getting-started.md | 5 +- docs/storage.md | 2 +- docs/troubleshooting.md | 2 +- sdks/python/tests/test_client.py | 5 +- 9 files changed, 269 insertions(+), 67 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 660061eef..833acbfcc 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -133,7 +133,10 @@ struct DoctorArgs { struct ImportArgs { #[arg(long, value_enum)] provider: Option, - #[arg(long)] + #[arg( + long, + help = "Import exactly this path; native provider paths require --provider" + )] path: Option, #[arg(long = "history-source", conflicts_with_all = ["provider", "path", "format", "all"])] history_source: Option, @@ -2030,6 +2033,7 @@ fn run_import_internal( analytics_properties: &mut AnalyticsProperties, options: ImportRunOptions, ) -> Result { + validate_import_args(args)?; fs::create_dir_all(&data_root)?; config::write_default_config(&data_root)?; let db_path = database_path(data_root.clone()); @@ -4900,6 +4904,15 @@ fn run_doctor( Ok(()) } +fn validate_import_args(args: &ImportArgs) -> Result<()> { + if args.path.is_some() && args.format.is_none() && args.provider.is_none() { + return Err(anyhow!( + "ctx import --path requires --provider for native provider history; use `ctx import --provider codex --path ` or `ctx import --format ctx-history-jsonl-v1 --path `" + )); + } + Ok(()) +} + fn import_requests(args: &ImportArgs) -> Result> { if args.history_source.is_some() || !args.history_source_manifest.is_empty() { return Ok(Vec::new()); @@ -4907,7 +4920,7 @@ fn import_requests(args: &ImportArgs) -> Result> { if let Some(path) = &args.path { let provider = args .provider - .unwrap_or(NativeProviderArg::Codex) + .context("ctx import --path requires --provider for native provider history")? .capture_provider(); let source = explicit_path_source(provider, path.clone()); if !source diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index cd7aa25ca..52d239ee6 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -6258,15 +6258,25 @@ fn import_rejects_nonexistent_path() { predicate::str::contains("import path does not exist") .and(predicate::str::contains(path)), ); +} + +#[test] +fn import_path_requires_provider_before_opening_store() { + let temp = tempdir(); + let path = temp.path().join("missing-codex-history"); + let path = path.to_str().unwrap(); ctx(&temp) .args(["import", "--path", path]) .assert() .failure() - .stderr( - predicate::str::contains("import path does not exist") - .and(predicate::str::contains(path)), - ); + .stderr(predicate::str::contains( + "ctx import --path requires --provider", + )); + assert!( + !temp.path().join("work.sqlite").exists(), + "native path import without provider should fail before opening the store" + ); } #[cfg(unix)] diff --git a/crates/ctx-history-capture/src/provider_sources.rs b/crates/ctx-history-capture/src/provider_sources.rs index 6834e29b0..18fa82ea0 100644 --- a/crates/ctx-history-capture/src/provider_sources.rs +++ b/crates/ctx-history-capture/src/provider_sources.rs @@ -1,6 +1,7 @@ use std::{ collections::HashSet, env, + io::ErrorKind, path::{Path, PathBuf}, }; @@ -536,23 +537,37 @@ fn provider_source_from_location( location: &ProviderDefaultLocation, path: PathBuf, ) -> ProviderSource { - let exists = path.exists(); - let status = if matches!(spec.import_support, ProviderImportSupport::Unsupported) { - ProviderSourceStatus::Unsupported - } else if !exists { - ProviderSourceStatus::Missing - } else { - match default_location_import_probe(spec.provider, location, &path) { - BoundedProbe::Found => ProviderSourceStatus::Available, - BoundedProbe::NotFound => ProviderSourceStatus::Empty, - BoundedProbe::BudgetExhausted => ProviderSourceStatus::Unknown, - } - }; - let unsupported_reason = match status { - ProviderSourceStatus::Empty => empty_source_reason(spec.provider), - ProviderSourceStatus::Unknown => unknown_source_reason(spec.provider), - _ => spec.unsupported_reason, - }; + let path_exists = path.try_exists(); + let exists = path_exists.as_ref().copied().unwrap_or(true); + let (status, unsupported_reason) = + if matches!(spec.import_support, ProviderImportSupport::Unsupported) { + (ProviderSourceStatus::Unsupported, spec.unsupported_reason) + } else { + match path_exists { + Ok(false) => (ProviderSourceStatus::Missing, spec.unsupported_reason), + Err(_) => ( + ProviderSourceStatus::Unknown, + probe_io_error_reason(spec.provider), + ), + Ok(true) => match default_location_import_probe(spec.provider, location, &path) { + BoundedProbe::Found => { + (ProviderSourceStatus::Available, spec.unsupported_reason) + } + BoundedProbe::NotFound => ( + ProviderSourceStatus::Empty, + empty_source_reason(spec.provider), + ), + BoundedProbe::BudgetExhausted => ( + ProviderSourceStatus::Unknown, + unknown_source_reason(spec.provider), + ), + BoundedProbe::IoError => ( + ProviderSourceStatus::Unknown, + probe_io_error_reason(spec.provider), + ), + }, + } + }; ProviderSource { provider: spec.provider, path, @@ -633,6 +648,51 @@ fn unknown_source_reason(provider: CaptureProvider) -> Option<&'static str> { } } +fn probe_io_error_reason(provider: CaptureProvider) -> Option<&'static str> { + match provider { + CaptureProvider::Codex => { + Some("path exists but Codex session transcripts could not be read; check permissions") + } + CaptureProvider::Pi => { + Some("path exists but the Pi session file could not be read; check permissions") + } + CaptureProvider::Claude => { + Some("path exists but Claude project transcripts could not be read; check permissions") + } + CaptureProvider::OpenCode => { + Some("path exists but the OpenCode database could not be read; check permissions") + } + CaptureProvider::Antigravity => { + Some("path exists but Antigravity transcripts could not be read; check permissions") + } + CaptureProvider::Gemini => { + Some("path exists but Gemini CLI chat transcripts could not be read; check permissions") + } + CaptureProvider::Cursor => { + Some("path exists but Cursor agent transcripts could not be read; check permissions") + } + CaptureProvider::CopilotCli => { + Some("path exists but Copilot CLI session events could not be read; check permissions") + } + CaptureProvider::FactoryAiDroid => { + Some("path exists but Factory AI Droid sessions could not be read; check permissions") + } + CaptureProvider::OpenClaw => Some( + "path exists but OpenClaw session transcripts could not be read; check permissions", + ), + CaptureProvider::Hermes => { + Some("path exists but the Hermes state database could not be read; check permissions") + } + CaptureProvider::NanoClaw => { + Some("path exists but the NanoClaw project store could not be read; check permissions") + } + CaptureProvider::AstrBot => { + Some("path exists but the AstrBot data database could not be read; check permissions") + } + _ => None, + } +} + fn default_location_import_probe( provider: CaptureProvider, location: &ProviderDefaultLocation, @@ -640,16 +700,16 @@ fn default_location_import_probe( ) -> BoundedProbe { match provider { CaptureProvider::Codex if location.source_format == "codex_history_jsonl" => { - BoundedProbe::from_bool(path.is_file()) + path_is_file_probe(path) } CaptureProvider::Codex => has_jsonl_file_under_matching(path, 10_000, |_| true), - CaptureProvider::Pi => BoundedProbe::from_bool(path.is_file()), - CaptureProvider::OpenCode => BoundedProbe::from_bool(path.is_file()), + CaptureProvider::Pi => path_is_file_probe(path), + CaptureProvider::OpenCode => path_is_file_probe(path), CaptureProvider::Claude => has_jsonl_file_under_matching(path, 10_000, |_| true), CaptureProvider::OpenClaw => has_openclaw_session_jsonl(path, 10_000), - CaptureProvider::Hermes => BoundedProbe::from_bool(path.is_file()), + CaptureProvider::Hermes => path_is_file_probe(path), CaptureProvider::NanoClaw => has_nanoclaw_project(path), - CaptureProvider::AstrBot => BoundedProbe::from_bool(path.is_file()), + CaptureProvider::AstrBot => path_is_file_probe(path), CaptureProvider::Antigravity => has_jsonl_file_under_matching(path, 10_000, |candidate| { matches!( candidate.file_name().and_then(|name| name.to_str()), @@ -664,29 +724,45 @@ fn default_location_import_probe( candidate.file_name().and_then(|name| name.to_str()) == Some("events.jsonl") }), CaptureProvider::FactoryAiDroid => has_jsonl_file_under_matching(path, 10_000, |_| true), - _ => BoundedProbe::from_bool(path.exists()), + CaptureProvider::Shell + | CaptureProvider::Git + | CaptureProvider::Jj + | CaptureProvider::Gh + | CaptureProvider::Custom + | CaptureProvider::Unknown => BoundedProbe::NotFound, } } fn has_gemini_chat_jsonl(root: &Path, max_entries: usize) -> BoundedProbe { let tmp = root.join("tmp"); - if !tmp.is_dir() { - return BoundedProbe::NotFound; + match path_is_dir_probe(&tmp) { + BoundedProbe::Found => {} + BoundedProbe::IoError => return BoundedProbe::IoError, + _ => return BoundedProbe::NotFound, } has_jsonl_file_under_matching(&tmp, max_entries, |path| path_has_component(path, "chats")) } fn has_openclaw_session_jsonl(root: &Path, max_entries: usize) -> BoundedProbe { - if root.is_file() { - return BoundedProbe::from_bool( - root.extension().and_then(|ext| ext.to_str()) == Some("jsonl"), - ); + match path_metadata_probe(root) { + PathProbe::File => { + return BoundedProbe::from_bool( + root.extension().and_then(|ext| ext.to_str()) == Some("jsonl"), + ); + } + PathProbe::Dir => {} + PathProbe::Missing | PathProbe::Other => return BoundedProbe::NotFound, + PathProbe::IoError => return BoundedProbe::IoError, } let agents = root.join("agents"); - if agents.is_dir() { - return has_jsonl_file_under_matching(&agents, max_entries, |path| { - path_has_component(path, "sessions") - }); + match path_is_dir_probe(&agents) { + BoundedProbe::Found => { + return has_jsonl_file_under_matching(&agents, max_entries, |path| { + path_has_component(path, "sessions") + }); + } + BoundedProbe::IoError => return BoundedProbe::IoError, + _ => {} } has_jsonl_file_under_matching(root, max_entries, |path| { path_has_component(path, "sessions") @@ -694,9 +770,14 @@ fn has_openclaw_session_jsonl(root: &Path, max_entries: usize) -> BoundedProbe { } fn has_nanoclaw_project(root: &Path) -> BoundedProbe { - BoundedProbe::from_bool( - root.join("data").join("v2.db").is_file() && root.join("data").join("v2-sessions").is_dir(), - ) + match ( + path_is_file_probe(&root.join("data").join("v2.db")), + path_is_dir_probe(&root.join("data").join("v2-sessions")), + ) { + (BoundedProbe::Found, BoundedProbe::Found) => BoundedProbe::Found, + (BoundedProbe::IoError, _) | (_, BoundedProbe::IoError) => BoundedProbe::IoError, + _ => BoundedProbe::NotFound, + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -704,6 +785,7 @@ enum BoundedProbe { Found, NotFound, BudgetExhausted, + IoError, } impl BoundedProbe { @@ -716,38 +798,81 @@ impl BoundedProbe { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PathProbe { + File, + Dir, + Other, + Missing, + IoError, +} + +fn path_metadata_probe(path: &Path) -> PathProbe { + match path.metadata() { + Ok(metadata) if metadata.is_file() => PathProbe::File, + Ok(metadata) if metadata.is_dir() => PathProbe::Dir, + Ok(_) => PathProbe::Other, + Err(err) if err.kind() == ErrorKind::NotFound => PathProbe::Missing, + Err(_) => PathProbe::IoError, + } +} + +fn path_is_file_probe(path: &Path) -> BoundedProbe { + match path_metadata_probe(path) { + PathProbe::File => BoundedProbe::Found, + PathProbe::IoError => BoundedProbe::IoError, + _ => BoundedProbe::NotFound, + } +} + +fn path_is_dir_probe(path: &Path) -> BoundedProbe { + match path_metadata_probe(path) { + PathProbe::Dir => BoundedProbe::Found, + PathProbe::IoError => BoundedProbe::IoError, + _ => BoundedProbe::NotFound, + } +} + fn has_jsonl_file_under_matching( root: &Path, max_entries: usize, matches_path: impl Fn(&Path) -> bool, ) -> BoundedProbe { - if root.is_file() { - return if root.extension().and_then(|ext| ext.to_str()) == Some("jsonl") - && matches_path(root) - { - BoundedProbe::Found - } else { - BoundedProbe::NotFound - }; - } - if !root.is_dir() { - return BoundedProbe::NotFound; + match path_metadata_probe(root) { + PathProbe::File => { + return if root.extension().and_then(|ext| ext.to_str()) == Some("jsonl") + && matches_path(root) + { + BoundedProbe::Found + } else { + BoundedProbe::NotFound + }; + } + PathProbe::Dir => {} + PathProbe::Missing | PathProbe::Other => return BoundedProbe::NotFound, + PathProbe::IoError => return BoundedProbe::IoError, } let mut visited = 0usize; let mut stack = vec![root.to_path_buf()]; while let Some(dir) = stack.pop() { - let Ok(entries) = std::fs::read_dir(&dir) else { - continue; + let entries = match std::fs::read_dir(&dir) { + Ok(entries) => entries, + Err(_) => return BoundedProbe::IoError, }; - for entry in entries.flatten() { + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(_) => return BoundedProbe::IoError, + }; visited = visited.saturating_add(1); if visited > max_entries { return BoundedProbe::BudgetExhausted; } let path = entry.path(); - let Ok(file_type) = entry.file_type() else { - continue; + let file_type = match entry.file_type() { + Ok(file_type) => file_type, + Err(_) => return BoundedProbe::IoError, }; if file_type.is_dir() { stack.push(path); @@ -945,6 +1070,55 @@ mod tests { ); } + #[test] + fn default_location_probe_does_not_fallback_to_path_existence_for_unhandled_providers() { + let temp = tempfile::tempdir().unwrap(); + let existing = temp.path().join("shell-history"); + std::fs::write(&existing, "{}\n").unwrap(); + let location = ProviderDefaultLocation { + path_components: &["shell-history"], + source_format: "shell_history", + source_kind: ProviderSourceKind::NativeHistory, + }; + + assert_eq!( + default_location_import_probe(CaptureProvider::Shell, &location, &existing), + BoundedProbe::NotFound + ); + } + + #[cfg(unix)] + #[test] + fn default_source_probe_reports_unreadable_directory_as_unknown() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let sessions = temp.path().join(".codex/sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let original_permissions = std::fs::metadata(&sessions).unwrap().permissions(); + std::fs::set_permissions(&sessions, std::fs::Permissions::from_mode(0o000)).unwrap(); + + if std::fs::read_dir(&sessions).is_ok() { + std::fs::set_permissions(&sessions, original_permissions).unwrap(); + return; + } + + let source = discover_provider_sources(temp.path()) + .into_iter() + .find(|source| { + source.provider == CaptureProvider::Codex + && source.source_format == "codex_session_jsonl_tree" + }) + .unwrap(); + std::fs::set_permissions(&sessions, original_permissions).unwrap(); + + assert_eq!(source.status, ProviderSourceStatus::Unknown); + assert!(source + .unsupported_reason + .unwrap() + .contains("could not be read")); + } + fn assert_source_status( home: &Path, provider: CaptureProvider, diff --git a/docs/cli-reference.md b/docs/cli-reference.md index a0e30958d..87c0f7b94 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -93,7 +93,7 @@ ctx import --provider gemini ctx import --provider cursor ctx import --provider copilot-cli ctx import --provider factory-ai-droid -ctx import --path ~/.codex/sessions +ctx import --provider codex --path ~/.codex/sessions ctx import --provider pi --path ~/.pi/sessions.jsonl ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl ctx import --history-source example-agent/default @@ -135,8 +135,8 @@ Import selection rules: history JSONL file; - with `--history-source`, import matching local plugin sources; - with `--history-source-manifest`, import sources from that manifest path; -- with `--path`, import exactly that path; -- with `--path` and no provider, parse the path as Codex format. +- with `--provider --path `, import exactly that native + provider path. Preview providers such as NanoClaw and AstrBot are not included in `--all` or pre-search refresh. Import them explicitly with `--provider` when discovery diff --git a/docs/first-10-minutes.md b/docs/first-10-minutes.md index ef656c3e7..cf5f97ddd 100644 --- a/docs/first-10-minutes.md +++ b/docs/first-10-minutes.md @@ -121,7 +121,8 @@ eligible for signed self-upgrades. ## Failure Paths - No sources listed: this machine may not have supported local provider - history. Use `ctx import --path` only for a known supported format. + history. Use `ctx import --provider --path ` only for a + known supported native provider format. - Import fails on a file: rerun with `--json` and inspect the per-source `failed` count. - Search returns no results: confirm `ctx status` shows indexed items, then diff --git a/docs/getting-started.md b/docs/getting-started.md index 4b9584208..3ac5b604d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -65,7 +65,7 @@ ctx import --all ctx import --provider codex ctx import --provider pi ctx import --provider cursor -ctx import --path ~/.codex/sessions +ctx import --provider codex --path ~/.codex/sessions ctx import --resume --json ``` @@ -79,7 +79,8 @@ After upgrading an older data root to `0.10.x` or newer, the first refresh or im re-read previously indexed provider transcripts once. That rebuilds search content with touched-file metadata and local/private transcript text. -When `--path` is used without `--provider`, ctx treats the path as Codex format. +Native provider `--path` imports require `--provider`. Custom JSONL imports use +`--format ctx-history-jsonl-v1 --path ` instead. ## 5. Search diff --git a/docs/storage.md b/docs/storage.md index 2cd5c93db..70231d116 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -125,7 +125,7 @@ Re-import or update the index: ```bash ctx import --all ctx import --resume -ctx import --path ~/.codex/sessions +ctx import --provider codex --path ~/.codex/sessions ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl ctx import --history-source example-agent/default ``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e4508f61f..b10deee70 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -12,7 +12,7 @@ Confirm the provider keeps history on this machine and pass an explicit path if needed: ```bash -ctx import --path ~/.codex/sessions +ctx import --provider codex --path ~/.codex/sessions ``` ## Search Misses Recent Work diff --git a/sdks/python/tests/test_client.py b/sdks/python/tests/test_client.py index fc3e1282d..1d6b34a1c 100644 --- a/sdks/python/tests/test_client.py +++ b/sdks/python/tests/test_client.py @@ -64,7 +64,10 @@ def test_init_sources_import_sync_search_and_inspect_methods(self) -> None: self.assertEqual(client.init(catalog_only=True)["operation"], "init") self.assertEqual(client.sources()["operation"], "sources") self.assertEqual(client.import_(provider="codex", resume=True)["operation"], "import") - self.assertEqual(client.sync(path="/tmp/history.jsonl")["operation"], "sync") + self.assertEqual( + client.sync(provider="codex", path="/tmp/history.jsonl")["operation"], + "sync", + ) self.assertEqual( client.search( "sqlite", From 1ab83050b0261f965e7db26060ab04d546605d3b Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 19:39:33 -0500 Subject: [PATCH 38/72] Prepare 0.17.0 release hardening Co-authored-by: luca-ctx <216224554+luca-ctx@users.noreply.github.com> --- Cargo.lock | 10 +- MODULE.bazel | 2 +- crates/ctx-cli/Cargo.toml | 2 +- crates/ctx-cli/src/upgrade.rs | 153 +++++++++++++++++++++++++- crates/ctx-cli/tests/cli.rs | 36 ++++++ crates/ctx-history-capture/Cargo.toml | 2 +- crates/ctx-history-core/Cargo.toml | 2 +- crates/ctx-history-search/Cargo.toml | 2 +- crates/ctx-history-store/Cargo.toml | 2 +- docs/cli-reference.md | 3 + docs/first-10-minutes.md | 4 +- docs/upgrade.md | 2 + scripts/build-public-cli-artifact.sh | 11 +- 13 files changed, 212 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c73a7ae42..ee0d1dde5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,7 +256,7 @@ dependencies = [ [[package]] name = "ctx" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "assert_cmd", @@ -282,7 +282,7 @@ dependencies = [ [[package]] name = "ctx-history-capture" -version = "0.16.0" +version = "0.17.0" dependencies = [ "chrono", "ctx-history-core", @@ -297,7 +297,7 @@ dependencies = [ [[package]] name = "ctx-history-core" -version = "0.16.0" +version = "0.17.0" dependencies = [ "chrono", "directories", @@ -310,7 +310,7 @@ dependencies = [ [[package]] name = "ctx-history-search" -version = "0.16.0" +version = "0.17.0" dependencies = [ "chrono", "ctx-history-core", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "ctx-history-store" -version = "0.16.0" +version = "0.17.0" dependencies = [ "chrono", "ctx-history-core", diff --git a/MODULE.bazel b/MODULE.bazel index 19a28dfc3..27ddfec22 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1 +1 @@ -module(name = "ctx_search", version = "0.13.0") +module(name = "ctx_search", version = "0.17.0") diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index f78231cc2..ecf44ca32 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx" -version = "0.16.0" +version = "0.17.0" description = "Local CLI for indexing and searching agent session history" edition.workspace = true autobins = false diff --git a/crates/ctx-cli/src/upgrade.rs b/crates/ctx-cli/src/upgrade.rs index 37ed6f18c..ef5a6ca57 100644 --- a/crates/ctx-cli/src/upgrade.rs +++ b/crates/ctx-cli/src/upgrade.rs @@ -126,9 +126,24 @@ struct UpgradePlan { update_available: bool, managed: bool, warnings: Vec, + path: PathDiagnostics, metadata: ReleaseMetadata, } +#[derive(Debug, Clone)] +struct PathDiagnostics { + current_exe: PathBuf, + entries: Vec, + warnings: Vec, +} + +#[derive(Debug, Clone)] +struct PathDiagnosticEntry { + path: PathBuf, + version: Option, + current: bool, +} + #[derive(Debug, Clone)] struct UpgradeOutcome { command: &'static str, @@ -164,6 +179,7 @@ impl UpgradeOutcome { "artifact_url": plan.map(|plan| plan.artifact_url.as_str()), "install_path": plan.map(|plan| plan.install_path.display().to_string()), "managed": plan.map(|plan| plan.managed).unwrap_or(false), + "path": plan.map(|plan| plan.path.json()), "applied": self.applied, "dry_run": self.dry_run, "warnings": self.warnings, @@ -171,6 +187,23 @@ impl UpgradeOutcome { } } +impl PathDiagnostics { + fn json(&self) -> Value { + json!({ + "current_exe": self.current_exe.display().to_string(), + "first_ctx": self.entries.first().map(|entry| entry.path.display().to_string()), + "entries": self.entries.iter().map(|entry| { + json!({ + "path": entry.path.display().to_string(), + "version": entry.version.as_deref(), + "current": entry.current, + }) + }).collect::>(), + "warnings": self.warnings, + }) + } +} + pub fn run(args: UpgradeArgs, data_root: PathBuf, config: AppConfig) -> Result<()> { if args.background { return run_background_apply(&data_root, &config); @@ -400,6 +433,9 @@ fn build_upgrade_plan( ¤t_version, &mut warnings, )?; + let managed = warnings.is_empty(); + let path = path_diagnostics(&marker.install_path, ¤t_version); + warnings.extend(path.warnings.clone()); let metadata_url = metadata_url(config, &channel); let signature_url = metadata_signature_url(&metadata_url); let metadata_bytes = net::get_bytes(&metadata_url) @@ -425,8 +461,9 @@ fn build_upgrade_plan( artifact_sha256: metadata.sha256.clone(), install_path: marker.install_path.clone(), update_available, - managed: warnings.is_empty(), + managed, warnings, + path, metadata, }) } @@ -450,6 +487,11 @@ fn render_status(data_root: &Path, json_output: bool) -> Result<()> { "status": "never_checked" }) }); + let current_version = env!("CARGO_PKG_VERSION"); + let current_exe = current_install_path().ok(); + let path_diagnostics = current_exe + .as_ref() + .map(|path| path_diagnostics(path, current_version)); let marker = read_verified_install_marker_for_current_exe() .map(|marker| { json!({ @@ -470,8 +512,14 @@ fn render_status(data_root: &Path, json_output: bool) -> Result<()> { let value = json!({ "schema_version": 1, "command": "upgrade_status", + "current_version": current_version, "state": state, "install": marker, + "path": path_diagnostics.as_ref().map(PathDiagnostics::json), + "warnings": path_diagnostics + .as_ref() + .map(|diagnostics| diagnostics.warnings.clone()) + .unwrap_or_default(), }); if json_output { println!("{}", serde_json::to_string_pretty(&value)?); @@ -484,11 +532,29 @@ fn render_status(data_root: &Path, json_output: bool) -> Result<()> { if let Some(path) = marker.get("install_path").and_then(Value::as_str) { println!("install: {path}"); } + if let Some(diagnostics) = &path_diagnostics { + println!("current_exe: {}", diagnostics.current_exe.display()); + if let Some(first) = diagnostics.entries.first() { + println!("path_ctx: {}", first.path.display()); + } + for warning in &diagnostics.warnings { + eprintln!("warning: {warning}"); + } + } } else { println!("ctx upgrade status: unmanaged install"); if let Some(reason) = marker.get("reason").and_then(Value::as_str) { println!("{reason}"); } + if let Some(diagnostics) = &path_diagnostics { + println!("current_exe: {}", diagnostics.current_exe.display()); + if let Some(first) = diagnostics.entries.first() { + println!("path_ctx: {}", first.path.display()); + } + for warning in &diagnostics.warnings { + eprintln!("warning: {warning}"); + } + } } Ok(()) } @@ -1004,6 +1070,91 @@ fn current_binary_sha() -> Result { Ok(sha256_hex(&bytes)) } +fn path_diagnostics(current_exe: &Path, current_version: &str) -> PathDiagnostics { + let current_identity = path_identity(current_exe); + let current_display = current_exe.display().to_string(); + let binary_name = if cfg!(windows) { "ctx.exe" } else { "ctx" }; + let mut entries = Vec::new(); + for dir in env::var_os("PATH") + .map(|path| env::split_paths(&path).collect::>()) + .unwrap_or_default() + { + let candidate = dir.join(binary_name); + if !candidate.is_file() { + continue; + } + if entries + .iter() + .any(|entry: &PathDiagnosticEntry| same_path(&entry.path, &candidate)) + { + continue; + } + let current = path_identity(&candidate) == current_identity; + entries.push(PathDiagnosticEntry { + version: ctx_binary_version(&candidate).ok(), + path: candidate, + current, + }); + } + + let mut warnings = Vec::new(); + match entries.first() { + Some(first) if !first.current => warnings.push(format!( + "PATH resolves ctx to {} before the current executable {}; your shell may keep using the earlier binary after upgrade", + first.path.display(), + current_display + )), + None => warnings.push(format!( + "current ctx executable {current_display} is not discoverable on PATH" + )), + _ => {} + } + if entries.len() > 1 { + warnings.push(format!( + "multiple ctx binaries are on PATH; first is {}", + entries[0].path.display() + )); + } + let expected = format!("ctx {current_version}"); + for entry in &entries { + if let Some(version) = &entry.version { + if version != &expected { + warnings.push(format!( + "ctx on PATH at {} reports {version}; current binary reports {expected}", + entry.path.display() + )); + } + } + } + + PathDiagnostics { + current_exe: current_exe.to_path_buf(), + entries, + warnings, + } +} + +fn path_identity(path: &Path) -> PathBuf { + fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +fn same_path(left: &Path, right: &Path) -> bool { + path_identity(left) == path_identity(right) +} + +fn ctx_binary_version(path: &Path) -> Result { + let output = Command::new(path) + .arg("--version") + .stdin(Stdio::null()) + .output() + .with_context(|| format!("run {} --version", path.display()))?; + if !output.status.success() { + return Err(anyhow!("{} --version failed", path.display())); + } + let stdout = String::from_utf8_lossy(&output.stdout); + Ok(stdout.lines().next().unwrap_or_default().trim().to_owned()) +} + fn write_state_checked(data_root: &Path, plan: &UpgradePlan, status: &str) -> Result<()> { let body = json!({ "schema_version": 1, diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 52d239ee6..3cd18535a 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -2359,6 +2359,42 @@ fn upgrade_status_check_and_apply_support_managed_installs() { assert_eq!(marker["sha256"], release.artifact_sha); } +#[cfg(unix)] +#[test] +fn upgrade_status_reports_path_shadowing() { + let temp = tempdir(); + let release = fake_release(&temp, "9.9.9"); + let shadow_dir = temp.path().join("shadow-bin"); + fs::create_dir_all(&shadow_dir).unwrap(); + let shadow_ctx = shadow_dir.join("ctx"); + write_fake_ctx_binary(&shadow_ctx, "0.9.0"); + let managed_dir = release.target.parent().unwrap(); + let path = std::env::join_paths([shadow_dir.as_path(), managed_dir]).unwrap(); + + let mut command = ctx(&temp); + command + .args(["upgrade", "status", "--json"]) + .env("PATH", path); + let status = json_output(fake_release_env(&mut command, &release)); + + assert_eq!(status["current_version"], env!("CARGO_PKG_VERSION")); + assert_eq!( + status["path"]["entries"][0]["path"], + shadow_ctx.display().to_string() + ); + assert_eq!(status["path"]["entries"][0]["version"], "ctx 0.9.0"); + assert!(status["warnings"] + .as_array() + .unwrap() + .iter() + .any(|warning| { warning.as_str().unwrap().contains("PATH resolves ctx to") })); + assert!(status["warnings"] + .as_array() + .unwrap() + .iter() + .any(|warning| { warning.as_str().unwrap().contains("reports ctx 0.9.0") })); +} + #[cfg(unix)] #[test] fn upgrade_rejects_unmanaged_install_before_network() { diff --git a/crates/ctx-history-capture/Cargo.toml b/crates/ctx-history-capture/Cargo.toml index 1619eef1d..ddb3d4d5b 100644 --- a/crates/ctx-history-capture/Cargo.toml +++ b/crates/ctx-history-capture/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-capture" -version = "0.16.0" +version = "0.17.0" description = "Internal provider import adapters for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-core/Cargo.toml b/crates/ctx-history-core/Cargo.toml index 117957db5..58f01d25b 100644 --- a/crates/ctx-history-core/Cargo.toml +++ b/crates/ctx-history-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-core" -version = "0.16.0" +version = "0.17.0" description = "Internal core types for ctx local agent history indexing" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-search/Cargo.toml b/crates/ctx-history-search/Cargo.toml index db4fabc39..b29bd370c 100644 --- a/crates/ctx-history-search/Cargo.toml +++ b/crates/ctx-history-search/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-search" -version = "0.16.0" +version = "0.17.0" description = "Internal search projection and ranking helpers for ctx" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-store/Cargo.toml b/crates/ctx-history-store/Cargo.toml index 00b301ab7..613c7be59 100644 --- a/crates/ctx-history-store/Cargo.toml +++ b/crates/ctx-history-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-store" -version = "0.16.0" +version = "0.17.0" description = "Internal SQLite storage layer for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 87c0f7b94..7acd83e98 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -389,6 +389,9 @@ binary, such as `~/.local/bin/ctx.install.json`, recording the managed install path, platform, version, channel, binary SHA-256, metadata URL, and artifact URL. Source builds, `cargo install`, package-manager installs, copied binaries, and mismatched sidecars are treated as unmanaged and will not self-upgrade. +`ctx upgrade status --json` also reports the current executable and every `ctx` +binary found on `PATH`, with warnings when an older binary shadows the managed +install or multiple `ctx` binaries are present. Official installer-managed installs default to background auto-upgrade after successful normal commands when signed release metadata explicitly allows diff --git a/docs/first-10-minutes.md b/docs/first-10-minutes.md index cf5f97ddd..f45862a40 100644 --- a/docs/first-10-minutes.md +++ b/docs/first-10-minutes.md @@ -115,8 +115,8 @@ ctx upgrade status ``` `ctx docs` is embedded in the binary for humans and agents. `ctx upgrade status` -shows whether the current binary is managed by the official installer and -eligible for signed self-upgrades. +shows whether the current binary is managed by the official installer, eligible +for signed self-upgrades, and shadowed by another `ctx` binary on `PATH`. ## Failure Paths diff --git a/docs/upgrade.md b/docs/upgrade.md index dd23d009b..e192655ec 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -19,6 +19,8 @@ The installer writes a sidecar marker next to the binary, such as version, channel, binary SHA-256, metadata URL, and artifact URL. Source builds, `cargo install`, package-manager installs, copied binaries, and mismatched sidecars are treated as unmanaged and will not self-upgrade. +`ctx upgrade status --json` also lists every `ctx` binary found on `PATH` and +warns when another binary shadows the managed install. Official installer-managed installs default to background auto-upgrade after successful normal commands when signed release metadata explicitly allows diff --git a/scripts/build-public-cli-artifact.sh b/scripts/build-public-cli-artifact.sh index 194fe3817..a86d5c9e7 100755 --- a/scripts/build-public-cli-artifact.sh +++ b/scripts/build-public-cli-artifact.sh @@ -113,10 +113,11 @@ ensure_darwin_cross_tools() { } version="$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys; data=json.load(sys.stdin); print(next(pkg["version"] for pkg in data["packages"] if pkg["name"] == "ctx"))')" -if [[ "${version}" != "0.16.0" ]]; then - echo "error: ctx package version must be 0.16.0 for this release, got ${version}" >&2 +if [[ -z "${version}" ]]; then + echo "error: could not determine ctx package version from Cargo metadata" >&2 exit 1 fi +echo "building ctx ${version} for ${platform}" rustup target add "${target}" >/dev/null out_dir="${CTX_PUBLIC_CLI_ARTIFACT_DIR:-target/public-cli-artifacts}" @@ -156,12 +157,12 @@ fi case "${platform}" in linux-x64) "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.16.0" "${staged}.version" >/dev/null + grep -Fx "ctx ${version}" "${staged}.version" >/dev/null ;; macos-arm64) if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.16.0" "${staged}.version" >/dev/null + grep -Fx "ctx ${version}" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi @@ -169,7 +170,7 @@ case "${platform}" in macos-x64) if [[ "$(uname -s)" == "Darwin" ]] && /usr/bin/arch -x86_64 /usr/bin/true >/dev/null 2>&1; then /usr/bin/arch -x86_64 "${staged}" --version | tee "${staged}.version" - grep -Fx "ctx 0.16.0" "${staged}.version" >/dev/null + grep -Fx "ctx ${version}" "${staged}.version" >/dev/null else printf 'not run on this host: %s\n' "${platform}" > "${staged}.version" fi From 613b1abc53912bb5597971207e73b04f9f2e9348 Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 22:28:03 -0500 Subject: [PATCH 39/72] Skip unreadable child dirs while probing providers Skip unreadable child dirs while probing providers - keep unreadable provider roots as probe errors - skip unreadable nested children while scanning for JSONL sessions - add regression coverage for readable sessions beside an unreadable child Tests: cargo fmt --check; cargo test -p ctx-history-capture provider_source --- .../src/provider_sources.rs | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/crates/ctx-history-capture/src/provider_sources.rs b/crates/ctx-history-capture/src/provider_sources.rs index 18fa82ea0..0ec12c234 100644 --- a/crates/ctx-history-capture/src/provider_sources.rs +++ b/crates/ctx-history-capture/src/provider_sources.rs @@ -854,16 +854,17 @@ fn has_jsonl_file_under_matching( } let mut visited = 0usize; - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { + let mut stack = vec![(root.to_path_buf(), true)]; + while let Some((dir, is_root)) = stack.pop() { let entries = match std::fs::read_dir(&dir) { Ok(entries) => entries, - Err(_) => return BoundedProbe::IoError, + Err(_) if is_root => return BoundedProbe::IoError, + Err(_) => continue, }; for entry in entries { let entry = match entry { Ok(entry) => entry, - Err(_) => return BoundedProbe::IoError, + Err(_) => continue, }; visited = visited.saturating_add(1); if visited > max_entries { @@ -872,10 +873,10 @@ fn has_jsonl_file_under_matching( let path = entry.path(); let file_type = match entry.file_type() { Ok(file_type) => file_type, - Err(_) => return BoundedProbe::IoError, + Err(_) => continue, }; if file_type.is_dir() { - stack.push(path); + stack.push((path, false)); } else if file_type.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("jsonl") && matches_path(&path) @@ -1119,6 +1120,40 @@ mod tests { .contains("could not be read")); } + #[cfg(unix)] + #[test] + fn default_source_probe_skips_unreadable_child_directory() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let sessions = temp.path().join(".codex/sessions"); + let readable = sessions.join("readable"); + let unreadable = sessions.join("unreadable"); + std::fs::create_dir_all(&readable).unwrap(); + std::fs::create_dir_all(&unreadable).unwrap(); + std::fs::write(readable.join("session.jsonl"), "{}\n").unwrap(); + + let original_permissions = std::fs::metadata(&unreadable).unwrap().permissions(); + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)).unwrap(); + + if std::fs::read_dir(&unreadable).is_ok() { + std::fs::set_permissions(&unreadable, original_permissions).unwrap(); + return; + } + + let source = discover_provider_sources(temp.path()) + .into_iter() + .find(|source| { + source.provider == CaptureProvider::Codex + && source.source_format == "codex_session_jsonl_tree" + }); + std::fs::set_permissions(&unreadable, original_permissions).unwrap(); + + let source = source.unwrap(); + assert_eq!(source.status, ProviderSourceStatus::Available); + assert_eq!(source.unsupported_reason, None); + } + fn assert_source_status( home: &Path, provider: CaptureProvider, From 80106f7ff89331e0073478ceb4fb329e31fce5cc Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 22:33:45 -0500 Subject: [PATCH 40/72] Make file-only search exact Make file-only search exact - keep ctx search --file valid as file-history lookup - enumerate exact file-touch scope instead of recent-record scanning - suppress import/source bookkeeping-only matches from normal search Tests: cargo fmt --check; cargo test -p ctx-history-search; cargo test -p ctx --test cli file_only_search_returns_touched_file_matches; cargo test -p ctx --test cli human_search_reports_no_results; cargo test -p ctx --test cli search_requires_query_term_or_file_before_refreshing --- crates/ctx-history-search/src/lib.rs | 289 +++++++++++++++++++++++---- 1 file changed, 250 insertions(+), 39 deletions(-) diff --git a/crates/ctx-history-search/src/lib.rs b/crates/ctx-history-search/src/lib.rs index 896ebb6fb..fd217f92e 100644 --- a/crates/ctx-history-search/src/lib.rs +++ b/crates/ctx-history-search/src/lib.rs @@ -1050,7 +1050,6 @@ fn ranked_candidates( file_scope: Option<&FileTouchScope>, ) -> Result { let target_candidates = options.limit.saturating_add(1); - let filtered = has_filters(&options.filters); let terms = query_terms(query.unwrap_or_default()); let mut candidates = Vec::new(); let mut seen = BTreeSet::::new(); @@ -1063,21 +1062,47 @@ fn ranked_candidates( }); } + if file_only { + let Some(scope) = file_scope else { + return Ok(CandidateSearch { + candidates, + scan_budget_exhausted, + }); + }; + for record_id in &scope.history_record_ids { + if !seen.insert(*record_id) { + continue; + } + let record = store.get_record(*record_id)?; + if let Some(candidate) = + candidate_for_record(store, record, &terms, &options.filters, file_scope)? + { + candidates.push(candidate); + } + } + normalize_scores(&mut candidates); + candidates.sort_by(compare_candidates); + if candidates.len() > target_candidates { + candidates.truncate(target_candidates); + } + return Ok(CandidateSearch { + candidates, + scan_budget_exhausted, + }); + } + + let filtered = has_filters(&options.filters); if filtered { let page_size = FILTERED_SEARCH_PAGE_SIZE.max(target_candidates); let mut offset = 0_usize; let mut pages_scanned = 0_usize; loop { pages_scanned = pages_scanned.saturating_add(1); - let records = if file_only { - store.list_records_page(page_size, offset)? - } else { - match query { - Some(query) if !query.trim().is_empty() => { - store.search_records_page(query, page_size, offset)? - } - _ => Vec::new(), + let records = match query { + Some(query) if !query.trim().is_empty() => { + store.search_records_page(query, page_size, offset)? } + _ => Vec::new(), }; let page_len = records.len(); @@ -1114,15 +1139,9 @@ fn ranked_candidates( } } else { let fetch_limit = target_candidates; - let records = if file_only { - store.list_records(fetch_limit)? - } else { - match query { - Some(query) if !query.trim().is_empty() => { - store.search_records(query, fetch_limit)? - } - _ => Vec::new(), - } + let records = match query { + Some(query) if !query.trim().is_empty() => store.search_records(query, fetch_limit)?, + _ => Vec::new(), }; for record in records { if !seen.insert(record.id) { @@ -1140,14 +1159,7 @@ fn ranked_candidates( } normalize_scores(&mut candidates); - candidates.sort_by(|left, right| { - right - .score - .total_cmp(&left.score) - .then_with(|| right.record.updated_at.cmp(&left.record.updated_at)) - .then_with(|| left.record.title.cmp(&right.record.title)) - .then_with(|| left.record.id.cmp(&right.record.id)) - }); + candidates.sort_by(compare_candidates); if candidates.len() > target_candidates { candidates.truncate(target_candidates); } @@ -1157,6 +1169,15 @@ fn ranked_candidates( }) } +fn compare_candidates(left: &Candidate, right: &Candidate) -> Ordering { + right + .score + .total_cmp(&left.score) + .then_with(|| right.record.updated_at.cmp(&left.record.updated_at)) + .then_with(|| left.record.title.cmp(&right.record.title)) + .then_with(|| left.record.id.cmp(&right.record.id)) +} + fn candidate_for_record( store: &Store, record: HistoryRecord, @@ -1394,19 +1415,23 @@ fn search_sections( ) -> Vec { let mut sections = Vec::new(); let record_hit = record_context_display_hit(context, filters, record.updated_at); - sections.push(SearchSection { - reason: "title", - weight: 8.0, - text: record.title.clone(), - citation: citation( - ContextCitationType::HistoryRecord, - record.id, - "session title", - record.updated_at, - ), - hit: record_hit.clone(), - }); - let include_record_text = record_text_matches_agent_scope(context, filters) + let include_record_bookkeeping_text = !is_agent_history_bookkeeping_record(record); + if include_record_bookkeeping_text { + sections.push(SearchSection { + reason: "title", + weight: 8.0, + text: record.title.clone(), + citation: citation( + ContextCitationType::HistoryRecord, + record.id, + "session title", + record.updated_at, + ), + hit: record_hit.clone(), + }); + } + let include_record_text = include_record_bookkeeping_text + && record_text_matches_agent_scope(context, filters) && !context_has_excluded_provider_session(context, filters); if include_record_text { sections.push(SearchSection { @@ -1624,6 +1649,19 @@ fn search_sections( sections } +fn is_agent_history_bookkeeping_record(record: &HistoryRecord) -> bool { + record.kind == "agent_history" + || record.tags.iter().any(|tag| tag == "agent-history") + || record + .body + .trim_start() + .starts_with("Indexed local agent history from ") + || record + .body + .trim_start() + .starts_with("Indexed custom agent history from ") +} + fn session_matches_agent_scope(session: &Session, filters: &SearchFilters) -> bool { if filters.session == Some(session.id) { return true; @@ -2507,6 +2545,7 @@ fn search_snippet( } } if !record.body.trim().is_empty() + && !is_agent_history_bookkeeping_record(record) && record_text_matches_agent_scope(context, filters) && !context_has_excluded_provider_session(context, filters) { @@ -4199,6 +4238,178 @@ mod tests { assert!(packet.results.is_empty()); } + #[test] + fn file_only_search_finds_old_sparse_file_touch_beyond_recent_scan_budget() { + let (_temp, store) = test_store(); + let old_time = fixed_time() - chrono::Duration::days(30); + let target_id = Uuid::parse_str("018f45d0-0000-7000-8003-ffffffffffff").unwrap(); + let mut target = HistoryRecord::new( + "Old sparse file touch", + "older session that only relates through file touch scope", + Vec::new(), + "task", + Some("/workspace/ctx".into()), + ); + target.id = target_id; + target.created_at = old_time; + target.updated_at = old_time; + store.upsert_record(&target).unwrap(); + store + .upsert_file_touched(&FileTouched { + id: Uuid::parse_str("018f45d0-0000-7000-8003-fffffffffffe").unwrap(), + history_record_id: Some(target_id), + run_id: None, + event_id: None, + vcs_workspace_id: None, + path: "crates/ctx-history-search/src/sparse_history.rs".into(), + change_kind: Some(FileChangeKind::Modified), + old_path: None, + line_count_delta: Some(1), + confidence: Confidence::Explicit, + timestamps: EntityTimestamps { + created_at: old_time, + updated_at: old_time, + }, + source_id: None, + sync: sync_metadata(), + }) + .unwrap(); + + let mut decoys = Vec::new(); + for index in 0..=(FILTERED_SEARCH_PAGE_SIZE * FILTERED_SEARCH_MAX_PAGES) { + let decoy_time = fixed_time() + chrono::Duration::seconds(index as i64); + let mut decoy = HistoryRecord::new( + "Recent unrelated session", + format!("recent non-file decoy {index:05}"), + Vec::new(), + "task", + Some("/workspace/other".into()), + ); + decoy.id = Uuid::parse_str(&format!("018f45d0-0000-7000-8004-{index:012x}")).unwrap(); + decoy.created_at = decoy_time; + decoy.updated_at = decoy_time; + decoys.push(decoy); + } + store.upsert_records(&decoys).unwrap(); + + let old_scan_window = store + .list_records_page(FILTERED_SEARCH_PAGE_SIZE * FILTERED_SEARCH_MAX_PAGES, 0) + .unwrap(); + assert!( + !old_scan_window.iter().any(|record| record.id == target_id), + "regression setup must place the file match beyond the old recent-record scan window" + ); + + let packet = search_packet( + &store, + "", + &PacketOptions { + limit: 5, + filters: SearchFilters { + file: Some("sparse_history.rs".into()), + ..SearchFilters::default() + }, + ..PacketOptions::default() + }, + ) + .unwrap(); + + assert_eq!( + packet + .results + .iter() + .map(|result| result.record_id) + .collect::>(), + vec![target_id] + ); + assert!(!packet.truncation.truncated); + assert!(packet.results[0] + .why_matched + .iter() + .any(|reason| reason == "file_touched")); + } + + #[test] + fn search_ignores_agent_history_bookkeeping_terms_without_content_evidence() { + let (_temp, store) = test_store(); + let mut record = HistoryRecord::new( + "codex agent history", + "Indexed local agent history from /tmp/codex/sessions.jsonl (codex_session_jsonl)", + vec!["agent-history".into(), "codex".into()], + "agent_history", + Some("/tmp/codex".into()), + ); + record.id = Uuid::parse_str("018f45d0-0000-7000-8005-000000000001").unwrap(); + record.created_at = fixed_time(); + record.updated_at = fixed_time(); + store.upsert_record(&record).unwrap(); + + for query in [ + "Indexed local agent history", + "agent-history", + "codex_session_jsonl", + ] { + let packet = search_packet(&store, query, &PacketOptions::default()).unwrap(); + assert!( + packet.results.is_empty(), + "bookkeeping-only query {query:?} returned {:?}", + packet.results + ); + } + + let session = Session { + id: Uuid::parse_str("018f45d0-0000-7000-8005-000000000002").unwrap(), + history_record_id: Some(record.id), + parent_session_id: None, + root_session_id: None, + capture_source_id: None, + provider: CaptureProvider::Codex, + external_session_id: Some("bookkeeping-content-session".into()), + external_agent_id: None, + agent_type: AgentType::Primary, + role_hint: Some("primary".into()), + is_primary: true, + status: SessionStatus::Imported, + transcript_blob_id: None, + started_at: fixed_time(), + ended_at: None, + timestamps: timestamps(), + sync: sync_metadata(), + }; + store.upsert_session(&session).unwrap(); + let event = Event { + id: Uuid::parse_str("018f45d0-0000-7000-8005-000000000003").unwrap(), + seq: 1, + history_record_id: Some(record.id), + session_id: Some(session.id), + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::Assistant), + occurred_at: fixed_time(), + capture_source_id: None, + payload: serde_json::json!({ + "text": "actual agent-history session evidence" + }), + payload_blob_id: None, + dedupe_key: None, + redaction_state: RedactionState::SafePreview, + sync: sync_metadata(), + }; + store.upsert_event(&event).unwrap(); + + let packet = search_packet(&store, "agent-history", &PacketOptions::default()).unwrap(); + assert_eq!(packet.results.len(), 1); + assert_eq!(packet.results[0].event_id, Some(event.id)); + assert!(packet.results[0] + .why_matched + .iter() + .any(|reason| reason == "message")); + assert!(!packet.results[0] + .why_matched + .iter() + .any(|reason| reason == "title" || reason == "tag")); + } + #[test] fn filtered_search_stops_at_scan_budget_when_no_candidates_match() { let (_temp, store) = test_store(); From 1ae35f9b55fba10643a199e76d61764b881ab1ff Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 22:33:58 -0500 Subject: [PATCH 41/72] Fail loudly on malformed config Fail loudly on malformed config - strictly parse present config.toml files - reject malformed keys and invalid privacy/update settings instead of silently using defaults - test that invalid config exits before setup or analytics side effects Tests: cargo fmt --check; cargo test -p ctx config; cargo test -p ctx malformed_present_config_fails_before_setup_and_analytics_side_effects --- crates/ctx-cli/src/config.rs | 324 +++++++++++++++++++++++++++++------ crates/ctx-cli/tests/cli.rs | 41 +++++ 2 files changed, 311 insertions(+), 54 deletions(-) diff --git a/crates/ctx-cli/src/config.rs b/crates/ctx-cli/src/config.rs index aade01a13..7d1ed4f3e 100644 --- a/crates/ctx-cli/src/config.rs +++ b/crates/ctx-cli/src/config.rs @@ -1,12 +1,12 @@ use std::{ collections::BTreeMap, env, fs, - io::Write, + io::{self, Write}, path::{Path, PathBuf}, time::Duration, }; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; pub const CONFIG_FILE: &str = "config.toml"; @@ -51,38 +51,50 @@ impl AppConfig { pub fn load(data_root: &Path) -> Result { let mut config = Self::default(); let path = data_root.join(CONFIG_FILE); - if path.exists() { - let text = - fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; - let parsed = parse_toml_subset(&text); - config.apply_values(&parsed); + match fs::read_to_string(&path) { + Ok(text) => { + let parsed = parse_toml_subset(&text) + .with_context(|| format!("parse {}", path.display()))?; + config + .apply_values(&parsed) + .with_context(|| format!("load {}", path.display()))?; + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(err).with_context(|| format!("read {}", path.display())), } config.apply_env(); Ok(config) } - fn apply_values(&mut self, values: &BTreeMap) { - if let Some(enabled) = parse_bool(values.get("analytics.enabled")) { - self.analytics.enabled = enabled; - } - if let Some(endpoint) = parse_string(values.get("analytics.endpoint")) { - self.analytics.endpoint = endpoint; - } - if let Some(auto) = parse_string(values.get("upgrade.auto")) { - self.upgrade.auto = auto; - } - if let Some(channel) = parse_string(values.get("upgrade.channel")) { - self.upgrade.channel = channel; - } - if let Some(hours) = parse_u64(values.get("upgrade.interval_hours")) { - self.upgrade.interval = Duration::from_secs(hours.saturating_mul(60 * 60)); - } - if let Some(seconds) = parse_u64(values.get("upgrade.interval_seconds")) { - self.upgrade.interval = Duration::from_secs(seconds); - } - if let Some(functions_base) = parse_string(values.get("upgrade.functions_base")) { - self.upgrade.functions_base = functions_base; + fn apply_values(&mut self, values: &BTreeMap) -> Result<()> { + for (key, value) in values { + match key.as_str() { + "analytics.enabled" => { + self.analytics.enabled = parse_config_bool(key, value)?; + } + "analytics.endpoint" => { + self.analytics.endpoint = parse_non_empty_string(key, value)?; + } + "upgrade.auto" => { + self.upgrade.auto = parse_upgrade_auto(value)?; + } + "upgrade.channel" => { + self.upgrade.channel = parse_non_empty_string(key, value)?; + } + "upgrade.interval_hours" => { + let hours = parse_config_u64(key, value)?; + self.upgrade.interval = Duration::from_secs(hours.saturating_mul(60 * 60)); + } + "upgrade.interval_seconds" => { + self.upgrade.interval = Duration::from_secs(parse_config_u64(key, value)?); + } + "upgrade.functions_base" => { + self.upgrade.functions_base = parse_non_empty_string(key, value)?; + } + _ => bail!("unknown config key `{key}` at line {}", value.line), + } } + Ok(()) } fn apply_env(&mut self) { @@ -144,54 +156,134 @@ interval_hours = 24\n", Ok(()) } -fn parse_toml_subset(text: &str) -> BTreeMap { +#[derive(Debug, Clone)] +struct ConfigValue { + raw: String, + line: usize, +} + +fn parse_toml_subset(text: &str) -> Result> { let mut section = String::new(); let mut values = BTreeMap::new(); - for raw_line in text.lines() { - let line = raw_line.split('#').next().unwrap_or_default().trim(); + for (index, raw_line) in text.lines().enumerate() { + let line_number = index + 1; + let line = strip_comment(raw_line).trim(); if line.is_empty() { continue; } - if line.starts_with('[') && line.ends_with(']') { - section = line - .trim_start_matches('[') - .trim_end_matches(']') - .trim() - .to_owned(); + if line.starts_with('[') { + if !line.ends_with(']') { + bail!("invalid config section header at line {line_number}: {line}"); + } + section = line[1..line.len() - 1].trim().to_owned(); + if section.is_empty() { + bail!("empty config section header at line {line_number}"); + } continue; } let Some((key, value)) = line.split_once('=') else { - continue; + bail!("invalid config line {line_number}: expected `[section]` or `key = value`"); }; let key = key.trim(); if key.is_empty() { - continue; + bail!("empty config key at line {line_number}"); } let full_key = if section.is_empty() { key.to_owned() } else { format!("{section}.{key}") }; - values.insert( - full_key, - value.trim().trim_end_matches(',').trim().to_owned(), - ); + let value = ConfigValue { + raw: value.trim().to_owned(), + line: line_number, + }; + if let Some(previous) = values.insert(full_key.clone(), value) { + bail!( + "duplicate config key `{full_key}` at line {line_number}; first set at line {}", + previous.line + ); + } } - values + Ok(values) } -fn parse_string(value: Option<&String>) -> Option { - value - .map(|value| value.trim().trim_matches('"').trim_matches('\'').to_owned()) - .filter(|value| !value.is_empty()) +fn strip_comment(line: &str) -> &str { + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut escaped = false; + for (index, ch) in line.char_indices() { + if in_double_quote { + if escaped { + escaped = false; + continue; + } + match ch { + '\\' => escaped = true, + '"' => in_double_quote = false, + _ => {} + } + continue; + } + if in_single_quote { + if ch == '\'' { + in_single_quote = false; + } + continue; + } + match ch { + '#' => return &line[..index], + '"' => in_double_quote = true, + '\'' => in_single_quote = true, + _ => {} + } + } + line } -fn parse_bool(value: Option<&String>) -> Option { - value.and_then(|value| parse_bool_value(value)) +fn parse_non_empty_string(key: &str, value: &ConfigValue) -> Result { + let parsed = parse_config_string(key, value)?; + if parsed.trim().is_empty() { + bail!("{key} at line {} must not be empty", value.line); + } + Ok(parsed) } -fn parse_u64(value: Option<&String>) -> Option { - value.and_then(|value| value.trim().trim_matches('"').parse::().ok()) +fn parse_config_string(key: &str, value: &ConfigValue) -> Result { + let raw = value.raw.trim(); + if raw.len() >= 2 + && ((raw.starts_with('"') && raw.ends_with('"')) + || (raw.starts_with('\'') && raw.ends_with('\''))) + { + return Ok(raw[1..raw.len() - 1].to_owned()); + } + bail!("{key} at line {} must be a quoted string", value.line); +} + +fn parse_config_bool(key: &str, value: &ConfigValue) -> Result { + match value.raw.trim() { + "true" => Ok(true), + "false" => Ok(false), + _ => bail!("{key} at line {} must be a boolean", value.line), + } +} + +fn parse_config_u64(key: &str, value: &ConfigValue) -> Result { + value + .raw + .trim() + .parse::() + .with_context(|| format!("{key} at line {} must be an unsigned integer", value.line)) +} + +fn parse_upgrade_auto(value: &ConfigValue) -> Result { + let auto = parse_non_empty_string("upgrade.auto", value)?; + match auto.to_ascii_lowercase().as_str() { + "apply" | "off" => Ok(auto.to_ascii_lowercase()), + _ => bail!( + "upgrade.auto at line {} must be either \"apply\" or \"off\"", + value.line + ), + } } fn parse_bool_value(value: &str) -> Option { @@ -228,7 +320,8 @@ auto = "off" channel = "beta" interval_seconds = 60 "#, - ); + ) + .unwrap(); let mut config = AppConfig::default(); assert_eq!( config.analytics.endpoint, @@ -236,10 +329,133 @@ interval_seconds = 60 ); assert!(config.analytics.enabled); assert_eq!(config.upgrade.auto, "apply"); - config.apply_values(&values); + config.apply_values(&values).unwrap(); assert!(!config.analytics.enabled); assert_eq!(config.upgrade.auto, "off"); assert_eq!(config.upgrade.channel, "beta"); assert_eq!(config.upgrade.interval, Duration::from_secs(60)); } + + #[test] + fn load_without_config_file_uses_defaults() { + let temp = tempfile::tempdir().unwrap(); + + let config = AppConfig::load(temp.path()).unwrap(); + + assert!(config.analytics.enabled); + assert_eq!(config.upgrade.auto, "apply"); + assert_eq!(config.upgrade.channel, "stable"); + assert_eq!(config.upgrade.interval, Duration::from_secs(24 * 60 * 60)); + } + + #[test] + fn load_valid_config_file_applies_values() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join(CONFIG_FILE), + r#" +[analytics] +enabled = false +endpoint = "file:///tmp/ctx-analytics.jsonl" + +[upgrade] +auto = "off" +channel = "beta" +interval_hours = 2 +functions_base = "https://example.test/functions/v1" +"#, + ) + .unwrap(); + + let config = AppConfig::load(temp.path()).unwrap(); + + assert!(!config.analytics.enabled); + assert_eq!(config.analytics.endpoint, "file:///tmp/ctx-analytics.jsonl"); + assert_eq!(config.upgrade.auto, "off"); + assert_eq!(config.upgrade.channel, "beta"); + assert_eq!(config.upgrade.interval, Duration::from_secs(2 * 60 * 60)); + assert_eq!( + config.upgrade.functions_base, + "https://example.test/functions/v1" + ); + } + + #[test] + fn rejects_invalid_config_booleans() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join(CONFIG_FILE), + "[analytics]\nenabled = flase\n", + ) + .unwrap(); + + let error = format!("{:#}", AppConfig::load(temp.path()).unwrap_err()); + + assert!(error.contains("analytics.enabled"), "{error}"); + assert!(error.contains("boolean"), "{error}"); + } + + #[test] + fn rejects_invalid_upgrade_auto_values() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join(CONFIG_FILE), + "[upgrade]\nauto = \"offf\"\n", + ) + .unwrap(); + + let error = format!("{:#}", AppConfig::load(temp.path()).unwrap_err()); + + assert!(error.contains("upgrade.auto"), "{error}"); + assert!(error.contains("\"apply\" or \"off\""), "{error}"); + } + + #[test] + fn rejects_unquoted_upgrade_auto_values() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join(CONFIG_FILE), "[upgrade]\nauto = offf\n").unwrap(); + + let error = format!("{:#}", AppConfig::load(temp.path()).unwrap_err()); + + assert!(error.contains("upgrade.auto"), "{error}"); + assert!(error.contains("quoted string"), "{error}"); + } + + #[test] + fn rejects_invalid_config_numbers() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join(CONFIG_FILE), + "[upgrade]\ninterval_seconds = nope\n", + ) + .unwrap(); + + let error = format!("{:#}", AppConfig::load(temp.path()).unwrap_err()); + + assert!(error.contains("upgrade.interval_seconds"), "{error}"); + assert!(error.contains("unsigned integer"), "{error}"); + } + + #[test] + fn rejects_malformed_config_lines() { + let error = parse_toml_subset("[upgrade]\nthis is not valid\n").unwrap_err(); + let error = error.to_string(); + + assert!(error.contains("invalid config line 2"), "{error}"); + } + + #[test] + fn rejects_unknown_config_keys() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join(CONFIG_FILE), + "[analytics]\nenabld = false\n", + ) + .unwrap(); + + let error = format!("{:#}", AppConfig::load(temp.path()).unwrap_err()); + + assert!(error.contains("unknown config key"), "{error}"); + assert!(error.contains("analytics.enabld"), "{error}"); + } } diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 3cd18535a..d73afd71c 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -838,6 +838,47 @@ fn setup_writes_day_one_config_contract_without_overwriting_existing_config() { ); } +#[test] +fn malformed_present_config_fails_before_setup_and_analytics_side_effects() { + let temp = tempdir(); + let state = temp.path().join("state"); + let events_path = temp.path().join("analytics.jsonl"); + fs::write( + temp.path().join("config.toml"), + "[analytics]\nenabled = flase\n", + ) + .unwrap(); + + ctx(&temp) + .arg("setup") + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .assert() + .failure() + .stderr( + predicate::str::contains("analytics.enabled").and(predicate::str::contains("boolean")), + ); + + assert!( + !temp.path().join("work.sqlite").exists(), + "setup must not create the store after config load fails" + ); + assert!( + !events_path.exists(), + "analytics endpoint should not be touched after config load fails" + ); + assert!( + !temp.path().join("install.json").exists(), + "analytics install identity should not be created after config load fails" + ); + assert!( + !expected_device_path(temp.path(), &state).exists(), + "analytics device identity should not be created after config load fails" + ); +} + #[test] fn setup_catalog_only_catalogs_codex_sessions_without_import() { let temp = tempdir(); From fd22ab073c6e70f3e76b11c949929e51bcf3d0fe Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 22:34:22 -0500 Subject: [PATCH 42/72] Harden upgrade diagnostics and locking Harden upgrade diagnostics and locking - avoid executing shadow PATH ctx binaries during diagnostics - bound staged version probes with timeout and output cap - recover stale dead-PID/aged upgrade locks while active locks still block Tests: cargo fmt --check; cargo test -p ctx --test cli upgrade_ -- --nocapture; cargo clippy -p ctx --all-targets -- -D warnings --- crates/ctx-cli/src/upgrade.rs | 277 ++++++++++++++++++++++++++++++---- crates/ctx-cli/tests/cli.rs | 121 ++++++++++++++- 2 files changed, 363 insertions(+), 35 deletions(-) diff --git a/crates/ctx-cli/src/upgrade.rs b/crates/ctx-cli/src/upgrade.rs index ef5a6ca57..5196bec49 100644 --- a/crates/ctx-cli/src/upgrade.rs +++ b/crates/ctx-cli/src/upgrade.rs @@ -1,10 +1,12 @@ use std::{ collections::BTreeMap, env, fs, - io::Write, + io::{Read, Write}, path::{Path, PathBuf}, process::{Command, Stdio}, - time::{Duration, SystemTime, UNIX_EPOCH}, + sync::mpsc, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use anyhow::{anyhow, Context, Result}; @@ -19,6 +21,9 @@ use crate::{config::AppConfig, net}; const STATE_FILE: &str = "upgrade-state.json"; const LOCK_FILE: &str = "upgrade.lock"; const LOG_FILE: &str = "logs/upgrade.log"; +const VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(2); +const VERSION_PROBE_OUTPUT_LIMIT: usize = 4096; +const STALE_UPGRADE_LOCK_AFTER: Duration = Duration::from_secs(30 * 60); const DEFAULT_METADATA_PUBLIC_KEY_PEM: &str = r#"-----BEGIN RSA PUBLIC KEY----- MIIBigKCAYEAyBPNIx3H/NwWlN9CPHY5kOEe9kQEshOJEMpv3Atq086H1FWqliTm 3BCWiO4s/89wNMn11Pla2JetCWNiWsbxm3BIxCd1o6cq8y9ur6Zk1RGOQBLQgqhF @@ -797,19 +802,12 @@ fn apply_artifact(plan: &UpgradePlan, bytes: &[u8]) -> Result { } fn verify_staged_version(staged: &Path, expected_version: &str) -> Result<()> { - let output = Command::new(staged) - .arg("--version") - .stdin(Stdio::null()) - .output() + let version = ctx_binary_version(staged) .with_context(|| format!("run staged ctx {}", staged.display()))?; - if !output.status.success() { - return Err(anyhow!("staged ctx --version failed")); - } - let stdout = String::from_utf8_lossy(&output.stdout); - if !stdout.contains(expected_version) { + if !version.contains(expected_version) { return Err(anyhow!( "staged ctx version mismatch: expected {expected_version}, got {}", - stdout.trim() + version.trim() )); } Ok(()) @@ -1091,7 +1089,7 @@ fn path_diagnostics(current_exe: &Path, current_version: &str) -> PathDiagnostic } let current = path_identity(&candidate) == current_identity; entries.push(PathDiagnosticEntry { - version: ctx_binary_version(&candidate).ok(), + version: current.then(|| format!("ctx {current_version}")), path: candidate, current, }); @@ -1143,18 +1141,110 @@ fn same_path(left: &Path, right: &Path) -> bool { } fn ctx_binary_version(path: &Path) -> Result { - let output = Command::new(path) - .arg("--version") - .stdin(Stdio::null()) - .output() - .with_context(|| format!("run {} --version", path.display()))?; + let output = run_ctx_version_command(path)?; if !output.status.success() { return Err(anyhow!("{} --version failed", path.display())); } + if output.truncated { + return Err(anyhow!( + "{} --version output exceeded {} bytes", + path.display(), + VERSION_PROBE_OUTPUT_LIMIT + )); + } let stdout = String::from_utf8_lossy(&output.stdout); Ok(stdout.lines().next().unwrap_or_default().trim().to_owned()) } +struct VersionCommandOutput { + status: std::process::ExitStatus, + stdout: Vec, + truncated: bool, +} + +fn run_ctx_version_command(path: &Path) -> Result { + let mut child = Command::new(path) + .arg("--version") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .with_context(|| format!("run {} --version", path.display()))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| anyhow!("capture {} --version output", path.display()))?; + let (output_tx, output_rx) = mpsc::channel(); + thread::spawn(move || { + let _ = output_tx.send(read_capped_output(stdout, VERSION_PROBE_OUTPUT_LIMIT)); + }); + let started = Instant::now(); + let mut status = None; + let mut output = None; + loop { + if status.is_none() { + status = child + .try_wait() + .with_context(|| format!("wait for {} --version", path.display()))?; + } + if output.is_none() { + match output_rx.try_recv() { + Ok(result) => { + output = + Some(result.with_context(|| { + format!("read {} --version output", path.display()) + })?); + } + Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => { + return Err(anyhow!( + "reader thread stopped for {} --version", + path.display() + )); + } + } + } + match (status.take(), output.take()) { + (Some(status), Some((stdout, truncated))) => { + return Ok(VersionCommandOutput { + status, + stdout, + truncated, + }); + } + (next_status, next_output) => { + status = next_status; + output = next_output; + } + } + if started.elapsed() >= VERSION_PROBE_TIMEOUT { + let _ = child.kill(); + let _ = child.wait(); + return Err(anyhow!( + "{} --version timed out after {}ms", + path.display(), + VERSION_PROBE_TIMEOUT.as_millis() + )); + } + thread::sleep(Duration::from_millis(10)); + } +} + +fn read_capped_output(mut reader: impl Read, limit: usize) -> std::io::Result<(Vec, bool)> { + let mut output = Vec::new(); + let mut buffer = [0_u8; 1024]; + while output.len() < limit { + let remaining = limit - output.len(); + let max_read = remaining.min(buffer.len()); + let read = reader.read(&mut buffer[..max_read])?; + if read == 0 { + return Ok((output, false)); + } + output.extend_from_slice(&buffer[..read]); + } + Ok((output, true)) +} + fn write_state_checked(data_root: &Path, plan: &UpgradePlan, status: &str) -> Result<()> { let body = json!({ "schema_version": 1, @@ -1223,21 +1313,148 @@ impl UpgradeLock { fn acquire(data_root: &Path) -> Result { fs::create_dir_all(data_root)?; let path = data_root.join(LOCK_FILE); - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&path) - { - Ok(mut file) => { - writeln!(file, "{} {}", std::process::id(), now_unix_s())?; - Ok(Self { path }) + for _ in 0..2 { + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(mut file) => { + writeln!(file, "{} {}", std::process::id(), now_unix_s())?; + return Ok(Self { path }); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + if stale_upgrade_lock_reason(&path).is_some() { + match fs::remove_file(&path) { + Ok(()) => continue, + Err(remove_error) + if remove_error.kind() == std::io::ErrorKind::NotFound => + { + continue; + } + Err(remove_error) => { + return Err(anyhow!( + "ctx upgrade lock is stale but could not be removed at {}: {remove_error}", + path.display() + )); + } + } + } + return Err(anyhow!( + "ctx upgrade lock is held at {}: {error}", + path.display() + )); + } + Err(error) => { + return Err(anyhow!( + "ctx upgrade lock is held at {}: {error}", + path.display() + )); + } } - Err(error) => Err(anyhow!( - "ctx upgrade lock is held at {}: {error}", - path.display() - )), } + Err(anyhow!( + "ctx upgrade lock could not be acquired at {}", + path.display() + )) + } +} + +fn stale_upgrade_lock_reason(path: &Path) -> Option { + let contents = fs::read_to_string(path).ok(); + let (pid, created_at) = contents + .as_deref() + .map(parse_upgrade_lock) + .unwrap_or((None, None)); + if let Some(pid) = pid { + match process_state(pid) { + ProcessState::Running => return None, + ProcessState::NotRunning => { + return Some(format!( + "recorded upgrade process {pid} is no longer running" + )); + } + ProcessState::Unknown => {} + } + } + if lock_age_seconds(path, created_at) + .is_some_and(|age| age >= STALE_UPGRADE_LOCK_AFTER.as_secs()) + { + return Some(format!( + "upgrade lock is older than {} seconds", + STALE_UPGRADE_LOCK_AFTER.as_secs() + )); + } + None +} + +fn parse_upgrade_lock(contents: &str) -> (Option, Option) { + let mut fields = contents.split_whitespace(); + let pid = fields.next().and_then(|value| value.parse::().ok()); + let created_at = fields.next().and_then(|value| value.parse::().ok()); + (pid, created_at) +} + +fn lock_age_seconds(path: &Path, created_at: Option) -> Option { + if let Some(created_at) = created_at { + return Some(now_unix_s().saturating_sub(created_at)); + } + fs::metadata(path) + .ok() + .and_then(|metadata| metadata.modified().ok()) + .and_then(|modified| modified.elapsed().ok()) + .map(|age| age.as_secs()) +} + +enum ProcessState { + Running, + NotRunning, + Unknown, +} + +#[cfg(unix)] +fn process_state(pid: u32) -> ProcessState { + if pid == 0 { + return ProcessState::NotRunning; } + let result = unsafe { libc::kill(pid as libc::pid_t, 0) }; + if result == 0 { + return ProcessState::Running; + } + match last_errno() { + Some(libc::ESRCH) => ProcessState::NotRunning, + Some(libc::EPERM) => ProcessState::Running, + _ => ProcessState::Unknown, + } +} + +#[cfg(not(unix))] +fn process_state(_pid: u32) -> ProcessState { + ProcessState::Unknown +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn last_errno() -> Option { + Some(unsafe { *libc::__errno_location() }) +} + +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd"))] +fn last_errno() -> Option { + Some(unsafe { *libc::__error() }) +} + +#[cfg(all( + unix, + not(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "ios", + target_os = "freebsd" + )) +))] +fn last_errno() -> Option { + None } impl Drop for UpgradeLock { diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index d73afd71c..82b797e3e 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -2244,6 +2244,21 @@ fn write_fake_ctx_binary(path: &Path, version: &str) -> Vec { bytes } +#[cfg(unix)] +fn write_hanging_ctx_binary(path: &Path) { + fs::write( + path, + "#!/bin/sh\n\ +if [ -n \"${CTX_SHADOW_MARKER:-}\" ]; then\n\ + touch \"$CTX_SHADOW_MARKER\"\n\ +fi\n\ +sleep 5\n\ +printf 'ctx 0.1.0\\n'\n", + ) + .unwrap(); + make_file_executable(path); +} + #[cfg(unix)] fn make_file_executable(path: &Path) { use std::os::unix::fs::PermissionsExt; @@ -2423,17 +2438,113 @@ fn upgrade_status_reports_path_shadowing() { status["path"]["entries"][0]["path"], shadow_ctx.display().to_string() ); - assert_eq!(status["path"]["entries"][0]["version"], "ctx 0.9.0"); + assert!(status["path"]["entries"][0]["version"].is_null()); assert!(status["warnings"] .as_array() .unwrap() .iter() .any(|warning| { warning.as_str().unwrap().contains("PATH resolves ctx to") })); - assert!(status["warnings"] - .as_array() +} + +#[cfg(unix)] +#[test] +fn upgrade_commands_do_not_execute_hanging_shadow_path_ctx() { + for args in [ + ["upgrade", "status", "--json"].as_slice(), + ["upgrade", "check", "--json"].as_slice(), + ["upgrade", "--json"].as_slice(), + ] { + let temp = tempdir(); + let release = fake_release(&temp, "9.9.9"); + let shadow_dir = temp.path().join("shadow-bin"); + fs::create_dir_all(&shadow_dir).unwrap(); + let shadow_ctx = shadow_dir.join("ctx"); + write_hanging_ctx_binary(&shadow_ctx); + let marker = temp.path().join("shadow-ran"); + let managed_dir = release.target.parent().unwrap(); + let path = std::env::join_paths([shadow_dir.as_path(), managed_dir]).unwrap(); + + let started = Instant::now(); + let mut command = ctx(&temp); + command + .args(args) + .env("PATH", &path) + .env("CTX_SHADOW_MARKER", &marker); + let output = json_output(fake_release_env(&mut command, &release)); + let elapsed = started.elapsed(); + + assert!( + elapsed < Duration::from_secs(2), + "ctx {args:?} should not wait for shadow PATH binaries; elapsed {elapsed:?}" + ); + assert_eq!( + output["path"]["entries"][0]["path"], + shadow_ctx.display().to_string() + ); + assert!( + output["path"]["entries"][0]["version"].is_null(), + "shadow ctx versions should not be probed" + ); + assert!( + !marker.exists(), + "PATH shadow ctx should not have been executed" + ); + } +} + +#[cfg(unix)] +#[test] +fn upgrade_recovers_stale_lock_for_dead_pid() { + let temp = tempdir(); + let release = fake_release(&temp, "9.9.9"); + let mut child = std::process::Command::new("sh") + .arg("-c") + .arg("exit 0") + .spawn() + .unwrap(); + let stale_pid = child.id(); + child.wait().unwrap(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) .unwrap() - .iter() - .any(|warning| { warning.as_str().unwrap().contains("reports ctx 0.9.0") })); + .as_secs(); + fs::write( + temp.path().join("upgrade.lock"), + format!("{stale_pid} {}\n", now.saturating_sub(60)), + ) + .unwrap(); + + let dry_run = json_output(fake_release_env( + ctx(&temp).args(["upgrade", "--dry-run", "--json"]), + &release, + )); + + assert_eq!(dry_run["status"], "dry_run"); + assert!(!temp.path().join("upgrade.lock").exists()); +} + +#[cfg(unix)] +#[test] +fn upgrade_lock_still_rejects_active_pid() { + let temp = tempdir(); + let release = fake_release(&temp, "9.9.9"); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + fs::write( + temp.path().join("upgrade.lock"), + format!("{} {now}\n", std::process::id()), + ) + .unwrap(); + + let stderr = failure_stderr(fake_release_env( + ctx(&temp).args(["upgrade", "--dry-run"]), + &release, + )); + + assert!(stderr.contains("ctx upgrade lock is held"), "{stderr}"); + assert!(temp.path().join("upgrade.lock").exists()); } #[cfg(unix)] From a6a3f947abb3cb377abc2e2e24a4a97f0fcf1e38 Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 22:36:02 -0500 Subject: [PATCH 43/72] Clarify local preview redaction state Clarify local preview redaction state - prefer LocalPreview in Rust while preserving serialized safe_preview for compatibility - rename capture preview helpers away from safe wording - document that safe_preview/safe_preview_text are local searchable previews, not share-safe redaction Tests: cargo test -p ctx-history-core safe_preview_is_legacy_local_preview_spelling; cargo test -p ctx-history-store event_search_local_preview_preserves_private_text_but_raw_is_withheld; cargo check -p ctx-history-search -p ctx; bash scripts/check-docs.sh --- crates/ctx-history-capture/src/lib.rs | 58 ++++++++++---------- crates/ctx-history-core/src/lib.rs | 33 ++++++++++-- crates/ctx-history-store/src/lib.rs | 78 +++++++++++++++++++++++++-- docs/contracts/json.md | 7 +++ docs/redaction-corpus.md | 4 +- docs/security-checks.md | 2 + docs/sql.md | 2 +- docs/storage.md | 3 ++ 8 files changed, 150 insertions(+), 37 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index c88108373..e4fa91adc 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -1087,7 +1087,7 @@ impl ProviderCaptureAdapter for CodexHistoryJsonlAdapter { role: Some(EventRole::User), occurred_at, fidelity: Fidelity::SummaryOnly, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: Some(format!( "provider-event:{}:{}:{}", CaptureProvider::Codex.as_str(), @@ -4918,7 +4918,7 @@ fn codex_session_event( .get("payload") .and_then(codex_json_text) .unwrap_or_else(|| "context compacted".to_owned()); - let (text, truncated) = codex_safe_preview(&text, CODEX_MAX_TEXT_CHARS); + let (text, truncated) = codex_local_preview(&text, CODEX_MAX_TEXT_CHARS); Some(codex_provider_event( line_number, occurred_at, @@ -5061,7 +5061,7 @@ fn codex_tool_call_event( format!("{tool_name}: {arguments_preview}") } }); - let (text, text_truncated) = codex_safe_preview(&text, CODEX_MAX_METADATA_TEXT_CHARS); + let (text, text_truncated) = codex_local_preview(&text, CODEX_MAX_METADATA_TEXT_CHARS); if let Some(call_id) = call_id { call_contexts.insert( @@ -5152,7 +5152,7 @@ fn codex_tool_output_event( }; let (output_preview, output_truncated) = if keep_preview { output_text_ref - .map(|text| codex_safe_preview(text, preview_limit)) + .map(|text| codex_local_preview(text, preview_limit)) .unwrap_or_else(|| (String::new(), false)) } else { (String::new(), output_bytes > 0) @@ -5187,7 +5187,7 @@ fn codex_tool_output_event( format!("{tool_name} output{command}: {status}{duration}, output_bytes={output_bytes}{timeout}{preview}") } }; - let (text, text_truncated) = codex_safe_preview(&text, CODEX_MAX_OUTPUT_PREVIEW_CHARS); + let (text, text_truncated) = codex_local_preview(&text, CODEX_MAX_OUTPUT_PREVIEW_CHARS); Some(codex_provider_event( line_number, @@ -5244,7 +5244,7 @@ fn codex_reasoning_event( .and_then(Value::as_str) .map(str::to_owned) })?; - let (summary, truncated) = codex_safe_preview(&summary, CODEX_MAX_TEXT_CHARS); + let (summary, truncated) = codex_local_preview(&summary, CODEX_MAX_TEXT_CHARS); Some(codex_provider_event( line_number, occurred_at, @@ -5319,7 +5319,7 @@ fn codex_provider_event( role, occurred_at, fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: Some(format!("provider-event:codex-session:{line_number}")), artifacts: Vec::new(), payload, @@ -5335,7 +5335,7 @@ fn codex_lifecycle_body(payload: &Value, msg_type: &str) -> Value { .or_else(|| payload.get("stderr")) .and_then(codex_json_text) .unwrap_or_else(|| format!("Codex lifecycle: {msg_type}")); - let (text, truncated) = codex_safe_preview(&preview, CODEX_MAX_METADATA_TEXT_CHARS); + let (text, truncated) = codex_local_preview(&preview, CODEX_MAX_METADATA_TEXT_CHARS); json!({ "text": text, "event_msg_type": msg_type, @@ -5373,7 +5373,7 @@ fn codex_command_preview(tool_name: &str, argument_value: Option<&Value>) -> Opt .or_else(|| parsed.get("shell_command")) .and_then(Value::as_str) .or_else(|| value.as_str())?; - Some(codex_safe_preview(command, CODEX_MAX_METADATA_TEXT_CHARS).0) + Some(codex_local_preview(command, CODEX_MAX_METADATA_TEXT_CHARS).0) } fn codex_value_preview(value: &Value, max_chars: usize) -> (String, bool) { @@ -5382,10 +5382,10 @@ fn codex_value_preview(value: &Value, max_chars: usize) -> (String, bool) { Value::Null => String::new(), _ => serde_json::to_string(value).unwrap_or_else(|_| value.to_string()), }; - codex_safe_preview(&rendered, max_chars) + codex_local_preview(&rendered, max_chars) } -fn codex_safe_preview(value: &str, max_chars: usize) -> (String, bool) { +fn codex_local_preview(value: &str, max_chars: usize) -> (String, bool) { capped_text(value, max_chars) } @@ -5521,7 +5521,7 @@ fn capped_text(value: &str, max_chars: usize) -> (String, bool) { (out, truncated) } -fn provider_safe_preview(value: &str, max_chars: usize) -> (String, bool) { +fn provider_local_preview(value: &str, max_chars: usize) -> (String, bool) { capped_text(value, max_chars) } @@ -6213,7 +6213,7 @@ fn claude_event( String::new() } }); - let (text, truncated) = provider_safe_preview(&text, PROVIDER_MAX_TEXT_CHARS); + let (text, truncated) = provider_local_preview(&text, PROVIDER_MAX_TEXT_CHARS); Some(ProviderEventEnvelope { provider_event_index: (line_number - 1) as u64, @@ -6223,7 +6223,7 @@ fn claude_event( role, occurred_at, fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: value .get("uuid") .and_then(Value::as_str) @@ -6292,12 +6292,12 @@ fn provider_capped_json(value: &Value, max_chars: usize) -> Value { match value { Value::Null => Value::Null, Value::String(text) => { - let (text, truncated) = provider_safe_preview(text, max_chars); + let (text, truncated) = provider_local_preview(text, max_chars); json!({ "text": text, "truncated": truncated }) } _ => { let rendered = serde_json::to_string(value).unwrap_or_else(|_| value.to_string()); - let (json_text, truncated) = provider_safe_preview(&rendered, max_chars); + let (json_text, truncated) = provider_local_preview(&rendered, max_chars); json!({ "json": json_text, "truncated": truncated }) } } @@ -6306,7 +6306,7 @@ fn provider_capped_json(value: &Value, max_chars: usize) -> Value { fn provider_capped_json_value(value: &Value, max_string_chars: usize) -> Value { match value { Value::String(text) => { - let (text, truncated) = provider_safe_preview(text, max_string_chars); + let (text, truncated) = provider_local_preview(text, max_string_chars); if truncated { json!({ "text": text, "truncated": true }) } else { @@ -6539,7 +6539,7 @@ struct NativeEventDraft { } fn native_event(draft: NativeEventDraft) -> ProviderEventEnvelope { - let (text, truncated) = provider_safe_preview(&draft.text, PROVIDER_MAX_TEXT_CHARS); + let (text, truncated) = provider_local_preview(&draft.text, PROVIDER_MAX_TEXT_CHARS); ProviderEventEnvelope { provider_event_index: draft.provider_event_index, provider_event_hash: draft.provider_event_hash, @@ -6548,7 +6548,7 @@ fn native_event(draft: NativeEventDraft) -> ProviderEventEnvelope { role: draft.role, occurred_at: draft.occurred_at, fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: Some(format!( "provider-event:{}:{}:{}", draft.provider.as_str(), @@ -8664,7 +8664,7 @@ fn opencode_event( let event_type = opencode_event_type(&row.entry_type, data); let role = Some(provider_role(Some(&row.entry_type))); let text = opencode_event_text(&row.entry_type, data, event_type); - let (text, truncated) = provider_safe_preview(&text, PROVIDER_MAX_TEXT_CHARS); + let (text, truncated) = provider_local_preview(&text, PROVIDER_MAX_TEXT_CHARS); ProviderEventEnvelope { provider_event_index: row.seq.max(0) as u64, provider_event_hash: Some(row.id.clone()), @@ -8676,7 +8676,7 @@ fn opencode_event( role, occurred_at, fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: Some(format!( "provider-event:opencode:{}:{}", row.session_id, row.id @@ -9187,7 +9187,7 @@ fn native_jsonl_event( let entry_type = native_jsonl_entry_type(provider, value); let role = native_jsonl_role(provider, value); let text = native_jsonl_event_text(provider, value, event_type, &entry_type); - let (text, truncated) = provider_safe_preview(&text, PROVIDER_MAX_TEXT_CHARS); + let (text, truncated) = provider_local_preview(&text, PROVIDER_MAX_TEXT_CHARS); let event_id = native_jsonl_event_id(provider, value, line_number); let tool_calls = if provider == CaptureProvider::Antigravity { value @@ -9205,7 +9205,7 @@ fn native_jsonl_event( role: Some(role), occurred_at, fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: Some(format!( "provider-event:{}:{source_format}:{event_id}", provider.as_str() @@ -9630,7 +9630,7 @@ fn pi_session_event(entry: &Value, line_number: usize) -> ProviderEventEnvelope role, occurred_at, fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: Some(format!("provider-event:pi:{line_number}")), artifacts: Vec::new(), payload: json!({ @@ -10395,7 +10395,7 @@ fn fixture_line_to_capture( role: event.role, occurred_at: event.occurred_at, fidelity, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: Some(format!( "provider-event:{}:{}:{}", fixture.provider.as_str(), @@ -10422,7 +10422,7 @@ fn effective_event_redaction_state( RedactionState::Redacted => RedactionState::Redacted, RedactionState::Raw if !sanitizer_redacted => RedactionState::Raw, _ if sanitizer_redacted => RedactionState::Redacted, - _ => RedactionState::SafePreview, + _ => RedactionState::LocalPreview, } } @@ -11609,7 +11609,7 @@ mod tests { let session_id = provider_session_uuid(CaptureProvider::Pi, "pi-session-1"); let events = store.events_for_session(session_id).unwrap(); assert_eq!(events.len(), 2); - assert_eq!(events[1].redaction_state, RedactionState::SafePreview); + assert_eq!(events[1].redaction_state, RedactionState::LocalPreview); assert!(events[1] .sync .metadata @@ -12413,7 +12413,7 @@ mod tests { role: Some(EventRole::Assistant), occurred_at: "2026-06-24T01:00:00Z".parse().unwrap(), fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: None, artifacts: Vec::new(), payload: serde_json::json!({}), @@ -12482,7 +12482,7 @@ mod tests { role: Some(EventRole::Assistant), occurred_at: "2026-06-24T01:00:00Z".parse().unwrap(), fidelity: Fidelity::Imported, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, idempotency_key: None, artifacts: Vec::new(), payload: serde_json::json!({}), diff --git a/crates/ctx-history-core/src/lib.rs b/crates/ctx-history-core/src/lib.rs index f7d819fd7..6a0d7663a 100644 --- a/crates/ctx-history-core/src/lib.rs +++ b/crates/ctx-history-core/src/lib.rs @@ -148,13 +148,26 @@ text_enum! { } text_enum! { + /// Payload handling state. + /// + /// The serialized value `safe_preview` is legacy contract spelling for a + /// local searchable preview. It is not a promise that output is share-safe. pub enum RedactionState { Raw => "raw", Redacted => "redacted", - SafePreview => "safe_preview", + LocalPreview => "safe_preview", Withheld => "withheld", } - default SafePreview + default LocalPreview +} + +impl RedactionState { + /// Compatibility alias for the legacy Rust API name. + /// + /// New code should prefer `LocalPreview`, which better matches the local + /// search contract while preserving the serialized `safe_preview` value. + #[allow(non_upper_case_globals)] + pub const SafePreview: Self = Self::LocalPreview; } text_enum! { @@ -1481,7 +1494,7 @@ mod tests { assert_eq!(Fidelity::default(), Fidelity::Partial); assert_eq!(SyncState::default(), SyncState::LocalOnly); assert_eq!(Confidence::default(), Confidence::Unknown); - assert_eq!(RedactionState::default(), RedactionState::SafePreview); + assert_eq!(RedactionState::default(), RedactionState::LocalPreview); assert_eq!( serde_json::from_str::("\"copilot_cli\"").unwrap(), CaptureProvider::CopilotCli @@ -1511,6 +1524,20 @@ mod tests { assert_eq!(outbox.sync_state, SyncState::Pending); } + #[test] + fn safe_preview_is_legacy_local_preview_spelling() { + assert_eq!(RedactionState::LocalPreview.as_str(), "safe_preview"); + assert_eq!( + "safe_preview".parse::().unwrap(), + RedactionState::LocalPreview + ); + assert_eq!( + serde_json::to_string(&RedactionState::LocalPreview).unwrap(), + "\"safe_preview\"" + ); + assert_eq!(RedactionState::SafePreview, RedactionState::LocalPreview); + } + #[test] fn history_record_json_names_are_public_names() { let record_id = Uuid::parse_str("018f45d0-0000-7000-8000-000000000001").unwrap(); diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 4e649327e..066682a59 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -973,6 +973,8 @@ CREATE INDEX IF NOT EXISTS idx_local_workspaces_vcs_workspace_id ON local_worksp CREATE INDEX IF NOT EXISTS idx_audit_log_source_id ON audit_log(source_id); "#; +// `safe_preview_text` is legacy schema naming. It stores local searchable +// preview text and must not be interpreted as share-safe redaction. const FTS_TABLES_SQL: &str = r#" CREATE VIRTUAL TABLE IF NOT EXISTS ctx_history_search USING fts5( record_id UNINDEXED, @@ -6045,8 +6047,8 @@ mod archive_validation_tests { blob_hash, byte_size, media_type: Some("text/markdown".into()), - preview_text: Some("synthetic public-safe blob".into()), - redaction_state: RedactionState::SafePreview, + preview_text: Some("synthetic local preview blob".into()), + redaction_state: RedactionState::LocalPreview, timestamps: EntityTimestamps { created_at: fixed_time(), updated_at: fixed_time(), @@ -7509,6 +7511,36 @@ mod search_order_tests { .with_timezone(&Utc) } + fn sync_metadata() -> SyncMetadata { + SyncMetadata { + visibility: Visibility::LocalOnly, + fidelity: Fidelity::Imported, + sync_state: SyncState::LocalOnly, + sync_version: 0, + deleted_at: None, + metadata: serde_json::json!({}), + } + } + + fn local_preview_event(seq: u64, text: &str, redaction_state: RedactionState) -> Event { + Event { + id: new_id(), + seq, + history_record_id: None, + session_id: None, + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::User), + occurred_at: fixed_time(), + capture_source_id: None, + payload: serde_json::json!({ "text": text }), + payload_blob_id: None, + dedupe_key: None, + redaction_state, + sync: sync_metadata(), + } + } + #[test] fn indexed_history_item_count_uses_sessions_and_events() { let temp = tempdir(); @@ -7657,6 +7689,46 @@ mod search_order_tests { assert!(store.search_records_page("", 10, 0).unwrap().is_empty()); } + #[test] + fn event_search_local_preview_preserves_private_text_but_raw_is_withheld() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let local_event = local_preview_event( + 1, + "cwd=/home/example/private token=ghp_1234567890abcdef", + RedactionState::LocalPreview, + ); + let raw_event = local_preview_event( + 2, + "raw cwd=/home/example/private token=ghp_1234567890abcdef", + RedactionState::Raw, + ); + + store.upsert_event(&local_event).unwrap(); + store.upsert_event(&raw_event).unwrap(); + + let local_preview: String = store + .conn + .query_row( + "SELECT safe_preview_text FROM event_search WHERE event_id = ?1", + [local_event.id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert!(local_preview.contains("/home/example/private")); + assert!(local_preview.contains("ghp_1234567890abcdef")); + + let raw_preview: String = store + .conn + .query_row( + "SELECT safe_preview_text FROM event_search WHERE event_id = ?1", + [raw_event.id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(raw_preview, "raw event payload withheld"); + } + #[test] fn upsert_record_updates_record_search_without_rebuilding_event_search() { let temp = tempdir(); @@ -8838,7 +8910,7 @@ mod catalog_tests { payload: serde_json::json!({"text": "migration source reference"}), payload_blob_id: None, dedupe_key: None, - redaction_state: RedactionState::SafePreview, + redaction_state: RedactionState::LocalPreview, sync: sync_metadata(), }; event_id = event.id; diff --git a/docs/contracts/json.md b/docs/contracts/json.md index ccb55203e..840844501 100644 --- a/docs/contracts/json.md +++ b/docs/contracts/json.md @@ -163,6 +163,13 @@ Writes nothing and returns: `occurred_at`, `source`, `cursor`, `text` or `preview`, and `redaction_state`. +`redaction_state` values describe local payload handling, not whether a row is +safe to publish. In particular, `safe_preview` is legacy contract spelling for a +local searchable preview: the text may be truncated or projected from provider +payloads, but it can still include absolute paths, token-shaped strings, command +output, and other private transcript content. Treat `safe_preview` output as +private unless a user separately reviews and redacts it. + ## Locate ```bash diff --git a/docs/redaction-corpus.md b/docs/redaction-corpus.md index e7bca1b1f..9697953da 100644 --- a/docs/redaction-corpus.md +++ b/docs/redaction-corpus.md @@ -15,4 +15,6 @@ credentials before indexing or display. Corpus tests should cover at least: Passing the corpus does not make output safe to share. It proves local search/show/SQLite projections preserve representative transcript text so users and agents can find exact local history. Share-safe or shared-service redaction -is outside the current local CLI contract. +is outside the current local CLI contract. Rows marked +`redaction_state: "safe_preview"` use that legacy spelling for a local searchable +preview and must still be treated as private local history. diff --git a/docs/security-checks.md b/docs/security-checks.md index db3071b8c..9ce02f463 100644 --- a/docs/security-checks.md +++ b/docs/security-checks.md @@ -32,6 +32,8 @@ the local retrieval product. - Search/show/locate JSON and SQLite search projections preserve local transcript text by default, including absolute paths and secret-shaped strings. They must be treated as private local data. +- The legacy `safe_preview` state and `safe_preview_text` columns mean local + searchable preview text, not share-safe redaction. - Unsupported providers remain explicit in the provider support matrix. ## Static Docs Checks diff --git a/docs/sql.md b/docs/sql.md index 9485ec45b..bd6b9e4bc 100644 --- a/docs/sql.md +++ b/docs/sql.md @@ -61,7 +61,7 @@ are implementation details and can change between versions. | `role` | Event role such as `user`, `assistant`, or `tool`, when known. | | `occurred_at_ms` | Unix epoch milliseconds. | | `payload_json` | Local private event payload. | -| `redaction_state` | Payload redaction/preview state. | +| `redaction_state` | Local payload handling state. `safe_preview` is legacy spelling for a local searchable preview, not share-safe redaction. | | `fidelity` | Import fidelity. | | `cwd`, `source_path` | Captured source context, when known. | diff --git a/docs/storage.md b/docs/storage.md index 70231d116..266f6271f 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -219,6 +219,9 @@ not remove provider-owned history such as `~/.codex/sessions`. No local search index can be considered share-safe by default. Indexed prompts, code, commands, file paths, and output previews may contain credentials, customer data, private repository names, or proprietary design notes. +The persisted `safe_preview` redaction state and `safe_preview_text` search +columns are legacy local-index names for searchable previews; they do not mean +the stored text has been redacted for sharing. Recommended handling: From 38241f0c1d167b2f98b358d8d8fa1c37807f66ac Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 22:38:53 -0500 Subject: [PATCH 44/72] Require search intent in SDKs Require search intent in SDKs - validate query/term/file intent locally before SDKs invoke ctx - update SDK tests and TypeScript types for invalid filters-only searches - fix docs/help/skill examples that implied bare ctx search commands Tests: npm test --prefix sdks/typescript; python3 -m unittest discover -s tests; go test ./...; sdks/jvm/scripts/test; dotnet run --project sdks/dotnet/tests/Ctx.AgentHistory.Tests/Ctx.AgentHistory.Tests.csproj; swift test; cargo test -p ctx-sdk; cargo test -p ctx --test cli human_search_reports_no_results; scripts/check-docs.sh --- contracts/agent-history-v1/README.md | 2 +- crates/ctx-cli/src/main.rs | 16 +++---- crates/ctx-cli/tests/cli.rs | 2 +- crates/ctx-sdk/src/lib.rs | 47 +++++++++++++++++++ docs/agent-usage.md | 6 +-- docs/contracts/json.md | 2 +- docs/search.md | 7 +-- .../skills/ctx-agent-history-search/SKILL.md | 8 ++-- sdks/dotnet/README.md | 2 +- .../Ctx.AgentHistory/AgentHistoryClient.cs | 13 +++++ .../tests/Ctx.AgentHistory.Tests/Program.cs | 22 ++++++++- sdks/go/README.md | 5 +- sdks/go/client.go | 16 +++++++ sdks/go/client_test.go | 21 +++++++++ .../ctx/agenthistory/AgentHistoryClient.java | 17 +++++++ .../agenthistory/AgentHistoryClientTest.java | 26 ++++++++++ sdks/python/src/ctx_agent_history/__init__.py | 2 + sdks/python/src/ctx_agent_history/client.py | 5 +- sdks/python/src/ctx_agent_history/errors.py | 19 ++++++++ .../python/src/ctx_agent_history/transport.py | 2 + .../src/ctx_agent_history/validation.py | 41 ++++++++++++++++ sdks/python/tests/test_client.py | 16 ++++++- .../CtxAgentHistory/AgentHistoryClient.swift | 18 +++++++ .../CtxAgentHistoryTests.swift | 6 +++ sdks/typescript/README.md | 2 +- sdks/typescript/src/index.d.ts | 11 ++++- sdks/typescript/src/index.js | 22 +++++++++ sdks/typescript/test/client.test.js | 12 +++++ sdks/typescript/test/types.test.ts | 9 ++++ skills/ctx-agent-history-search/SKILL.md | 8 ++-- 30 files changed, 351 insertions(+), 34 deletions(-) create mode 100644 sdks/python/src/ctx_agent_history/validation.py diff --git a/contracts/agent-history-v1/README.md b/contracts/agent-history-v1/README.md index 34626b1d3..3c8bea823 100644 --- a/contracts/agent-history-v1/README.md +++ b/contracts/agent-history-v1/README.md @@ -70,7 +70,7 @@ them into `agent-history-v1` wrappers: - `ctx setup --json` - `ctx sources --json` - `ctx import --json` -- `ctx search ... --json` +- `ctx search |--term |--file --json` - `ctx show event ... --format json` - `ctx show session ... --format json` - `ctx locate event ... --format json` diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 833acbfcc..4a766dc04 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -2885,7 +2885,7 @@ fn error_summary(error: &anyhow::Error) -> String { .map(ToString::to_string) .unwrap_or_else(|| top.clone()); if is_sqlite_busy_text(&top) || is_sqlite_busy_text(&root) { - return "ctx index is busy because another ctx import or search refresh is writing to the local database; retry in a moment, or use `ctx search --refresh off` to search the existing index".to_owned(); + return "ctx index is busy because another ctx import or search refresh is writing to the local database; retry in a moment, or rerun the search with `--refresh off` to use the existing index".to_owned(); } if root == top || top.contains(&root) { top @@ -3355,14 +3355,14 @@ fn push_session_metadata_markdown( fn resolve_session_by_id_text(store: &Store, value: &str) -> Result { if let Ok(id) = Uuid::parse_str(value.trim()) { return store.get_session(id).with_context(|| { - format!("session {id} was not found; use `ctx search --verbose` to get ctx_session_id") + format!("session {id} was not found; rerun the search that found it with `--verbose` to get ctx_session_id") }); } let prefix = normalize_uuid_prefix(value, "session")?; match store.sessions_by_id_prefix(&prefix)?.as_slice() { [session] => Ok(session.clone()), [] => Err(anyhow!( - "session id prefix {prefix:?} was not found; use `ctx search --verbose` to get ctx_session_id" + "session id prefix {prefix:?} was not found; rerun the search that found it with `--verbose` to get ctx_session_id" )), matches => Err(anyhow!( "session id prefix {prefix:?} is ambiguous; first matches are {} and {}; use a longer ctx_session_id", @@ -3380,7 +3380,7 @@ fn resolve_event(store: &Store, value: &str) -> Result { if let Ok(id) = Uuid::parse_str(value.trim()) { return store.get_event(id).with_context(|| { format!( - "event {id} was not found; use `ctx search --events --verbose` to get ctx_event_id" + "event {id} was not found; rerun the event search with `--events --verbose` to get ctx_event_id" ) }); } @@ -3388,7 +3388,7 @@ fn resolve_event(store: &Store, value: &str) -> Result { match store.events_by_id_prefix(&prefix)?.as_slice() { [event] => Ok(event.clone()), [] => Err(anyhow!( - "event id prefix {prefix:?} was not found; use `ctx search --events --verbose` to get ctx_event_id" + "event id prefix {prefix:?} was not found; rerun the event search with `--events --verbose` to get ctx_event_id" )), matches => Err(anyhow!( "event id prefix {prefix:?} is ambiguous; first matches are {} and {}; use a longer ctx_event_id", @@ -3407,7 +3407,7 @@ fn normalize_uuid_prefix(value: &str, kind: &str) -> Result { } if prefix.contains('-') || !prefix.chars().all(|ch| ch.is_ascii_hexdigit()) { return Err(anyhow!( - "{kind} id must be a full ctx UUID or an unambiguous hex prefix from `ctx search --verbose`" + "{kind} id must be a full ctx UUID or an unambiguous hex prefix from verbose search output" )); } Ok(prefix.to_ascii_lowercase()) @@ -4492,7 +4492,7 @@ fn run_search( if indexed_items == 0 { println!("next: ctx import --all"); } else { - println!("next: try broader terms with ctx search --term"); + println!("next: try broader terms with ctx search --term \"\""); } } } @@ -4665,7 +4665,7 @@ fn refresh_before_search(args: &SearchArgs, data_root: &Path) -> Result\"")); let term_only = ctx(&temp) .args(["search", "--term", "term-only-no-results"]) diff --git a/crates/ctx-sdk/src/lib.rs b/crates/ctx-sdk/src/lib.rs index 46434df24..9f696e73a 100644 --- a/crates/ctx-sdk/src/lib.rs +++ b/crates/ctx-sdk/src/lib.rs @@ -115,6 +115,21 @@ impl Default for SearchOptions { } } +impl SearchOptions { + fn has_intent(&self) -> bool { + self.query + .as_deref() + .map(str::trim) + .is_some_and(|query| !query.is_empty()) + || self.terms.iter().any(|term| !term.trim().is_empty()) + || self + .file + .as_ref() + .map(|path| !path.to_string_lossy().trim().is_empty()) + .unwrap_or(false) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SearchRefresh { Auto, @@ -215,6 +230,13 @@ impl AgentHistoryClient { &self, options: SearchOptions, ) -> Result { + if !options.has_intent() { + return Err(AgentHistoryError::new( + AgentHistoryErrorCode::InvalidRequest, + "search requires a query, term, or file option", + false, + )); + } let mut owned = Vec::::new(); owned.push("search".to_owned()); if let Some(query) = options.query { @@ -675,6 +697,31 @@ mod tests { assert_eq!(options.terms, vec!["ctx"]); } + #[test] + fn search_requires_query_term_or_file_before_cli() { + let client = AgentHistoryClient::local(LocalBackendConfig { + ctx_binary: PathBuf::from("/definitely/missing/ctx"), + data_root: None, + timeout: Duration::from_secs(1), + }); + + for options in [ + SearchOptions::default(), + SearchOptions { + refresh: SearchRefresh::Off, + ..SearchOptions::default() + }, + SearchOptions { + query: Some(" ".to_owned()), + terms: vec!["".to_owned(), " ".to_owned()], + ..SearchOptions::default() + }, + ] { + let err = client.search(options).unwrap_err(); + assert_eq!(err.body.code, AgentHistoryErrorCode::InvalidRequest); + } + } + #[test] fn local_client_can_dogfood_fake_ctx_without_private_history() { let temp = tempfile::tempdir().unwrap(); diff --git a/docs/agent-usage.md b/docs/agent-usage.md index e149b20c0..13a6e4433 100644 --- a/docs/agent-usage.md +++ b/docs/agent-usage.md @@ -18,9 +18,9 @@ ctx show event --window 5 ``` Normal `ctx search` uses `--refresh auto`, which can import newly discovered -provider history into the local ctx index before querying. Use -`ctx search ... --refresh off` when the task requires a strictly read-only -query over the existing index. +provider history into the local ctx index before querying. Rerun the same +search with `--refresh off` when the task requires a strictly read-only query +over the existing index. Use `ctx sql` only when normal search does not express the question, such as exact counts, joins, audits, or scripting over stable `ctx_*` views. It is diff --git a/docs/contracts/json.md b/docs/contracts/json.md index 840844501..06ad01378 100644 --- a/docs/contracts/json.md +++ b/docs/contracts/json.md @@ -263,7 +263,7 @@ external publication. - `error`, present when refresh failed but results were still served. `suggested_next_commands` can include `ctx show event`, `ctx show session`, -`ctx search ... --session `, `ctx locate event`, and +`ctx search "" --session `, `ctx locate event`, and `ctx locate session` command strings when the required ctx IDs are known. When ctx can identify the active Codex provider session through diff --git a/docs/search.md b/docs/search.md index 685ac062a..c9078190e 100644 --- a/docs/search.md +++ b/docs/search.md @@ -135,9 +135,10 @@ only retrieves indexed local evidence; it does not synthesize conclusions. ## Machine Output -Use default text output for agent reading. Use `ctx search --json` for scripts, -`jq`, or exact field extraction. JSON results include the same result metadata -and citations as the human output, plus a top-level `freshness` object +Use default text output for agent reading. Use `ctx search --json` or a +term/file search with `--json` for scripts, `jq`, or exact field extraction. +JSON results include the same result metadata and citations as the human output, +plus a top-level `freshness` object describing the pre-search refresh mode and outcome. A citation with `source_exists: false` means ctx can return indexed text, but the raw provider file was not available at the stored path when the result was built. diff --git a/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md b/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md index f1e99b091..364ecd52f 100644 --- a/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md +++ b/plugins/ctx-agent-history-search/skills/ctx-agent-history-search/SKILL.md @@ -63,9 +63,9 @@ Use this skill in two modes: When the prompt asks for a topic history or report across multiple sessions, run several `ctx search` queries with different wording and filters to find - promising sessions. Use scoped `ctx search ... --session ` - when a session looks relevant and you need dense event-level matches from - that session. + promising sessions. Use scoped + `ctx search "" --session ` when a session looks + relevant and you need dense event-level matches from that session. Default search returns primary-agent sessions so human intent and decisions stay prominent. Use `--include-subagents` when implementation details, code @@ -144,7 +144,7 @@ material. chronology, alternatives, or detailed evidence. 2. Run several targeted searches. Vary query terms across user wording, file or module names, error text, commands, branch names, and decision terms. Start - with default `ctx search`, then broaden with `--term` or narrow with + with `ctx search ""`, then broaden with `--term` or narrow with `--workspace`, `--provider`, `--file`, `--since`, or `--session `. Use `--include-subagents` when reviews, implementation attempts, test output, diff --git a/sdks/dotnet/README.md b/sdks/dotnet/README.md index ea92ff6e3..1b95fbf5f 100644 --- a/sdks/dotnet/README.md +++ b/sdks/dotnet/README.md @@ -81,7 +81,7 @@ future fields remain additive and accessible. SDK failures derive from - `ctx setup --json` - `ctx sources --json` - `ctx import --json` -- `ctx search ... --json` +- `ctx search |--term |--file --json` - `ctx show event ... --format json` - `ctx show session ... --format json` - `ctx locate event ... --format json` diff --git a/sdks/dotnet/src/Ctx.AgentHistory/AgentHistoryClient.cs b/sdks/dotnet/src/Ctx.AgentHistory/AgentHistoryClient.cs index c01c10584..b06362c5e 100644 --- a/sdks/dotnet/src/Ctx.AgentHistory/AgentHistoryClient.cs +++ b/sdks/dotnet/src/Ctx.AgentHistory/AgentHistoryClient.cs @@ -71,6 +71,7 @@ public async Task SyncAsync(ImportOptions? options = null, Cance public async Task SearchAsync(SearchOptions? options = null, CancellationToken cancellationToken = default) { options ??= new SearchOptions(); + RequireSearchIntent(options); var args = new List { "search" }; if (!string.IsNullOrWhiteSpace(options.Query)) { @@ -252,6 +253,18 @@ private static List BuildSessionLookupArgs(string command, string kind, return args; } + private static void RequireSearchIntent(SearchOptions options) + { + if (!string.IsNullOrWhiteSpace(options.Query) + || !string.IsNullOrWhiteSpace(options.File) + || (options.Terms?.Any(term => !string.IsNullOrWhiteSpace(term)) ?? false)) + { + return; + } + + throw new CtxAgentHistoryValidationException("search requires a query, term, or file option"); + } + private static void RequireValue(string value, string name) { if (string.IsNullOrWhiteSpace(value)) diff --git a/sdks/dotnet/tests/Ctx.AgentHistory.Tests/Program.cs b/sdks/dotnet/tests/Ctx.AgentHistory.Tests/Program.cs index ae2ec2355..2241ab281 100644 --- a/sdks/dotnet/tests/Ctx.AgentHistory.Tests/Program.cs +++ b/sdks/dotnet/tests/Ctx.AgentHistory.Tests/Program.cs @@ -12,6 +12,7 @@ private static async Task Main() ("builds local CLI operation arguments", BuildsOperationArguments), ("normalizes setup init status", NormalizesSetupInitStatus), ("builds search flags", BuildsSearchFlags), + ("rejects search without intent", RejectsSearchWithoutIntent), ("wraps show and locate commands", WrapsShowAndLocate), ("reports versioning metadata", ReportsVersioning), ("uses agent-history-v1 error codes", UsesAgentHistoryV1ErrorCodes), @@ -127,6 +128,25 @@ private static async Task BuildsSearchFlags() Equal("off", response.Search.Freshness!.Mode ?? ""); } + private static async Task RejectsSearchWithoutIntent() + { + var transport = new RecordingTransport("""{"schema_version":1,"results":[]}"""); + var client = new AgentHistoryClient(transport); + + await ThrowsAsync(() => client.SearchAsync()); + await ThrowsAsync(() => client.SearchAsync(new SearchOptions + { + Refresh = "off", + Limit = 5 + })); + await ThrowsAsync(() => client.SearchAsync(new SearchOptions + { + Query = " " + })); + + Equal(0, transport.Calls.Count); + } + private static async Task WrapsShowAndLocate() { var transport = new RecordingTransport("""{"schema_version":1,"events":[],"source":{"path":"/tmp/source.jsonl"},"ctx_session_id":"session-1","provider":"codex"}"""); @@ -227,7 +247,7 @@ private static async Task LoadsSharedFixtures() } break; case "search": - _ = (await ClientFor(node["search"]).SearchAsync()).Search.Results; + _ = (await ClientFor(node["search"]).SearchAsync(new SearchOptions { Query = "fixture search" })).Search.Results; break; case "showEvent": _ = (await ClientFor(node["event"]).ShowEventAsync("event-1")).Event.Events; diff --git a/sdks/go/README.md b/sdks/go/README.md index 302dcbd3d..4d0ce6fba 100644 --- a/sdks/go/README.md +++ b/sdks/go/README.md @@ -58,8 +58,9 @@ client := ctxagenthistory.NewLocalClient( ``` The adapter runs JSON-producing CLI commands such as `ctx status --json`, -`ctx search --json`, and `ctx show event --format json`, then normalizes CLI -JSON into `agent-history-v1` wrappers with `contractVersion` and `schemaVersion`. +`ctx search |--term |--file --json`, and +`ctx show event --format json`, then normalizes CLI JSON into +`agent-history-v1` wrappers with `contractVersion` and `schemaVersion`. ## Errors diff --git a/sdks/go/client.go b/sdks/go/client.go index 46924c1ba..c1445f511 100644 --- a/sdks/go/client.go +++ b/sdks/go/client.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strconv" + "strings" ) // Operation is the adapter-neutral command executed by a transport. @@ -156,6 +157,9 @@ func (c *Client) Sync(ctx context.Context, opts ImportOptions) (*ImportResponse, } func (c *Client) Search(ctx context.Context, opts SearchOptions) (*SearchResponse, error) { + if !opts.hasIntent() { + return nil, sdkError(ErrorKindInvalidArgument, "search requires a query, term, or file option", nil) + } args := []string{"search"} if opts.Query != "" { args = append(args, opts.Query) @@ -198,6 +202,18 @@ func (c *Client) Search(ctx context.Context, opts SearchOptions) (*SearchRespons return &out, nil } +func (opts SearchOptions) hasIntent() bool { + if strings.TrimSpace(opts.Query) != "" || strings.TrimSpace(opts.File) != "" { + return true + } + for _, term := range opts.Terms { + if strings.TrimSpace(term) != "" { + return true + } + } + return false +} + func (c *Client) ShowSession(ctx context.Context, opts ShowSessionOptions) (*ShowSessionResponse, error) { args := []string{"show", "session"} if opts.ID != "" { diff --git a/sdks/go/client_test.go b/sdks/go/client_test.go index 5835acb15..9202f25ec 100644 --- a/sdks/go/client_test.go +++ b/sdks/go/client_test.go @@ -91,6 +91,27 @@ func TestSearchBuildsAgentHistoryV1Operation(t *testing.T) { } } +func TestSearchRequiresQueryTermOrFileBeforeTransport(t *testing.T) { + transport := &recordingTransport{response: `{"schema_version":1,"results":[]}`} + client := NewClient(WithTransport(transport)) + + for name, opts := range map[string]SearchOptions{ + "empty": {}, + "filters only": {Refresh: "off", Limit: 5}, + "blank query": {Query: " "}, + "blank terms": {Terms: []string{"", " "}}, + } { + t.Run(name, func(t *testing.T) { + if _, err := client.Search(context.Background(), opts); !IsErrorKind(err, ErrorKindInvalidArgument) { + t.Fatalf("Search error kind mismatch: %v", err) + } + }) + } + if transport.op.Args != nil { + t.Fatalf("Search invoked transport despite invalid input: %#v", transport.op.Args) + } +} + func TestShowAndLocateValidateRequiredEventID(t *testing.T) { client := NewClient(WithTransport(fakeTransport{response: `{}`})) if _, err := client.ShowEvent(context.Background(), ShowEventOptions{}); !IsErrorKind(err, ErrorKindInvalidArgument) { diff --git a/sdks/jvm/src/main/java/rs/ctx/agenthistory/AgentHistoryClient.java b/sdks/jvm/src/main/java/rs/ctx/agenthistory/AgentHistoryClient.java index bde02bea9..62e36a3ed 100644 --- a/sdks/jvm/src/main/java/rs/ctx/agenthistory/AgentHistoryClient.java +++ b/sdks/jvm/src/main/java/rs/ctx/agenthistory/AgentHistoryClient.java @@ -84,6 +84,7 @@ public SearchResponse search(String query) { public SearchResponse search(AgentHistoryOptions.Search options) { AgentHistoryOptions.Search safe = options == null ? AgentHistoryOptions.search() : options; + requireSearchIntent(safe); List args = new ArrayList<>(); args.add("search"); if (safe.query() != null && !safe.query().isEmpty()) { @@ -112,6 +113,18 @@ public SearchResponse search(AgentHistoryOptions.Search options) { return new SearchResponse(executeEnvelope("search", args)); } + private static void requireSearchIntent(AgentHistoryOptions.Search options) { + if (hasText(options.query()) || hasText(options.file())) { + return; + } + for (String term : options.terms()) { + if (hasText(term)) { + return; + } + } + throw new CtxAgentHistoryException.Validation("search requires a query, term, or file option"); + } + public ShowEventResponse showEvent(String id, AgentHistoryOptions.ShowEvent options) { if (id == null || id.isEmpty()) { throw new CtxAgentHistoryException.Validation("event id is required"); @@ -234,6 +247,10 @@ private static void add(List args, String flag, String value) { } } + private static boolean hasText(String value) { + return value != null && !value.trim().isEmpty(); + } + private static void addInt(List args, String flag, Integer value) { if (value != null) { args.add(flag); diff --git a/sdks/jvm/src/test/java/rs/ctx/agenthistory/AgentHistoryClientTest.java b/sdks/jvm/src/test/java/rs/ctx/agenthistory/AgentHistoryClientTest.java index 2aaaf4a6c..8dfbf8936 100644 --- a/sdks/jvm/src/test/java/rs/ctx/agenthistory/AgentHistoryClientTest.java +++ b/sdks/jvm/src/test/java/rs/ctx/agenthistory/AgentHistoryClientTest.java @@ -15,6 +15,7 @@ public static void main(String[] args) throws Exception { decodesAllCanonicalFixturesThroughTypedResponses(); normalizesRawShowAndLocateResponses(); buildsSearchCommand(); + searchRequiresIntent(); hostedIsExplicitlyUnsupported(); } @@ -173,6 +174,21 @@ private static void buildsSearchCommand() { assertContainsInOrder(transport.lastOperation.args(), "--refresh", "off"); } + private static void searchRequiresIntent() { + FakeTransport transport = new FakeTransport( + "local-cli", + "{\"schema_version\":1,\"query\":\"client\",\"results\":[]}"); + AgentHistoryClient client = AgentHistoryClient.withTransport(transport); + + assertValidation(() -> client.search()); + assertValidation(() -> client.search(AgentHistoryOptions.search().refresh("off").limit(5))); + assertValidation(() -> client.search(" ")); + assertValidation(() -> client.search(AgentHistoryOptions.search().term(" "))); + if (transport.lastOperation != null) { + throw new AssertionError("invalid search invoked transport: " + transport.lastOperation.args()); + } + } + private static void hostedIsExplicitlyUnsupported() { AgentHistoryClient client = AgentHistoryClient.hosted(HostedConfig.builder().baseUrl("https://ctx.example.invalid").build()); try { @@ -214,6 +230,16 @@ private static void assertEquals(Object want, Object got) { } } + private static void assertValidation(Runnable action) { + try { + action.run(); + } catch (CtxAgentHistoryException.Validation error) { + assertEquals("invalid_request", error.code()); + return; + } + throw new AssertionError("expected validation error"); + } + private static final class FakeTransport implements AgentHistoryTransport { private final String name; private final String response; diff --git a/sdks/python/src/ctx_agent_history/__init__.py b/sdks/python/src/ctx_agent_history/__init__.py index 23089272f..425396e3a 100644 --- a/sdks/python/src/ctx_agent_history/__init__.py +++ b/sdks/python/src/ctx_agent_history/__init__.py @@ -7,6 +7,7 @@ CtxAgentHistoryError, CtxAgentHistoryProtocolError, CtxAgentHistoryTimeoutError, + CtxAgentHistoryValidationError, HostedTransportNotImplementedError, ) from .types import ( @@ -35,6 +36,7 @@ "CtxAgentHistoryError", "CtxAgentHistoryProtocolError", "CtxAgentHistoryTimeoutError", + "CtxAgentHistoryValidationError", "ErrorResponse", "HostedConfig", "HostedTransportNotImplementedError", diff --git a/sdks/python/src/ctx_agent_history/client.py b/sdks/python/src/ctx_agent_history/client.py index 0531a0bc0..5a4cf9156 100644 --- a/sdks/python/src/ctx_agent_history/client.py +++ b/sdks/python/src/ctx_agent_history/client.py @@ -20,6 +20,7 @@ StatusResponse, SyncResponse, ) +from .validation import validate_search_intent from .version import API_VERSION, SDK_VERSION, VersionInfo Pathish = Union[str, Path] @@ -134,13 +135,15 @@ def search( refresh: Optional[str] = None, include_current_session: bool = False, ) -> SearchResponse: + file_value = str(file) if file is not None else None + validate_search_intent(query=query, terms=terms, file=file_value) return self._transport.search( query=query, provider=provider, workspace=workspace, since=since, event_type=event_type, - file=str(file) if file is not None else None, + file=file_value, session=session, terms=list(terms) if terms is not None else None, events=events, diff --git a/sdks/python/src/ctx_agent_history/errors.py b/sdks/python/src/ctx_agent_history/errors.py index d9ff9c934..ab8622621 100644 --- a/sdks/python/src/ctx_agent_history/errors.py +++ b/sdks/python/src/ctx_agent_history/errors.py @@ -84,6 +84,25 @@ def __init__( ) +class CtxAgentHistoryValidationError(CtxAgentHistoryError): + """Raised before invoking ctx for invalid SDK input.""" + + def __init__( + self, + message: str, + *, + details: Optional[Mapping[str, Any]] = None, + cause: Optional[BaseException] = None, + ) -> None: + super().__init__( + message, + code="invalid_request", + details=details, + retryable=False, + cause=cause, + ) + + class CtxAgentHistoryTimeoutError(CtxAgentHistoryError): """Raised when the local ctx CLI exceeds the configured timeout.""" diff --git a/sdks/python/src/ctx_agent_history/transport.py b/sdks/python/src/ctx_agent_history/transport.py index fd2d10a95..a302b6531 100644 --- a/sdks/python/src/ctx_agent_history/transport.py +++ b/sdks/python/src/ctx_agent_history/transport.py @@ -40,6 +40,7 @@ StatusResponse, SyncResponse, ) +from .validation import validate_search_intent class AgentHistoryTransport(Protocol): @@ -235,6 +236,7 @@ def search( refresh: Optional[str] = None, include_current_session: bool = False, ) -> SearchResponse: + validate_search_intent(query=query, terms=terms, file=file) args = ["search", "--json"] if query is not None: args.append(query) diff --git a/sdks/python/src/ctx_agent_history/validation.py b/sdks/python/src/ctx_agent_history/validation.py new file mode 100644 index 000000000..6f9584edc --- /dev/null +++ b/sdks/python/src/ctx_agent_history/validation.py @@ -0,0 +1,41 @@ +"""SDK input validation helpers.""" + +from __future__ import annotations + +from typing import Optional, Sequence + +from .errors import CtxAgentHistoryValidationError + + +def validate_search_intent( + *, + query: Optional[str], + terms: Optional[Sequence[str]], + file: Optional[str], +) -> None: + if _has_text(query) or _has_text(file) or _has_term(terms): + return + raise CtxAgentHistoryValidationError( + "search requires a query, term, or file option", + details={"query": query, "terms": _term_details(terms), "file": file}, + ) + + +def _has_term(terms: Optional[Sequence[str]]) -> bool: + if terms is None: + return False + if isinstance(terms, str): + return _has_text(terms) + return any(_has_text(term) for term in terms) + + +def _has_text(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _term_details(terms: Optional[Sequence[str]]) -> list[str]: + if terms is None: + return [] + if isinstance(terms, str): + return [terms] + return list(terms) diff --git a/sdks/python/tests/test_client.py b/sdks/python/tests/test_client.py index 1d6b34a1c..5bf1808a6 100644 --- a/sdks/python/tests/test_client.py +++ b/sdks/python/tests/test_client.py @@ -22,7 +22,7 @@ AgentHistoryClient, ) from ctx_agent_history.errors import CtxAgentHistoryCliError, CtxAgentHistoryProtocolError -from ctx_agent_history.errors import CtxAgentHistoryTimeoutError +from ctx_agent_history.errors import CtxAgentHistoryTimeoutError, CtxAgentHistoryValidationError from ctx_agent_history.types import AgentHistoryErrorCode import dogfood_local @@ -99,6 +99,20 @@ def test_init_sources_import_sync_search_and_inspect_methods(self) -> None: self.assertEqual(client.locate_session("session-1")["operation"], "locateSession") self.assertEqual(client.locateSession("session-1")["operation"], "locateSession") + def test_search_requires_query_term_or_file_before_cli(self) -> None: + with fake_ctx(fail=True) as cli: + client = AgentHistoryClient.local(ctx_binary=str(cli)) + + for call in ( + lambda: client.search(), + lambda: client.search(refresh="off", limit=5), + lambda: client.search(" "), + ): + with self.subTest(call=call): + with self.assertRaises(CtxAgentHistoryValidationError) as raised: + call() + self.assertEqual(raised.exception.code, "invalid_request") + def test_versioning_reports_sdk_api_transport_and_ctx_version(self) -> None: with fake_ctx() as cli: client = AgentHistoryClient.local(ctx_binary=str(cli)) diff --git a/sdks/swift/Sources/CtxAgentHistory/AgentHistoryClient.swift b/sdks/swift/Sources/CtxAgentHistory/AgentHistoryClient.swift index 7fe272024..1d76caf37 100644 --- a/sdks/swift/Sources/CtxAgentHistory/AgentHistoryClient.swift +++ b/sdks/swift/Sources/CtxAgentHistory/AgentHistoryClient.swift @@ -70,6 +70,7 @@ public struct AgentHistoryClient: Sendable { } public func search(_ query: String? = nil, options: SearchOptions = SearchOptions()) throws -> SearchResponse { + try requireSearchIntent(query: query, options: options) var arguments = ["search"] if let query { arguments.append(query) @@ -293,6 +294,23 @@ private func appendOption(_ arguments: inout [String], _ name: String, _ value: } } +private func requireSearchIntent(query: String?, options: SearchOptions) throws { + if hasSearchText(query) || hasSearchText(options.file) || options.terms.contains(where: { hasSearchText($0) }) { + return + } + throw CtxAgentHistorySDKError( + code: .invalidRequest, + message: "search requires a query, term, or file option" + ) +} + +private func hasSearchText(_ value: String?) -> Bool { + guard let value else { + return false + } + return !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty +} + private func requireID(_ name: String, _ id: String) throws { if id.isEmpty { throw CtxAgentHistorySDKError(code: .invalidRequest, message: "\(name) is required") diff --git a/sdks/swift/Tests/CtxAgentHistoryTests/CtxAgentHistoryTests.swift b/sdks/swift/Tests/CtxAgentHistoryTests/CtxAgentHistoryTests.swift index 20f965219..7af773cab 100644 --- a/sdks/swift/Tests/CtxAgentHistoryTests/CtxAgentHistoryTests.swift +++ b/sdks/swift/Tests/CtxAgentHistoryTests/CtxAgentHistoryTests.swift @@ -188,6 +188,12 @@ final class CtxAgentHistoryTests: XCTestCase { XCTAssertThrowsError(try parse.showSession(ShowSessionOptions(provider: "codex"))) { error in XCTAssertEqual((error as? CtxAgentHistorySDKError)?.code, .invalidRequest) } + XCTAssertThrowsError(try parse.search(options: SearchOptions(refresh: "off"))) { error in + XCTAssertEqual((error as? CtxAgentHistorySDKError)?.code, .invalidRequest) + } + XCTAssertThrowsError(try parse.search(" ")) { error in + XCTAssertEqual((error as? CtxAgentHistorySDKError)?.code, .invalidRequest) + } } func testAllStructuredErrorCodesRoundTripThroughContractError() throws { diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index de205a15f..b23361219 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -22,7 +22,7 @@ const results = await client.search("sqlite storage", { refresh: "off" }); - `import(options)` wraps `ctx import --json`. - `sync(options)` is an alias for `import(options)`. - `search(query, options)` and file/term-based `search(options)` wrap - `ctx search --json`. + `ctx search |--term |--file --json`. - `showEvent(id, { before, after, window })` wraps `ctx show event --format json`. - `showSession(id, { mode })` wraps `ctx show session --format json`. - `showSession({ provider, providerSession, mode })` looks up by provider-owned session ID. diff --git a/sdks/typescript/src/index.d.ts b/sdks/typescript/src/index.d.ts index 68e9a22b6..5c2b2f3de 100644 --- a/sdks/typescript/src/index.d.ts +++ b/sdks/typescript/src/index.d.ts @@ -91,6 +91,13 @@ export interface SearchOptions { includeCurrentSession?: boolean; } +export type SearchIntentOptions = SearchOptions & ( + | { query: string } + | { term: string | string[] } + | { terms: [string, ...string[]] } + | { file: string } +); + export interface ShowEventOptions { before?: number; after?: number; @@ -408,8 +415,8 @@ export declare class LocalAgentHistoryClient { sources(): Promise; import(options?: ImportOptions): Promise>; sync(options?: ImportOptions): Promise>; - search(query?: string, options?: SearchOptions): Promise; - search(options?: SearchOptions): Promise; + search(query: string, options?: Omit): Promise; + search(options: SearchIntentOptions): Promise; showEvent(id: string, options?: ShowEventOptions): Promise; showSession(id: string, options?: Omit): Promise; showSession(options: ShowSessionOptions): Promise; diff --git a/sdks/typescript/src/index.js b/sdks/typescript/src/index.js index fcdafd86c..8eeca89ea 100644 --- a/sdks/typescript/src/index.js +++ b/sdks/typescript/src/index.js @@ -163,6 +163,7 @@ export class LocalAgentHistoryClient { typeof queryOrOptions === "string" ? { ...maybeOptions, query: queryOrOptions } : { ...queryOrOptions }; + validateSearchIntent(options); const args = ["search"]; if (options.query) { args.push(options.query); @@ -431,6 +432,27 @@ function appendSearchArgs(args, options) { appendFlag(args, "--include-current-session", options.includeCurrentSession); } +function validateSearchIntent(options) { + if (hasSearchText(options.query) || hasSearchText(options.file) || hasSearchTerm(options)) { + return; + } + throw new CtxValidationError("search requires a query, term, or file option", { + details: { options }, + }); +} + +function hasSearchTerm(options) { + const value = options.terms ?? options.term; + if (Array.isArray(value)) { + return value.some(hasSearchText); + } + return hasSearchText(value); +} + +function hasSearchText(value) { + return typeof value === "string" && value.trim().length > 0; +} + function appendSessionLookupArgs(args, options) { if (options.id) { args.push(options.id); diff --git a/sdks/typescript/test/client.test.js b/sdks/typescript/test/client.test.js index b7df0ede4..badfbb96e 100644 --- a/sdks/typescript/test/client.test.js +++ b/sdks/typescript/test/client.test.js @@ -178,6 +178,18 @@ test("builds search flags and normalizes nested CLI search output", async () => ]); }); +test("rejects search without query, term, or file before invoking CLI", async () => { + const { client, calls } = mockClient(() => { + throw new Error("runner should not be called"); + }); + + await assert.rejects(() => client.search(), CtxValidationError); + await assert.rejects(() => client.search({ refresh: "off", limit: 5 }), CtxValidationError); + await assert.rejects(() => client.search(" "), CtxValidationError); + + assert.equal(calls.length, 0); +}); + test("wraps show and locate commands by ctx id and provider session id", async () => { const { client, calls } = mockClient(() => "{}"); diff --git a/sdks/typescript/test/types.test.ts b/sdks/typescript/test/types.test.ts index 20eaa1b3f..f7166b749 100644 --- a/sdks/typescript/test/types.test.ts +++ b/sdks/typescript/test/types.test.ts @@ -44,6 +44,15 @@ expectType(search.search.results[0]!.ctxEventId); // @ts-expect-error search results expose ctxEventId, not ctx_event_id. search.search.results[0]!.ctx_event_id; +const termSearch = await client.search({ terms: ["local agent history"], refresh: "off" }); +expectType(termSearch); +const fileSearch = await client.search({ file: "src/lib.rs", refresh: "off" }); +expectType(fileSearch); +// @ts-expect-error search requires a query, term, or file option. +await client.search(); +// @ts-expect-error search filters alone are not a search intent. +await client.search({ refresh: "off", limit: 5 }); + const shown = await client.showEvent("11111111-1111-4111-8111-111111111111"); expectType(shown); expectType(shown.event.events[0]!.ctxSessionId); diff --git a/skills/ctx-agent-history-search/SKILL.md b/skills/ctx-agent-history-search/SKILL.md index f1e99b091..364ecd52f 100644 --- a/skills/ctx-agent-history-search/SKILL.md +++ b/skills/ctx-agent-history-search/SKILL.md @@ -63,9 +63,9 @@ Use this skill in two modes: When the prompt asks for a topic history or report across multiple sessions, run several `ctx search` queries with different wording and filters to find - promising sessions. Use scoped `ctx search ... --session ` - when a session looks relevant and you need dense event-level matches from - that session. + promising sessions. Use scoped + `ctx search "" --session ` when a session looks + relevant and you need dense event-level matches from that session. Default search returns primary-agent sessions so human intent and decisions stay prominent. Use `--include-subagents` when implementation details, code @@ -144,7 +144,7 @@ material. chronology, alternatives, or detailed evidence. 2. Run several targeted searches. Vary query terms across user wording, file or module names, error text, commands, branch names, and decision terms. Start - with default `ctx search`, then broaden with `--term` or narrow with + with `ctx search ""`, then broaden with `--term` or narrow with `--workspace`, `--provider`, `--file`, `--since`, or `--session `. Use `--include-subagents` when reviews, implementation attempts, test output, From 2630d13e5199ebc912c66564fae5c2611cafc37e Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 23:18:42 -0500 Subject: [PATCH 45/72] Scope provider provenance by source (#36) Co-authored-by: luca-ctx <216224554+luca-ctx@users.noreply.github.com> --- crates/ctx-history-capture/src/lib.rs | 852 +++++++++++++++++++++++--- crates/ctx-history-store/src/lib.rs | 229 +++++-- 2 files changed, 946 insertions(+), 135 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index e4fa91adc..22761fff3 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -797,6 +797,8 @@ pub struct ProviderFileTouchedEnvelope { pub provider_touch_index: u64, #[serde(default, skip_serializing_if = "Option::is_none")] pub provider_event_index: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw_source_path: Option, pub path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub change_kind: Option, @@ -1134,6 +1136,10 @@ impl ProviderCaptureAdapter for CodexSessionJsonlAdapter { let mut result = ProviderNormalizationResult::default(); let mut header = None; let mut call_contexts: BTreeMap = BTreeMap::new(); + let raw_source_path = context + .source_path + .as_ref() + .map(|path| path.display().to_string()); let mut line_number = 0usize; let mut line = Vec::new(); @@ -1212,11 +1218,14 @@ impl ProviderCaptureAdapter for CodexSessionJsonlAdapter { let mut line_capture = codex_session_line_capture( header, &value, - line_number, - occurred_at, &mut call_contexts, - context.tool_output_mode, - context.event_mode, + CodexSessionLineContext { + line_number, + occurred_at, + tool_output_mode: context.tool_output_mode, + event_mode: context.event_mode, + raw_source_path: raw_source_path.as_deref(), + }, ); if let Some(event) = line_capture.event.take() { if !context.include_notices && event.event_type == EventType::Notice { @@ -2081,6 +2090,10 @@ pub fn import_codex_session_jsonl_tail( wrap_transaction: false, fast_event_inserts: true, }; + let raw_source_path = context + .source_path + .as_ref() + .map(|path| path.display().to_string()); report_codex_import_progress( &options, @@ -2182,11 +2195,14 @@ pub fn import_codex_session_jsonl_tail( let mut line_capture = codex_session_line_capture( &header, &value, - line_number, - occurred_at, &mut call_contexts, - options.tool_output_mode, - options.event_mode, + CodexSessionLineContext { + line_number, + occurred_at, + tool_output_mode: options.tool_output_mode, + event_mode: options.event_mode, + raw_source_path: raw_source_path.as_deref(), + }, ); if let Some(event) = line_capture.event.take() { if !options.include_notices && event.event_type == EventType::Notice { @@ -2200,6 +2216,7 @@ pub fn import_codex_session_jsonl_tail( options.history_record_id, line_number, context.imported_at, + raw_source_path.as_deref(), )?); } } @@ -2624,6 +2641,10 @@ fn import_codex_session_path_fast( wrap_transaction: false, fast_event_inserts: true, }; + let raw_source_path = context + .source_path + .as_ref() + .map(|path| path.display().to_string()); let mut header = None; let mut call_contexts: BTreeMap = BTreeMap::new(); @@ -2720,11 +2741,14 @@ fn import_codex_session_path_fast( let mut line_capture = codex_session_line_capture( header, &value, - line_number, - occurred_at, &mut call_contexts, - options.tool_output_mode, - options.event_mode, + CodexSessionLineContext { + line_number, + occurred_at, + tool_output_mode: options.tool_output_mode, + event_mode: options.event_mode, + raw_source_path: raw_source_path.as_deref(), + }, ); if let Some(event) = line_capture.event.take() { if !options.include_notices && event.event_type == EventType::Notice { @@ -2738,6 +2762,7 @@ fn import_codex_session_path_fast( options.history_record_id, line_number, context.imported_at, + raw_source_path.as_deref(), )?; summary.merge(line_summary); } @@ -2756,36 +2781,45 @@ fn import_codex_provider_event_fast( history_record_id: Option, line_number: usize, imported_at: DateTime, + raw_source_path: Option<&str>, ) -> Result { let mut summary = ProviderImportSummary::default(); let provider = CaptureProvider::Codex; let session_id = provider_session_uuid(provider, &header.id); - let source_id = provider_source_uuid(provider, &header.id); + let source_id = provider_scoped_source_uuid( + provider, + &header.id, + CODEX_SESSION_SOURCE_FORMAT, + raw_source_path, + ); let (payload, redacted_payload) = sanitize_value(event.payload.clone()); let (event_metadata, redacted_metadata) = sanitize_value(event.metadata.clone()); let event_hash = event .provider_event_hash .clone() .unwrap_or(compute_payload_hash(&payload)?); - let dedupe_key = Store::provider_event_dedupe_key( + let event_identity = provider_event_import_identity( + store, provider, &header.id, + source_id, event.provider_event_index, &event_hash, - ); + )?; let command_run = provider_command_run_from_event(ProviderCommandRunInput { provider, provider_session_id: &header.id, session_id, source_id, + run_source_id: event_identity.run_source_id, history_record_id, event, payload: &payload, event_hash: &event_hash, }); let normalized_event = Event { - id: provider_event_uuid(provider, &header.id, event.provider_event_index), - seq: provider_event_seq(provider, &header.id, event.provider_event_index), + id: event_identity.id, + seq: event_identity.seq, history_record_id, session_id: Some(session_id), run_id: command_run.as_ref().map(|run| run.id), @@ -2803,7 +2837,7 @@ fn import_codex_provider_event_fast( "body": payload, }), payload_blob_id: None, - dedupe_key: Some(dedupe_key), + dedupe_key: Some(event_identity.dedupe_key), redaction_state: effective_event_redaction_state( event.redaction_state, redacted_payload || redacted_metadata, @@ -3684,6 +3718,8 @@ struct CustomHistoryJsonlV1SourceCursorImport { struct CustomHistoryJsonlV1EdgeImport { provider_key: String, source_id: String, + source_format: String, + raw_source_path: Option, from_provider_session_id: String, to_provider_session_id: String, edge_id: Option, @@ -3965,7 +4001,7 @@ fn normalize_custom_history_jsonl_v1_reader( .1; result.files_touched.push(( line_number, - custom_history_file_touch_envelope(source, &file_touch), + custom_history_file_touch_envelope(source, &file_touch, context), )); } @@ -3977,7 +4013,7 @@ fn normalize_custom_history_jsonl_v1_reader( .1; custom_edges.push(( line_number, - custom_history_edge_import(source, &edge, context.imported_at), + custom_history_edge_import(source, &edge, context), )); } @@ -4237,12 +4273,7 @@ fn custom_history_session_capture( .clone() .unwrap_or_else(|| context.machine_id.clone()), observed_at: source.observed_at.unwrap_or(context.imported_at), - raw_source_path: source.raw_source_path.clone().or_else(|| { - context - .source_path - .as_ref() - .map(|path| path.display().to_string()) - }), + raw_source_path: custom_history_effective_raw_source_path(source, context), raw_retention: source.raw_retention, redaction_boundary: source.redaction_boundary, trust: match source.trust { @@ -4353,6 +4384,7 @@ fn custom_history_event_envelope( fn custom_history_file_touch_envelope( source: &CtxHistoryJsonlSourceRecord, file_touch: &CtxHistoryJsonlFileTouchRecord, + context: &ProviderAdapterContext, ) -> ProviderFileTouchedEnvelope { ProviderFileTouchedEnvelope { provider: CaptureProvider::Custom, @@ -4363,6 +4395,7 @@ fn custom_history_file_touch_envelope( ), provider_touch_index: file_touch.touch_index, provider_event_index: file_touch.event_index, + raw_source_path: custom_history_effective_raw_source_path(source, context), path: file_touch.path.clone(), change_kind: file_touch.change_kind, old_path: file_touch.old_path.clone(), @@ -4384,11 +4417,13 @@ fn custom_history_file_touch_envelope( fn custom_history_edge_import( source: &CtxHistoryJsonlSourceRecord, edge: &CtxHistoryJsonlEdgeRecord, - imported_at: DateTime, + context: &ProviderAdapterContext, ) -> CustomHistoryJsonlV1EdgeImport { CustomHistoryJsonlV1EdgeImport { provider_key: source.provider_key.clone(), source_id: source.source_id.clone(), + source_format: source.source_format.clone(), + raw_source_path: custom_history_effective_raw_source_path(source, context), from_provider_session_id: custom_history_internal_session_id( &source.provider_key, &source.source_id, @@ -4402,7 +4437,7 @@ fn custom_history_edge_import( edge_id: edge.edge_id.clone(), edge_type: edge.edge_type, confidence: edge.confidence, - occurred_at: edge.occurred_at.unwrap_or(imported_at), + occurred_at: edge.occurred_at.unwrap_or(context.imported_at), fidelity: edge.fidelity, metadata: custom_history_metadata( edge.metadata.clone(), @@ -4417,6 +4452,18 @@ fn custom_history_edge_import( } } +fn custom_history_effective_raw_source_path( + source: &CtxHistoryJsonlSourceRecord, + context: &ProviderAdapterContext, +) -> Option { + source.raw_source_path.clone().or_else(|| { + context + .source_path + .as_ref() + .map(|path| path.display().to_string()) + }) +} + fn custom_history_internal_session_id( provider_key: &str, source_id: &str, @@ -4538,7 +4585,12 @@ fn import_custom_history_edges( provider_session_uuid(CaptureProvider::Custom, &edge.from_provider_session_id); let to_session_id = provider_session_uuid(CaptureProvider::Custom, &edge.to_provider_session_id); - let source_id = provider_source_uuid(CaptureProvider::Custom, &edge.to_provider_session_id); + let source_id = provider_scoped_source_uuid( + CaptureProvider::Custom, + &edge.to_provider_session_id, + &edge.source_format, + edge.raw_source_path.as_deref(), + ); let mut exists_cache = BTreeMap::::new(); if !provider_session_exists_cached(store, from_session_id, &mut exists_cache)? || !provider_session_exists_cached(store, to_session_id, &mut exists_cache)? @@ -4833,15 +4885,27 @@ fn codex_session_capture( } } -fn codex_session_line_capture( - header: &CodexSessionHeader, - value: &Value, +struct CodexSessionLineContext<'a> { line_number: usize, occurred_at: DateTime, - call_contexts: &mut BTreeMap, tool_output_mode: CodexToolOutputMode, event_mode: CodexEventImportMode, + raw_source_path: Option<&'a str>, +} + +fn codex_session_line_capture( + header: &CodexSessionHeader, + value: &Value, + call_contexts: &mut BTreeMap, + context: CodexSessionLineContext<'_>, ) -> CodexSessionLineCapture { + let CodexSessionLineContext { + line_number, + occurred_at, + tool_output_mode, + event_mode, + raw_source_path, + } = context; let event = codex_session_event( value, line_number, @@ -4865,6 +4929,7 @@ fn codex_session_line_capture( provider: CaptureProvider::Codex, provider_session_id: &header.id, source_format: CODEX_SESSION_SOURCE_FORMAT, + raw_source_path, occurred_at, provider_event_index: event.as_ref().map(|event| event.provider_event_index), provider_touch_base_index: (line_number as u64) << 16, @@ -5538,6 +5603,7 @@ fn provider_file_touches_from_event( provider: CaptureProvider, provider_session_id: &str, source_format: &str, + raw_source_path: Option<&str>, event: &ProviderEventEnvelope, line_number: usize, ) -> Vec<(usize, ProviderFileTouchedEnvelope)> { @@ -5562,6 +5628,7 @@ fn provider_file_touches_from_event( provider, provider_session_id, source_format, + raw_source_path, occurred_at: event.occurred_at, provider_event_index: Some(event.provider_event_index), provider_touch_base_index: event.provider_event_index << 16, @@ -5575,6 +5642,7 @@ fn provider_file_touches_from_raw_value( provider: CaptureProvider, provider_session_id: &str, source_format: &str, + raw_source_path: Option<&str>, raw_value: &Value, event: &ProviderEventEnvelope, line_number: usize, @@ -5600,6 +5668,7 @@ fn provider_file_touches_from_raw_value( provider, provider_session_id, source_format, + raw_source_path, occurred_at: event.occurred_at, provider_event_index: Some(event.provider_event_index), provider_touch_base_index: event.provider_event_index << 16, @@ -5617,6 +5686,7 @@ struct ProviderFileTouchEnvelopeContext<'a> { provider: CaptureProvider, provider_session_id: &'a str, source_format: &'a str, + raw_source_path: Option<&'a str>, occurred_at: DateTime, provider_event_index: Option, provider_touch_base_index: u64, @@ -5646,6 +5716,7 @@ fn provider_file_touch_envelopes( provider_session_id: context.provider_session_id.to_owned(), provider_touch_index, provider_event_index: context.provider_event_index, + raw_source_path: context.raw_source_path.map(str::to_owned), path: draft.path, change_kind: draft.change_kind, old_path: draft.old_path, @@ -6069,6 +6140,7 @@ fn normalize_claude_projects_jsonl_file( .get("gitBranch") .and_then(Value::as_str) .map(str::to_owned); + let raw_source_path = path.display().to_string(); for (line_number, value, occurred_at) in rows { let event = claude_event(&value, line_number, occurred_at); @@ -6079,6 +6151,7 @@ fn normalize_claude_projects_jsonl_file( CaptureProvider::Claude, &provider_session_id, CLAUDE_PROJECTS_SOURCE_FORMAT, + Some(raw_source_path.as_str()), &value, event, line_number, @@ -6093,7 +6166,7 @@ fn normalize_claude_projects_jsonl_file( source_format: CLAUDE_PROJECTS_SOURCE_FORMAT.to_owned(), machine_id: context.machine_id.clone(), observed_at: context.imported_at, - raw_source_path: Some(path.display().to_string()), + raw_source_path: Some(raw_source_path.clone()), raw_retention: ProviderRawRetention::PathReference, redaction_boundary: ProviderRedactionBoundary::BeforeExport, trust: ProviderSourceTrust::ProviderNative, @@ -6115,7 +6188,7 @@ fn normalize_claude_projects_jsonl_file( metadata: json!({ "adapter": CLAUDE_PROJECTS_SOURCE_FORMAT, "native_session_id": native_session_id, - "source_path": path.display().to_string(), + "source_path": raw_source_path.clone(), }), }, session: ProviderSessionEnvelope { @@ -8222,6 +8295,7 @@ fn normalize_opencode_sqlite( .into_iter() .map(|session| (session.id.clone(), session)) .collect::>(); + let raw_source_path = path.display().to_string(); for row in messages { let Some(session) = sessions_by_id.get(&row.session_id) else { @@ -8260,6 +8334,7 @@ fn normalize_opencode_sqlite( CaptureProvider::OpenCode, &session.id, OPENCODE_SQLITE_SOURCE_FORMAT, + Some(raw_source_path.as_str()), &data, &event, row.seq.max(0) as usize, @@ -8274,7 +8349,7 @@ fn normalize_opencode_sqlite( source_format: OPENCODE_SQLITE_SOURCE_FORMAT.to_owned(), machine_id: context.machine_id.clone(), observed_at: context.imported_at, - raw_source_path: Some(path.display().to_string()), + raw_source_path: Some(raw_source_path.clone()), raw_retention: ProviderRawRetention::PathReference, redaction_boundary: ProviderRedactionBoundary::BeforeExport, trust: ProviderSourceTrust::ProviderNative, @@ -8925,6 +9000,7 @@ fn normalize_native_jsonl_session_file( .unwrap_or(context.imported_at); let cwd = native_jsonl_header_cwd(provider, &header); let is_subagent = parent_provider_session_id.is_some() || agent_type == AgentType::Subagent; + let raw_source_path = path.display().to_string(); for (line_number, value) in rows { let occurred_at = native_jsonl_timestamp(&value).unwrap_or(started_at); @@ -8936,6 +9012,7 @@ fn normalize_native_jsonl_session_file( provider, &provider_session_id, source_format, + Some(raw_source_path.as_str()), &value, event, line_number, @@ -8950,7 +9027,7 @@ fn normalize_native_jsonl_session_file( source_format: source_format.to_owned(), machine_id: context.machine_id.clone(), observed_at: context.imported_at, - raw_source_path: Some(path.display().to_string()), + raw_source_path: Some(raw_source_path.clone()), raw_retention: ProviderRawRetention::PathReference, redaction_boundary: ProviderRedactionBoundary::BeforeExport, trust: ProviderSourceTrust::ProviderNative, @@ -8970,7 +9047,7 @@ fn normalize_native_jsonl_session_file( metadata: json!({ "adapter": source_format, "native_session_id": native_session_id, - "source_path": path.display().to_string(), + "source_path": raw_source_path.clone(), }), }, session: ProviderSessionEnvelope { @@ -9777,6 +9854,7 @@ fn import_provider_capture_lines( capture.provider, &capture.session.provider_session_id, &capture.source.source_format, + capture.source.raw_source_path.as_deref(), event, *line_number, )); @@ -9840,16 +9918,31 @@ fn import_provider_file_touched_line( options: &NormalizedProviderImportOptions, ) -> Result<()> { let session_id = provider_session_uuid(file.provider, &file.provider_session_id); - let source_id = provider_source_uuid(file.provider, &file.provider_session_id); - let event_id = file - .provider_event_index - .map(|index| provider_event_uuid(file.provider, &file.provider_session_id, index)); - let touched = FileTouched { - id: provider_file_touch_uuid( + let source_id = provider_scoped_source_uuid( + file.provider, + &file.provider_session_id, + &file.source_format, + file.raw_source_path.as_deref(), + ); + let event_id = match file.provider_event_index { + Some(index) => provider_file_touch_event_id( + store, file.provider, &file.provider_session_id, - file.provider_touch_index, - ), + source_id, + index, + )?, + None => None, + }; + let touch_id = provider_file_touch_import_id( + store, + file.provider, + &file.provider_session_id, + source_id, + file.provider_touch_index, + )?; + let touched = FileTouched { + id: touch_id, history_record_id: options.history_record_id, run_id: None, event_id, @@ -9868,6 +9961,8 @@ fn import_provider_file_touched_line( "provider_session_id": file.provider_session_id, "provider_touch_index": file.provider_touch_index, "provider_event_index": file.provider_event_index, + "raw_source_path": file.raw_source_path, + "source_id": source_id, "source_format": file.source_format, "metadata": file.metadata, "session_id": session_id, @@ -9923,7 +10018,13 @@ fn import_provider_capture_line( let source = &capture.source; let imported_at = source.observed_at; let session_id = provider_session_uuid(provider, &session.provider_session_id); - let source_id = provider_source_uuid(provider, &session.provider_session_id); + let source_identity_key = provider_scoped_source_identity_key( + provider, + &session.provider_session_id, + &source.source_format, + source.raw_source_path.as_deref(), + ); + let source_id = stable_capture_uuid(&source_identity_key, "source"); let requested_parent_session_id = session .parent_provider_session_id .as_ref() @@ -9978,6 +10079,7 @@ fn import_provider_capture_line( "fixture_line": line_number, "imported_at": imported_at, "source_idempotency_key": source.idempotency_key, + "source_identity_key": source_identity_key, "source_metadata": source_metadata, "session_metadata": session_metadata, }), @@ -10104,33 +10206,28 @@ fn import_provider_capture_line( .provider_event_hash .clone() .unwrap_or(compute_payload_hash(&payload)?); - let dedupe_key = Store::provider_event_dedupe_key( + let event_identity = provider_event_import_identity( + store, provider, &session.provider_session_id, + source_id, event.provider_event_index, &event_hash, - ); + )?; let command_run = provider_command_run_from_event(ProviderCommandRunInput { provider, provider_session_id: &session.provider_session_id, session_id, source_id, + run_source_id: event_identity.run_source_id, history_record_id: options.history_record_id, event, payload: &payload, event_hash: &event_hash, }); let normalized_event = Event { - id: provider_event_uuid( - provider, - &session.provider_session_id, - event.provider_event_index, - ), - seq: provider_event_seq( - provider, - &session.provider_session_id, - event.provider_event_index, - ), + id: event_identity.id, + seq: event_identity.seq, history_record_id: options.history_record_id, session_id: Some(session_id), run_id: command_run.as_ref().map(|run| run.id), @@ -10148,7 +10245,7 @@ fn import_provider_capture_line( "body": payload, }), payload_blob_id: None, - dedupe_key: Some(dedupe_key.clone()), + dedupe_key: Some(event_identity.dedupe_key.clone()), redaction_state: effective_event_redaction_state( event.redaction_state, redacted_payload || redacted_metadata, @@ -10175,7 +10272,7 @@ fn import_provider_capture_line( } !store.insert_event_if_absent(&normalized_event)? } else { - let was_present = provider_event_exists(store, &dedupe_key)?; + let was_present = provider_event_exists(store, &event_identity.dedupe_key)?; if let Some(run) = &command_run { store.upsert_run(run)?; } @@ -10781,6 +10878,130 @@ fn provider_event_exists(store: &Store, dedupe_key: &str) -> Result { } } +#[derive(Clone)] +struct ProviderEventImportIdentity { + id: Uuid, + seq: u64, + dedupe_key: String, + run_source_id: Option, +} + +fn provider_event_import_identity( + store: &Store, + provider: CaptureProvider, + provider_session_id: &str, + source_id: Uuid, + provider_event_index: u64, + event_hash: &str, +) -> Result { + let source_identity = + provider_source_event_import_identity(source_id, provider_event_index, event_hash); + if provider_event_exists(store, &source_identity.dedupe_key)? + || provider_event_id_exists(store, source_identity.id)? + { + return Ok(source_identity); + } + + let legacy_identity = provider_legacy_event_import_identity( + provider, + provider_session_id, + provider_event_index, + event_hash, + ); + if provider_event_exists(store, &legacy_identity.dedupe_key)? + || provider_event_id_exists(store, legacy_identity.id)? + { + Ok(legacy_identity) + } else { + Ok(source_identity) + } +} + +fn provider_source_event_import_identity( + source_id: Uuid, + provider_event_index: u64, + event_hash: &str, +) -> ProviderEventImportIdentity { + ProviderEventImportIdentity { + id: provider_source_event_uuid(source_id, provider_event_index), + seq: provider_source_event_seq(source_id, provider_event_index), + dedupe_key: Store::provider_source_event_dedupe_key( + source_id, + provider_event_index, + event_hash, + ), + run_source_id: Some(source_id), + } +} + +fn provider_legacy_event_import_identity( + provider: CaptureProvider, + provider_session_id: &str, + provider_event_index: u64, + event_hash: &str, +) -> ProviderEventImportIdentity { + ProviderEventImportIdentity { + id: provider_event_uuid(provider, provider_session_id, provider_event_index), + seq: provider_event_seq(provider, provider_session_id, provider_event_index), + dedupe_key: Store::provider_event_dedupe_key( + provider, + provider_session_id, + provider_event_index, + event_hash, + ), + run_source_id: None, + } +} + +fn provider_file_touch_event_id( + store: &Store, + provider: CaptureProvider, + provider_session_id: &str, + source_id: Uuid, + provider_event_index: u64, +) -> Result> { + let source_event_id = provider_source_event_uuid(source_id, provider_event_index); + if provider_event_id_exists(store, source_event_id)? { + return Ok(Some(source_event_id)); + } + + let legacy_event_id = provider_event_uuid(provider, provider_session_id, provider_event_index); + if provider_event_id_exists(store, legacy_event_id)? { + Ok(Some(legacy_event_id)) + } else { + Ok(None) + } +} + +fn provider_file_touch_import_id( + store: &Store, + provider: CaptureProvider, + provider_session_id: &str, + source_id: Uuid, + provider_touch_index: u64, +) -> Result { + let source_touch_id = provider_source_file_touch_uuid(source_id, provider_touch_index); + if store.file_touched_exists(source_touch_id)? { + return Ok(source_touch_id); + } + + let legacy_touch_id = + provider_file_touch_uuid(provider, provider_session_id, provider_touch_index); + if store.file_touched_exists(legacy_touch_id)? { + Ok(legacy_touch_id) + } else { + Ok(source_touch_id) + } +} + +fn provider_event_id_exists(store: &Store, id: Uuid) -> Result { + match store.get_event(id) { + Ok(_) => Ok(true), + Err(StoreError::NotFound(_)) => Ok(false), + Err(err) => Err(CaptureError::Store(err)), + } +} + fn provider_session_exists(store: &Store, session_id: Uuid) -> Result { match store.get_session(session_id) { Ok(_) => Ok(true), @@ -10807,6 +11028,7 @@ struct ProviderCommandRunInput<'a> { provider_session_id: &'a str, session_id: Uuid, source_id: Uuid, + run_source_id: Option, history_record_id: Option, event: &'a ProviderEventEnvelope, payload: &'a Value, @@ -10819,6 +11041,7 @@ fn provider_command_run_from_event(input: ProviderCommandRunInput<'_>) -> Option provider_session_id, session_id, source_id, + run_source_id, history_record_id, event, payload, @@ -10844,7 +11067,9 @@ fn provider_command_run_from_event(input: ProviderCommandRunInput<'_>) -> Option }) .unwrap_or(event.occurred_at); Some(Run { - id: provider_run_uuid(provider, provider_session_id, key), + id: run_source_id + .map(|source_id| provider_source_run_uuid(source_id, key)) + .unwrap_or_else(|| provider_run_uuid(provider, provider_session_id, key)), history_record_id, session_id: Some(session_id), run_type: RunType::Command, @@ -10889,6 +11114,7 @@ fn provider_command_run_status(payload: &Value) -> RunStatus { } } +#[cfg(test)] fn provider_source_uuid(provider: CaptureProvider, provider_session_id: &str) -> Uuid { stable_capture_uuid( &format!("provider:{}:{provider_session_id}", provider.as_str()), @@ -10896,6 +11122,39 @@ fn provider_source_uuid(provider: CaptureProvider, provider_session_id: &str) -> ) } +fn provider_scoped_source_uuid( + provider: CaptureProvider, + provider_session_id: &str, + source_format: &str, + raw_source_path: Option<&str>, +) -> Uuid { + stable_capture_uuid( + &provider_scoped_source_identity_key( + provider, + provider_session_id, + source_format, + raw_source_path, + ), + "source", + ) +} + +fn provider_scoped_source_identity_key( + provider: CaptureProvider, + provider_session_id: &str, + source_format: &str, + raw_source_path: Option<&str>, +) -> String { + serde_json::to_string(&( + "provider-source-v2", + provider.as_str(), + provider_session_id, + source_format, + raw_source_path, + )) + .expect("provider source identity key should serialize") +} + fn provider_session_uuid(provider: CaptureProvider, provider_session_id: &str) -> Uuid { stable_capture_uuid( &format!("provider:{}:{provider_session_id}", provider.as_str()), @@ -10913,6 +11172,10 @@ fn provider_run_uuid(provider: CaptureProvider, provider_session_id: &str, run_k ) } +fn provider_source_run_uuid(source_id: Uuid, run_key: &str) -> Uuid { + stable_capture_uuid(&format!("provider-source:{source_id}:run:{run_key}"), "run") +} + fn provider_event_uuid( provider: CaptureProvider, provider_session_id: &str, @@ -10927,6 +11190,23 @@ fn provider_event_uuid( ) } +fn provider_event_seq( + provider: CaptureProvider, + provider_session_id: &str, + provider_event_index: u64, +) -> u64 { + let session_key = format!("provider:{}:{provider_session_id}", provider.as_str()); + ((fnv1a64(session_key.as_bytes()) & 0x0000_07ff_ffff_ffff) << 20) + | (provider_event_index & 0x000f_ffff) +} + +fn provider_source_event_uuid(source_id: Uuid, provider_event_index: u64) -> Uuid { + stable_capture_uuid( + &format!("provider-source:{source_id}:event:{provider_event_index}"), + "event", + ) +} + fn provider_file_touch_uuid( provider: CaptureProvider, provider_session_id: &str, @@ -10941,14 +11221,17 @@ fn provider_file_touch_uuid( ) } -fn provider_event_seq( - provider: CaptureProvider, - provider_session_id: &str, - provider_event_index: u64, -) -> u64 { - let session_key = format!("provider:{}:{provider_session_id}", provider.as_str()); - ((fnv1a64(session_key.as_bytes()) & 0x0000_07ff_ffff_ffff) << 20) - | (provider_event_index & 0x000f_ffff) +fn provider_source_file_touch_uuid(source_id: Uuid, provider_touch_index: u64) -> Uuid { + stable_capture_uuid( + &format!("provider-source:{source_id}:file-touch:{provider_touch_index}"), + "file-touch", + ) +} + +fn provider_source_event_seq(source_id: Uuid, provider_event_index: u64) -> u64 { + let source_key = source_id.to_string(); + ((fnv1a64(source_key.as_bytes()) & 0x0000_0000_7fff_ffff) << 32) + | (provider_event_index & 0xffff_ffff) } fn provider_edge_uuid( @@ -12447,6 +12730,7 @@ mod tests { CaptureProvider::Antigravity, "agy-session", ANTIGRAVITY_CLI_SOURCE_FORMAT, + None, &antigravity, &event, 1, @@ -12455,6 +12739,7 @@ mod tests { CaptureProvider::Cursor, "cursor-session", CURSOR_AGENT_TRANSCRIPT_SOURCE_FORMAT, + None, &cursor, &event, 1, @@ -12559,6 +12844,7 @@ mod tests { provider, "provider-session", source_format, + None, &raw, &event, 1, @@ -13569,6 +13855,432 @@ mod tests { .is_some()); } + #[test] + fn provider_import_scopes_provenance_by_source_format_and_path() { + let temp = tempdir(); + let shared_path = temp + .path() + .join("shared-source.jsonl") + .display() + .to_string(); + assert_provider_source_collision_is_distinct( + "provider_format_a", + &shared_path, + "provider_format_b", + &shared_path, + ); + + let first_path = temp.path().join("first-source.jsonl").display().to_string(); + let second_path = temp + .path() + .join("second-source.jsonl") + .display() + .to_string(); + assert_provider_source_collision_is_distinct( + "provider_format", + &first_path, + "provider_format", + &second_path, + ); + } + + #[test] + fn provider_import_reuses_existing_legacy_provider_event_identity() { + let temp = tempdir(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let provider = CaptureProvider::Claude; + let provider_session_id = "legacy-provider-session"; + let source_format = "provider_format"; + let raw_source_path = temp + .path() + .join("legacy-source.jsonl") + .display() + .to_string(); + let occurred_at = DateTime::parse_from_rfc3339("2026-06-23T17:00:01Z") + .unwrap() + .with_timezone(&Utc); + let legacy_source_id = provider_source_uuid(provider, provider_session_id); + let new_source_id = provider_scoped_source_uuid( + provider, + provider_session_id, + source_format, + Some(&raw_source_path), + ); + let session_id = provider_session_uuid(provider, provider_session_id); + let legacy_event_id = provider_event_uuid(provider, provider_session_id, 0); + let legacy_touch_id = provider_file_touch_uuid(provider, provider_session_id, 0); + let event_hash = + compute_payload_hash(&json!({"text": "same provider event payload"})).unwrap(); + assert_ne!(legacy_source_id, new_source_id); + + store + .upsert_capture_source(&CaptureSource { + id: legacy_source_id, + descriptor: CaptureSourceDescriptor { + kind: CaptureSourceKind::ProviderImport, + provider, + machine_id: "test-machine".to_owned(), + process_id: None, + cwd: Some("/workspace/example".to_owned()), + raw_source_path: None, + external_session_id: Some(provider_session_id.to_owned()), + }, + started_at: occurred_at, + ended_at: None, + sync: provider_sync_metadata(Fidelity::Imported, json!({"legacy": true})), + }) + .unwrap(); + store + .upsert_session(&Session { + id: session_id, + history_record_id: None, + parent_session_id: None, + root_session_id: None, + capture_source_id: Some(legacy_source_id), + provider, + external_session_id: Some(provider_session_id.to_owned()), + external_agent_id: None, + agent_type: AgentType::Primary, + role_hint: Some("primary".to_owned()), + is_primary: true, + status: SessionStatus::Imported, + transcript_blob_id: None, + started_at: occurred_at, + ended_at: None, + timestamps: timestamps(occurred_at), + sync: provider_sync_metadata(Fidelity::Imported, json!({"legacy": true})), + }) + .unwrap(); + store + .upsert_event(&Event { + id: legacy_event_id, + seq: provider_event_seq(provider, provider_session_id, 0), + history_record_id: None, + session_id: Some(session_id), + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::User), + occurred_at, + capture_source_id: Some(legacy_source_id), + payload: json!({"body": {"text": "same provider event payload"}}), + payload_blob_id: None, + dedupe_key: Some(Store::provider_event_dedupe_key( + provider, + provider_session_id, + 0, + &event_hash, + )), + redaction_state: RedactionState::LocalPreview, + sync: provider_sync_metadata(Fidelity::Imported, json!({"legacy": true})), + }) + .unwrap(); + store + .upsert_file_touched(&FileTouched { + id: legacy_touch_id, + history_record_id: None, + run_id: None, + event_id: Some(legacy_event_id), + vcs_workspace_id: None, + path: "src/lib.rs".to_owned(), + change_kind: Some(FileChangeKind::Modified), + old_path: None, + line_count_delta: Some(1), + confidence: Confidence::Explicit, + timestamps: timestamps(occurred_at), + source_id: Some(legacy_source_id), + sync: provider_sync_metadata(Fidelity::Imported, json!({"legacy": true})), + }) + .unwrap(); + + let normalization = ProviderNormalizationResult { + summary: ProviderImportSummary::default(), + captures: vec![( + 1, + provider_collision_capture( + provider, + provider_session_id, + source_format, + &raw_source_path, + occurred_at, + ), + )], + files_touched: vec![( + 1, + provider_collision_file_touch( + provider, + provider_session_id, + source_format, + &raw_source_path, + occurred_at, + ), + )], + }; + + let summary = import_normalized_provider_captures( + &mut store, + normalization, + NormalizedProviderImportOptions::default(), + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.skipped_events, 1); + let events = store.events_for_session(session_id).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].id, legacy_event_id); + assert_eq!(events[0].capture_source_id, Some(legacy_source_id)); + + let archive = store.export_archive().unwrap(); + assert_eq!(archive.files_touched.len(), 1); + assert_eq!(archive.files_touched[0].id, legacy_touch_id); + assert_eq!(archive.files_touched[0].event_id, Some(legacy_event_id)); + assert_eq!(archive.files_touched[0].source_id, Some(new_source_id)); + } + + #[test] + fn provider_source_event_seq_keeps_large_provider_indices_distinct() { + let source_id = Uuid::parse_str("018fe2e4-2266-7000-8000-000000000001").unwrap(); + + assert_ne!( + provider_source_event_seq(source_id, 0), + provider_source_event_seq(source_id, 1_048_576) + ); + assert_eq!( + provider_source_event_seq(source_id, 1_048_576) & 0xffff_ffff, + 1_048_576 + ); + } + + fn assert_provider_source_collision_is_distinct( + first_source_format: &str, + first_source_path: &str, + second_source_format: &str, + second_source_path: &str, + ) { + let temp = tempdir(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let provider = CaptureProvider::Claude; + let provider_session_id = "shared-provider-session"; + let occurred_at = DateTime::parse_from_rfc3339("2026-06-23T17:00:01Z") + .unwrap() + .with_timezone(&Utc); + let first_source_id = provider_scoped_source_uuid( + provider, + provider_session_id, + first_source_format, + Some(first_source_path), + ); + let second_source_id = provider_scoped_source_uuid( + provider, + provider_session_id, + second_source_format, + Some(second_source_path), + ); + assert_ne!(first_source_id, second_source_id); + + let normalization = ProviderNormalizationResult { + summary: ProviderImportSummary::default(), + captures: vec![ + ( + 1, + provider_collision_capture( + provider, + provider_session_id, + first_source_format, + first_source_path, + occurred_at, + ), + ), + ( + 2, + provider_collision_capture( + provider, + provider_session_id, + second_source_format, + second_source_path, + occurred_at, + ), + ), + ], + files_touched: vec![ + ( + 1, + provider_collision_file_touch( + provider, + provider_session_id, + first_source_format, + first_source_path, + occurred_at, + ), + ), + ( + 2, + provider_collision_file_touch( + provider, + provider_session_id, + second_source_format, + second_source_path, + occurred_at, + ), + ), + ], + }; + + let summary = import_normalized_provider_captures( + &mut store, + normalization, + NormalizedProviderImportOptions::default(), + ) + .unwrap(); + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_events, 2); + assert_eq!(store.capture_source_count().unwrap(), 2); + + let first_source = store.get_capture_source(first_source_id).unwrap(); + let second_source = store.get_capture_source(second_source_id).unwrap(); + assert_eq!( + first_source.descriptor.raw_source_path.as_deref(), + Some(first_source_path) + ); + assert_eq!( + first_source.sync.metadata["source_format"].as_str(), + Some(first_source_format) + ); + assert_eq!( + second_source.descriptor.raw_source_path.as_deref(), + Some(second_source_path) + ); + assert_eq!( + second_source.sync.metadata["source_format"].as_str(), + Some(second_source_format) + ); + + let session_id = provider_session_uuid(provider, provider_session_id); + let event_source_ids = store + .events_for_session(session_id) + .unwrap() + .into_iter() + .map(|event| event.capture_source_id.unwrap()) + .collect::>(); + assert_eq!( + event_source_ids, + BTreeSet::from([first_source_id, second_source_id]) + ); + + let archive = store.export_archive().unwrap(); + assert_eq!(archive.files_touched.len(), 2); + let touched_source_ids = archive + .files_touched + .iter() + .map(|file| file.source_id.unwrap()) + .collect::>(); + assert_eq!( + touched_source_ids, + BTreeSet::from([first_source_id, second_source_id]) + ); + for file in archive.files_touched { + let source_id = file.source_id.unwrap(); + assert_eq!( + file.event_id, + Some(provider_source_event_uuid(source_id, 0)) + ); + } + } + + fn provider_collision_capture( + provider: CaptureProvider, + provider_session_id: &str, + source_format: &str, + raw_source_path: &str, + occurred_at: DateTime, + ) -> ProviderCaptureEnvelope { + ProviderCaptureEnvelope { + schema_version: PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, + provider, + source: ProviderSourceEnvelope { + source_format: source_format.to_owned(), + machine_id: "test-machine".to_owned(), + observed_at: occurred_at, + raw_source_path: Some(raw_source_path.to_owned()), + raw_retention: ProviderRawRetention::PathReference, + redaction_boundary: ProviderRedactionBoundary::BeforeExport, + trust: ProviderSourceTrust::ProviderExport, + fidelity: Fidelity::Imported, + cursor: None, + idempotency_key: Some(format!( + "provider-source:{}:{}:{}", + provider.as_str(), + source_format, + provider_session_id + )), + metadata: json!({}), + }, + session: ProviderSessionEnvelope { + provider_session_id: provider_session_id.to_owned(), + parent_provider_session_id: None, + root_provider_session_id: None, + external_agent_id: None, + agent_type: AgentType::Primary, + role_hint: Some("primary".to_owned()), + is_primary: true, + status: SessionStatus::Imported, + started_at: occurred_at, + ended_at: None, + cwd: Some("/workspace/example".to_owned()), + fidelity: Fidelity::Imported, + idempotency_key: Some(format!( + "provider-session:{}:{}", + provider.as_str(), + provider_session_id + )), + artifacts: Vec::new(), + metadata: json!({}), + }, + event: Some(ProviderEventEnvelope { + provider_event_index: 0, + provider_event_hash: None, + cursor: None, + event_type: EventType::Message, + role: Some(EventRole::User), + occurred_at, + fidelity: Fidelity::Imported, + redaction_state: RedactionState::LocalPreview, + idempotency_key: Some(format!( + "provider-event:{}:{}:0", + provider.as_str(), + provider_session_id + )), + artifacts: Vec::new(), + payload: json!({"text": "same provider event payload"}), + metadata: json!({}), + }), + } + } + + fn provider_collision_file_touch( + provider: CaptureProvider, + provider_session_id: &str, + source_format: &str, + raw_source_path: &str, + occurred_at: DateTime, + ) -> ProviderFileTouchedEnvelope { + ProviderFileTouchedEnvelope { + provider, + provider_session_id: provider_session_id.to_owned(), + provider_touch_index: 0, + provider_event_index: Some(0), + raw_source_path: Some(raw_source_path.to_owned()), + path: "src/lib.rs".to_owned(), + change_kind: Some(FileChangeKind::Modified), + old_path: None, + line_count_delta: Some(1), + confidence: Confidence::Explicit, + occurred_at, + source_format: source_format.to_owned(), + metadata: json!({}), + } + } + #[test] fn codex_history_import_is_prompt_only_summary_fidelity_and_idempotent() { let temp = tempdir(); diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 066682a59..2537851ac 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -2524,6 +2524,14 @@ impl Store { ) } + pub fn provider_source_event_dedupe_key( + source_id: Uuid, + provider_index: u64, + payload_hash: &str, + ) -> String { + format!("provider-source:{source_id}:{provider_index}:{payload_hash}") + } + pub fn upsert_event(&self, event: &Event) -> Result { let event_id = if let Some(dedupe_key) = &event.dedupe_key { reject_provider_event_hash_conflict(&self.conn, dedupe_key)?; @@ -3069,6 +3077,18 @@ impl Store { Ok(()) } + pub fn file_touched_exists(&self, id: Uuid) -> Result { + Ok(self + .conn + .query_row( + "SELECT 1 FROM files_touched WHERE id = ?1", + params![id.to_string()], + |_| Ok(()), + ) + .optional()? + .is_some()) + } + fn list_files_touched(&self) -> Result> { let mut stmt = self .conn @@ -5300,12 +5320,10 @@ fn table_exists(conn: &Connection, table: &str) -> Result { } fn reject_provider_event_hash_conflict(conn: &Connection, dedupe_key: &str) -> Result<()> { - let Some((provider, external_session_id, provider_index, _new_hash)) = - parse_provider_event_dedupe_key(dedupe_key) - else { + let Some(parsed) = parse_provider_event_dedupe_key(dedupe_key) else { return Ok(()); }; - let prefix = provider_event_dedupe_key_prefix(&provider, &external_session_id, provider_index); + let prefix = provider_event_dedupe_key_prefix(&parsed); let upper_bound = provider_event_dedupe_key_upper_bound(&prefix); let mut stmt = conn.prepare( "SELECT dedupe_key FROM events @@ -5317,12 +5335,10 @@ fn reject_provider_event_hash_conflict(conn: &Connection, dedupe_key: &str) -> R } fn reject_provider_event_hash_conflict_tx(tx: &Transaction<'_>, dedupe_key: &str) -> Result<()> { - let Some((provider, external_session_id, provider_index, _new_hash)) = - parse_provider_event_dedupe_key(dedupe_key) - else { + let Some(parsed) = parse_provider_event_dedupe_key(dedupe_key) else { return Ok(()); }; - let prefix = provider_event_dedupe_key_prefix(&provider, &external_session_id, provider_index); + let prefix = provider_event_dedupe_key_prefix(&parsed); let upper_bound = provider_event_dedupe_key_upper_bound(&prefix); let mut stmt = tx.prepare( "SELECT dedupe_key FROM events @@ -5337,41 +5353,56 @@ fn reject_provider_event_hash_conflict_from_rows( dedupe_key: &str, rows: rusqlite::MappedRows<'_, impl FnMut(&rusqlite::Row<'_>) -> rusqlite::Result>, ) -> Result<()> { - let Some((provider, external_session_id, provider_index, new_hash)) = - parse_provider_event_dedupe_key(dedupe_key) - else { + let Some(incoming) = parse_provider_event_dedupe_key(dedupe_key) else { return Ok(()); }; for row in rows { let existing_key = row?; - let Some((existing_provider, existing_session_id, existing_index, existing_hash)) = - parse_provider_event_dedupe_key(&existing_key) - else { + let Some(existing) = parse_provider_event_dedupe_key(&existing_key) else { continue; }; - if existing_provider == provider - && existing_session_id == external_session_id - && existing_index == provider_index - && existing_hash != new_hash + if existing.has_same_event_identity(&incoming) + && existing.payload_hash != incoming.payload_hash { return Err(StoreError::ProviderEventConflict { - provider, - external_session_id, - provider_index, - existing_hash, - new_hash, + provider: incoming.provider, + external_session_id: incoming.external_session_id, + provider_index: incoming.provider_index, + existing_hash: existing.payload_hash, + new_hash: incoming.payload_hash, }); } } Ok(()) } -fn provider_event_dedupe_key_prefix( - provider: &str, - external_session_id: &str, +#[derive(Debug, Clone)] +struct ParsedProviderEventDedupeKey { + provider: String, + external_session_id: String, + source_id: Option, provider_index: u64, -) -> String { - format!("provider:{provider}:{external_session_id}:{provider_index}:") + payload_hash: String, +} + +impl ParsedProviderEventDedupeKey { + fn has_same_event_identity(&self, other: &Self) -> bool { + self.provider == other.provider + && self.external_session_id == other.external_session_id + && self.source_id == other.source_id + && self.provider_index == other.provider_index + } +} + +fn provider_event_dedupe_key_prefix(parsed: &ParsedProviderEventDedupeKey) -> String { + if let Some(source_id) = &parsed.source_id { + format!("provider-source:{source_id}:{}:", parsed.provider_index) + } else { + format!( + "provider:{}:{}:{}:", + parsed.provider, parsed.external_session_id, parsed.provider_index + ) + } } fn provider_event_dedupe_key_upper_bound(prefix: &str) -> String { @@ -5380,7 +5411,24 @@ fn provider_event_dedupe_key_upper_bound(prefix: &str) -> String { upper_bound } -fn parse_provider_event_dedupe_key(dedupe_key: &str) -> Option<(String, String, u64, String)> { +fn parse_provider_event_dedupe_key(dedupe_key: &str) -> Option { + if let Some(rest) = dedupe_key.strip_prefix("provider-source:") { + let mut parts = rest.splitn(3, ':'); + let source_id = parts.next()?.to_owned(); + let provider_index = parts.next()?.parse().ok()?; + let payload_hash = parts.next()?.to_owned(); + if source_id.is_empty() || payload_hash.is_empty() { + return None; + } + return Some(ParsedProviderEventDedupeKey { + provider: "provider-source".to_owned(), + external_session_id: source_id.clone(), + source_id: Some(source_id), + provider_index, + payload_hash, + }); + } + let mut parts = dedupe_key.splitn(5, ':'); let prefix = parts.next()?; if prefix != "provider" { @@ -5393,7 +5441,13 @@ fn parse_provider_event_dedupe_key(dedupe_key: &str) -> Option<(String, String, if provider.is_empty() || external_session_id.is_empty() || payload_hash.is_empty() { None } else { - Some((provider, external_session_id, provider_index, payload_hash)) + Some(ParsedProviderEventDedupeKey { + provider, + external_session_id, + source_id: None, + provider_index, + payload_hash, + }) } } @@ -5608,18 +5662,6 @@ fn reject_rich_import_conflicts( "capture_source", source.id, )?; - if let Some(external_session_id) = &source.descriptor.external_session_id { - reject_entity_conflict( - existing_capture_source_by_external_session( - tx, - source.descriptor.provider, - external_session_id, - )?, - source, - "capture_source", - source.id, - )?; - } } for workspace in &archive.vcs_workspaces { reject_entity_conflict( @@ -5740,7 +5782,8 @@ fn reject_rich_import_conflicts( fn reject_archive_event_internal_conflicts(archive: &SessionHistoryArchive) -> Result<()> { let mut seen_seq: HashMap = HashMap::new(); - let mut seen_provider_events: HashMap<(String, String, u64), String> = HashMap::new(); + let mut seen_provider_events: HashMap<(String, String, Option, u64), String> = + HashMap::new(); for event in &archive.events { if let Some(existing) = seen_seq.insert(event.seq, event) { @@ -5755,24 +5798,27 @@ fn reject_archive_event_internal_conflicts(archive: &SessionHistoryArchive) -> R let Some(dedupe_key) = &event.dedupe_key else { continue; }; - let Some((provider, external_session_id, provider_index, payload_hash)) = - parse_provider_event_dedupe_key(dedupe_key) - else { + let Some(parsed) = parse_provider_event_dedupe_key(dedupe_key) else { continue; }; - let key = (provider, external_session_id, provider_index); + let key = ( + parsed.provider, + parsed.external_session_id, + parsed.source_id, + parsed.provider_index, + ); if let Some(existing_hash) = seen_provider_events.get(&key) { - if existing_hash != &payload_hash { + if existing_hash != &parsed.payload_hash { return Err(StoreError::ProviderEventConflict { provider: key.0, external_session_id: key.1, - provider_index: key.2, + provider_index: key.3, existing_hash: existing_hash.clone(), - new_hash: payload_hash, + new_hash: parsed.payload_hash, }); } } else { - seen_provider_events.insert(key, payload_hash); + seen_provider_events.insert(key, parsed.payload_hash); } } @@ -5803,20 +5849,6 @@ fn existing_capture_source_by_id(tx: &Transaction<'_>, id: Uuid) -> Result, - provider: CaptureProvider, - external_session_id: &str, -) -> Result> { - tx.query_row( - "SELECT id, kind, provider, machine_id, process_id, cwd, raw_source_path, external_session_id, started_at_ms, ended_at_ms, fidelity, visibility, sync_state, sync_version, metadata_json FROM capture_sources WHERE provider = ?1 AND external_session_id = ?2 ORDER BY started_at_ms DESC LIMIT 1", - params![provider.as_str(), external_session_id], - capture_source_from_row, - ) - .optional() - .map_err(StoreError::from) -} - fn existing_session_by_id(tx: &Transaction<'_>, id: Uuid) -> Result> { tx.query_row( session_select_sql("WHERE id = ?1").as_str(), @@ -8866,6 +8898,73 @@ mod catalog_tests { assert_eq!(catalog_count, 3); } + #[test] + fn archive_import_allows_multiple_capture_sources_for_same_provider_session() { + let temp = tempdir(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let external_session_id = "provider-session-1"; + let first_source = provider_archive_source( + "018f45d0-0000-7000-8000-000000080001", + external_session_id, + "/tmp/provider/first.jsonl", + ); + let second_source = provider_archive_source( + "018f45d0-0000-7000-8000-000000080002", + external_session_id, + "/tmp/provider/second.jsonl", + ); + + store + .import_archive(&archive_with_source(first_source.clone()), false) + .unwrap(); + store + .import_archive(&archive_with_source(second_source.clone()), false) + .unwrap(); + + let sources = store.list_capture_sources().unwrap(); + assert_eq!(sources.len(), 2); + assert_eq!( + sources + .iter() + .map(|source| source.id) + .collect::>(), + BTreeSet::from([first_source.id, second_source.id]) + ); + assert!(sources + .iter() + .all(|source| source.descriptor.external_session_id.as_deref() + == Some(external_session_id))); + } + + fn archive_with_source(source: CaptureSource) -> SessionHistoryArchive { + SessionHistoryArchive { + capture_sources: vec![source], + ..SessionHistoryArchive::default() + } + } + + fn provider_archive_source( + id: &str, + external_session_id: &str, + raw_source_path: &str, + ) -> CaptureSource { + CaptureSource { + id: Uuid::parse_str(id).unwrap(), + descriptor: CaptureSourceDescriptor { + kind: ctx_history_core::CaptureSourceKind::ProviderImport, + provider: CaptureProvider::Claude, + machine_id: "test-machine".to_owned(), + process_id: None, + cwd: Some("/repo".to_owned()), + raw_source_path: Some(raw_source_path.to_owned()), + external_session_id: Some(external_session_id.to_owned()), + }, + started_at: fixed_time(), + ended_at: None, + sync: sync_metadata(), + } + } + #[test] fn schema_v15_rebuilds_provider_checks_with_referenced_sources_and_indexes() { let temp = tempdir(); From cc059d63019dacec80c4b4dbcf8dcbf623c1b1a8 Mon Sep 17 00:00:00 2001 From: Luca King Date: Thu, 2 Jul 2026 23:42:52 -0500 Subject: [PATCH 46/72] Initialize stores in refresh-off CLI tests (#37) Co-authored-by: luca-ctx <216224554+luca-ctx@users.noreply.github.com> --- crates/ctx-cli/tests/cli.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index ff8034e9f..7dac4ef0d 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -28,6 +28,26 @@ fn ctx(temp: &TempDir) -> Command { command } +fn initialize_empty_store(temp: &TempDir) { + fs::create_dir_all(temp.path().join(".codex").join("sessions")).unwrap(); + ctx(temp) + .args(["setup", "--catalog-only", "--progress", "none"]) + .assert() + .success(); +} + +fn initialize_empty_store_with_env(temp: &TempDir, data_root: &Path, home: &Path, state: &Path) { + fs::create_dir_all(home.join(".codex").join("sessions")).unwrap(); + ctx(temp) + .args(["setup", "--catalog-only", "--progress", "none"]) + .env("CTX_DATA_ROOT", data_root) + .env("HOME", home) + .env("XDG_STATE_HOME", state) + .env("LOCALAPPDATA", state) + .assert() + .success(); +} + fn provider_history_fixture(name: &str) -> String { materialized_fixture("provider-history", name) } @@ -1858,6 +1878,8 @@ fn provider_help_matches_implemented_importers() { #[test] fn provider_json_names_are_accepted_as_cli_filter_aliases() { let temp = tempdir(); + initialize_empty_store(&temp); + for (provider, expected) in [ ("copilot_cli", "copilot_cli"), ("factory_ai_droid", "factory_ai_droid"), @@ -2973,6 +2995,7 @@ fn analytics_payloads_omit_sensitive_command_data() { let data_root = temp.path().join("ctx-data"); let events_path = temp.path().join("analytics.jsonl"); fs::create_dir_all(&home).unwrap(); + initialize_empty_store_with_env(&temp, &data_root, &home, &state); let private_query = "prompt text /home/alice/private/acme-secret repo@example.com host.internal 192.0.2.44"; From c2e0ab6d61f0e7732a7f94fb40bfd66c46bb4779 Mon Sep 17 00:00:00 2001 From: Luca King Date: Fri, 3 Jul 2026 13:34:59 -0500 Subject: [PATCH 47/72] Add CLI install handoff diagnostics Co-authored-by: luca-ctx <216224554+luca-ctx@users.noreply.github.com> --- Cargo.lock | 10 +- MODULE.bazel | 2 +- crates/ctx-cli/Cargo.toml | 2 +- crates/ctx-cli/src/analytics.rs | 60 ++++--- crates/ctx-cli/src/install_marker.rs | 106 ++++++++++++ crates/ctx-cli/src/main.rs | 14 ++ crates/ctx-cli/src/upgrade.rs | 40 ++++- crates/ctx-cli/tests/cli.rs | 238 ++++++++++++++++++++++++++ crates/ctx-history-capture/Cargo.toml | 2 +- crates/ctx-history-core/Cargo.toml | 2 +- crates/ctx-history-search/Cargo.toml | 2 +- crates/ctx-history-store/Cargo.toml | 2 +- 12 files changed, 443 insertions(+), 37 deletions(-) create mode 100644 crates/ctx-cli/src/install_marker.rs diff --git a/Cargo.lock b/Cargo.lock index ee0d1dde5..f793d8fec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,7 +256,7 @@ dependencies = [ [[package]] name = "ctx" -version = "0.17.0" +version = "0.18.0" dependencies = [ "anyhow", "assert_cmd", @@ -282,7 +282,7 @@ dependencies = [ [[package]] name = "ctx-history-capture" -version = "0.17.0" +version = "0.18.0" dependencies = [ "chrono", "ctx-history-core", @@ -297,7 +297,7 @@ dependencies = [ [[package]] name = "ctx-history-core" -version = "0.17.0" +version = "0.18.0" dependencies = [ "chrono", "directories", @@ -310,7 +310,7 @@ dependencies = [ [[package]] name = "ctx-history-search" -version = "0.17.0" +version = "0.18.0" dependencies = [ "chrono", "ctx-history-core", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "ctx-history-store" -version = "0.17.0" +version = "0.18.0" dependencies = [ "chrono", "ctx-history-core", diff --git a/MODULE.bazel b/MODULE.bazel index 27ddfec22..1e4e30683 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1 +1 @@ -module(name = "ctx_search", version = "0.17.0") +module(name = "ctx_search", version = "0.18.0") diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index ecf44ca32..537b062c7 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx" -version = "0.17.0" +version = "0.18.0" description = "Local CLI for indexing and searching agent session history" edition.workspace = true autobins = false diff --git a/crates/ctx-cli/src/analytics.rs b/crates/ctx-cli/src/analytics.rs index e87eb612b..e63930f52 100644 --- a/crates/ctx-cli/src/analytics.rs +++ b/crates/ctx-cli/src/analytics.rs @@ -5,7 +5,7 @@ use ctx_history_core::utc_now; use serde_json::{json, Map, Value}; use uuid::Uuid; -use crate::{config::AppConfig, identity, net}; +use crate::{config::AppConfig, identity, install_marker, net}; pub type AnalyticsProperties = Map; @@ -38,6 +38,7 @@ fn send_cli_event_inner( let install_id = identity::install_id(data_root)?; let status = if event.success { "ok" } else { "error" }; let duration_ms = event.duration.as_millis().min(i64::MAX as u128) as i64; + let install_marker = install_marker::current_exe_install_marker(); let mut properties = event.properties; properties.insert("action".to_owned(), Value::String(event.action.to_owned())); properties.insert("json_output".to_owned(), Value::Bool(event.json_output)); @@ -45,12 +46,47 @@ fn send_cli_event_inner( "analytics_client".to_owned(), Value::String("ctx-cli".to_owned()), ); + if install_marker.is_some() { + properties.insert( + "install_manager".to_owned(), + Value::String("ctx-hosted-installer".to_owned()), + ); + } if !event.success { properties.insert( "failure_kind".to_owned(), Value::String("command_error".to_owned()), ); } + let mut cli_event = json!({ + "event_id": Uuid::now_v7().to_string(), + "event_name": "cli_invocation", + "event_version": 1, + "occurred_at": utc_now(), + "plane": "product", + "delivery": "remote", + "origin_runtime": "cli", + "origin_install_id": install_id, + "origin_device_id": device_id, + "app_version": env!("CARGO_PKG_VERSION"), + "os": std::env::consts::OS, + "arch": std::env::consts::ARCH, + "surface": "cli", + "source": "ctx-cli", + "duration_ms": duration_ms, + "duration_bucket": duration_bucket(event.duration), + "status": status, + "success": event.success, + "properties": properties + }); + if let Some(marker) = install_marker { + if let Some(object) = cli_event.as_object_mut() { + object.insert( + "install_attempt_id".to_owned(), + Value::String(marker.install_attempt_id), + ); + } + } let payload = json!({ "broker_install_id": install_id, "broker_device_id": device_id, @@ -58,27 +94,7 @@ fn send_cli_event_inner( "broker_app_version": env!("CARGO_PKG_VERSION"), "broker_os": std::env::consts::OS, "broker_arch": std::env::consts::ARCH, - "events": [{ - "event_id": Uuid::now_v7().to_string(), - "event_name": "cli_invocation", - "event_version": 1, - "occurred_at": utc_now(), - "plane": "product", - "delivery": "remote", - "origin_runtime": "cli", - "origin_install_id": install_id, - "origin_device_id": device_id, - "app_version": env!("CARGO_PKG_VERSION"), - "os": std::env::consts::OS, - "arch": std::env::consts::ARCH, - "surface": "cli", - "source": "ctx-cli", - "duration_ms": duration_ms, - "duration_bucket": duration_bucket(event.duration), - "status": status, - "success": event.success, - "properties": properties - }] + "events": [cli_event] }); let body = serde_json::to_vec(&payload)?; net::post_json(&config.analytics.endpoint, &body) diff --git a/crates/ctx-cli/src/install_marker.rs b/crates/ctx-cli/src/install_marker.rs new file mode 100644 index 000000000..aa371ae78 --- /dev/null +++ b/crates/ctx-cli/src/install_marker.rs @@ -0,0 +1,106 @@ +use std::{ + env, fs, + io::Read, + path::{Path, PathBuf}, +}; + +use serde_json::Value; + +const MAX_MARKER_BYTES: u64 = 16 * 1024; +const MAX_INSTALL_ATTEMPT_ID_CHARS: usize = 128; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InstallMarker { + pub install_attempt_id: String, +} + +pub fn current_exe_install_marker() -> Option { + let exe = env::current_exe().ok()?; + read_install_marker(&install_marker_path(&exe)) +} + +fn read_install_marker(path: &Path) -> Option { + let metadata = fs::metadata(path).ok()?; + if !metadata.is_file() || metadata.len() > MAX_MARKER_BYTES { + return None; + } + let file = fs::File::open(path).ok()?; + let mut reader = file.take(MAX_MARKER_BYTES + 1); + let mut bytes = Vec::new(); + if reader.read_to_end(&mut bytes).is_err() || bytes.len() as u64 > MAX_MARKER_BYTES { + return None; + } + parse_install_marker(&bytes) +} + +fn parse_install_marker(bytes: &[u8]) -> Option { + let value: Value = serde_json::from_slice(bytes).ok()?; + let id = value.get("install_attempt_id")?.as_str()?.trim(); + if is_valid_install_attempt_id(id) { + Some(InstallMarker { + install_attempt_id: id.to_owned(), + }) + } else { + None + } +} + +fn is_valid_install_attempt_id(value: &str) -> bool { + !value.is_empty() + && value.chars().count() <= MAX_INSTALL_ATTEMPT_ID_CHARS + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +fn install_marker_path(exe: &Path) -> PathBuf { + let mut marker = exe.as_os_str().to_owned(); + marker.push(".install.json"); + PathBuf::from(marker) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_bounded_install_attempt_id() { + let marker = parse_install_marker(br#"{"install_attempt_id":"attempt_01-HOSTED"}"#) + .expect("valid marker"); + + assert_eq!(marker.install_attempt_id, "attempt_01-HOSTED"); + } + + #[test] + fn ignores_malformed_or_unbounded_install_attempt_id() { + assert!(parse_install_marker(b"{not-json").is_none()); + assert!(parse_install_marker(br#"{"install_attempt_id":""}"#).is_none()); + assert!(parse_install_marker(br#"{"install_attempt_id":"contains space"}"#).is_none()); + assert!(parse_install_marker( + format!( + r#"{{"install_attempt_id":"{}"}}"#, + "a".repeat(MAX_INSTALL_ATTEMPT_ID_CHARS + 1) + ) + .as_bytes() + ) + .is_none()); + } + + #[test] + fn appends_marker_suffix_to_full_exe_path() { + assert_eq!( + install_marker_path(Path::new("/tmp/ctx.exe")), + PathBuf::from("/tmp/ctx.exe.install.json") + ); + } + + #[test] + fn ignores_missing_or_oversized_marker_file() { + let temp = tempfile::tempdir().unwrap(); + assert!(read_install_marker(&temp.path().join("missing.install.json")).is_none()); + + let path = temp.path().join("ctx.install.json"); + fs::write(&path, vec![b'a'; MAX_MARKER_BYTES as usize + 1]).unwrap(); + assert!(read_install_marker(&path).is_none()); + } +} diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 4a766dc04..60d2aa0f9 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -20,6 +20,7 @@ mod config; mod docs; mod history_source_plugins; mod identity; +mod install_marker; mod mcp; mod net; mod upgrade; @@ -1389,6 +1390,19 @@ fn main() -> Result<()> { .unwrap_or_else(default_data_root) .context("resolve ctx data root")?; let config = AppConfig::load(&data_root)?; + if matches!(&cli.command, CommandRoot::Setup(_)) && sends_analytics { + analytics::send_cli_event( + &data_root, + &config, + AnalyticsEvent { + action: "setup_started", + json_output, + success: true, + duration: StdDuration::ZERO, + properties: analytics_properties.clone(), + }, + ); + } let result = match cli.command { CommandRoot::Setup(args) => run_setup(args, data_root.clone(), &mut analytics_properties), diff --git a/crates/ctx-cli/src/upgrade.rs b/crates/ctx-cli/src/upgrade.rs index 5196bec49..b6a759f7c 100644 --- a/crates/ctx-cli/src/upgrade.rs +++ b/crates/ctx-cli/src/upgrade.rs @@ -24,6 +24,7 @@ const LOG_FILE: &str = "logs/upgrade.log"; const VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(2); const VERSION_PROBE_OUTPUT_LIMIT: usize = 4096; const STALE_UPGRADE_LOCK_AFTER: Duration = Duration::from_secs(30 * 60); +const MAX_INSTALL_ATTEMPT_ID_CHARS: usize = 128; const DEFAULT_METADATA_PUBLIC_KEY_PEM: &str = r#"-----BEGIN RSA PUBLIC KEY----- MIIBigKCAYEAyBPNIx3H/NwWlN9CPHY5kOEe9kQEshOJEMpv3Atq086H1FWqliTm 3BCWiO4s/89wNMn11Pla2JetCWNiWsbxm3BIxCd1o6cq8y9ur6Zk1RGOQBLQgqhF @@ -848,7 +849,8 @@ fn replace_binary(staged: &Path, plan: &UpgradePlan) -> Result { let script = staged.with_extension("ps1"); let marker_tmp = staged.with_extension("install.json.tmp"); let marker_path = install_marker_path(target); - write_install_marker_to(&marker_tmp, plan)?; + let install_attempt_id = existing_install_attempt_id(&marker_path); + write_install_marker_to(&marker_tmp, plan, install_attempt_id.as_deref())?; let parent = std::process::id(); let body = format!( r#"$ErrorActionPreference = 'Stop' @@ -1016,11 +1018,16 @@ fn verify_install_marker(marker: &InstallMarker, platform: &str) -> Result<()> { fn write_install_marker_after_upgrade(plan: &UpgradePlan) -> Result<()> { let marker_path = install_marker_path(&plan.install_path); - write_install_marker_to(&marker_path, plan) + let install_attempt_id = existing_install_attempt_id(&marker_path); + write_install_marker_to(&marker_path, plan, install_attempt_id.as_deref()) } -fn write_install_marker_to(marker_path: &Path, plan: &UpgradePlan) -> Result<()> { - let body = json!({ +fn write_install_marker_to( + marker_path: &Path, + plan: &UpgradePlan, + install_attempt_id: Option<&str>, +) -> Result<()> { + let mut body = json!({ "schema_version": 1, "manager": "ctx-hosted-installer", "install_path": plan.install_path, @@ -1035,9 +1042,34 @@ fn write_install_marker_to(marker_path: &Path, plan: &UpgradePlan) -> Result<()> "store_schema_version": plan.metadata.store_schema_version, "installed_at": utc_now(), }); + if let Some(install_attempt_id) = install_attempt_id { + if let Some(object) = body.as_object_mut() { + object.insert( + "install_attempt_id".to_owned(), + Value::String(install_attempt_id.to_owned()), + ); + } + } atomic_write_json(marker_path, &body) } +fn existing_install_attempt_id(marker_path: &Path) -> Option { + read_json_file(marker_path).and_then(|value| optional_install_attempt_id(&value)) +} + +fn optional_install_attempt_id(value: &Value) -> Option { + let id = value.get("install_attempt_id")?.as_str()?.trim(); + is_valid_install_attempt_id(id).then(|| id.to_owned()) +} + +fn is_valid_install_attempt_id(value: &str) -> bool { + !value.is_empty() + && value.chars().count() <= MAX_INSTALL_ATTEMPT_ID_CHARS + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + fn string_field(value: &Value, key: &str) -> Result { value .get(key) diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 7dac4ef0d..d4d645e6c 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -28,6 +28,39 @@ fn ctx(temp: &TempDir) -> Command { command } +fn ctx_from_binary(temp: &TempDir, binary: &Path) -> Command { + let mut command = Command::new(binary); + command.env("CTX_DATA_ROOT", temp.path()); + command.env("HOME", temp.path()); + command.env("CTX_ANALYTICS_OFF", "1"); + command +} + +fn copied_ctx_binary(temp: &TempDir) -> PathBuf { + let source = PathBuf::from(Command::cargo_bin("ctx").unwrap().get_program().to_owned()); + let target = temp.path().join(if cfg!(windows) { + "ctx-test-copy.exe" + } else { + "ctx-test-copy" + }); + fs::copy(&source, &target).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mut permissions = fs::metadata(&target).unwrap().permissions(); + permissions.set_mode(permissions.mode() | 0o700); + fs::set_permissions(&target, permissions).unwrap(); + } + target +} + +fn hosted_install_marker_path(binary: &Path) -> PathBuf { + let mut marker = binary.as_os_str().to_owned(); + marker.push(".install.json"); + PathBuf::from(marker) +} + fn initialize_empty_store(temp: &TempDir) { fs::create_dir_all(temp.path().join(".codex").join("sessions")).unwrap(); ctx(temp) @@ -365,6 +398,10 @@ fn analytics_event_properties(event: &Value) -> &serde_json::Map event["events"][0]["properties"].as_object().unwrap() } +fn analytics_cli_event(event: &Value) -> &Value { + &event["events"][0] +} + fn sha256_hex(bytes: &[u8]) -> String { use sha2::{Digest, Sha256}; let digest = Sha256::digest(bytes); @@ -2321,6 +2358,7 @@ fn fake_release(temp: &TempDir, latest_version: &str) -> FakeRelease { let marker = json!({ "schema_version": 1, "manager": "ctx-hosted-installer", + "install_attempt_id": "ia_test_upgrade_attempt", "install_path": target, "platform": test_platform_key().replace('_', "-"), "channel": "stable", @@ -2435,6 +2473,7 @@ fn upgrade_status_check_and_apply_support_managed_installs() { serde_json::from_slice(&fs::read(install_marker_path(&release.target)).unwrap()).unwrap(); assert_eq!(marker["version"], "9.9.9"); assert_eq!(marker["sha256"], release.artifact_sha); + assert_eq!(marker["install_attempt_id"], "ia_test_upgrade_attempt"); } #[cfg(unix)] @@ -3116,6 +3155,204 @@ fn analytics_payloads_omit_sensitive_command_data() { } } +#[test] +fn hosted_install_marker_enriches_analytics_event_without_properties_leak() { + let temp = tempdir(); + let data_root = temp.path().join("ctx-data"); + let home = temp.path().join("home"); + let state = temp.path().join("state"); + let events_path = temp.path().join("analytics.jsonl"); + let binary = copied_ctx_binary(&temp); + let install_attempt_id = "attempt_01JZCTXHOSTED"; + let marker_secret = "marker-secret-must-not-leak"; + fs::write( + hosted_install_marker_path(&binary), + serde_json::to_vec_pretty(&json!({ + "schema_version": 1, + "install_attempt_id": install_attempt_id, + "installer_private_note": marker_secret, + })) + .unwrap(), + ) + .unwrap(); + + ctx_from_binary(&temp, &binary) + .arg("status") + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .env("CTX_UPGRADE_OFF", "1") + .assert() + .success(); + + let events = read_analytics_events(&events_path); + assert_eq!(events.len(), 1); + let cli_event = analytics_cli_event(&events[0]); + assert_eq!(cli_event["install_attempt_id"], install_attempt_id); + let properties = analytics_event_properties(&events[0]); + assert_eq!(properties["install_manager"], "ctx-hosted-installer"); + assert!( + properties.get("install_attempt_id").is_none(), + "raw marker id must stay out of analytics properties: {properties:#?}" + ); + assert_no_json_string_contains( + &Value::Object(properties.clone()), + &[install_attempt_id, marker_secret], + ); +} + +#[test] +fn malformed_hosted_install_marker_is_ignored() { + let temp = tempdir(); + let data_root = temp.path().join("ctx-data"); + let home = temp.path().join("home"); + let state = temp.path().join("state"); + let events_path = temp.path().join("analytics.jsonl"); + let binary = copied_ctx_binary(&temp); + fs::write( + hosted_install_marker_path(&binary), + b"{not-json marker-secret-must-not-leak", + ) + .unwrap(); + + ctx_from_binary(&temp, &binary) + .arg("status") + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .env("CTX_UPGRADE_OFF", "1") + .assert() + .success(); + + let events = read_analytics_events(&events_path); + assert_eq!(events.len(), 1); + let cli_event = analytics_cli_event(&events[0]); + assert!(cli_event.get("install_attempt_id").is_none()); + let properties = analytics_event_properties(&events[0]); + assert!(properties.get("install_manager").is_none()); + assert_no_json_string_contains( + &Value::Object(properties.clone()), + &["marker-secret-must-not-leak"], + ); +} + +#[test] +fn setup_analytics_emits_start_and_completion_events() { + let temp = tempdir(); + let data_root = temp.path().join("ctx-data"); + let home = temp.path().join("home"); + let state = temp.path().join("state"); + let events_path = temp.path().join("analytics.jsonl"); + fs::create_dir_all(home.join(".codex").join("sessions")).unwrap(); + + ctx(&temp) + .args(["setup", "--catalog-only", "--progress", "none"]) + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .env("CTX_UPGRADE_OFF", "1") + .assert() + .success(); + + let events = read_analytics_events(&events_path); + assert_eq!(events.len(), 2); + let actions = events + .iter() + .map(|event| { + analytics_event_properties(event)["action"] + .as_str() + .unwrap() + .to_owned() + }) + .collect::>(); + assert_eq!(actions, ["setup_started", "setup"]); + for event in &events { + assert_eq!(analytics_cli_event(event)["event_name"], "cli_invocation"); + assert_eq!(analytics_cli_event(event)["status"], "ok"); + assert_eq!(analytics_cli_event(event)["success"], true); + assert_analytics_properties_are_allowlisted(analytics_event_properties(event)); + } +} + +#[test] +fn setup_analytics_opt_out_suppresses_start_completion_and_identities() { + let temp = tempdir(); + let data_root = temp.path().join("ctx-data"); + let home = temp.path().join("home"); + let state = temp.path().join("state"); + let events_path = temp.path().join("analytics.jsonl"); + fs::create_dir_all(home.join(".codex").join("sessions")).unwrap(); + + ctx(&temp) + .args(["setup", "--catalog-only", "--progress", "none"]) + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .env("CTX_UPGRADE_OFF", "1") + .assert() + .success(); + + assert!( + !events_path.exists(), + "setup analytics opt-out should suppress start and completion events" + ); + assert!( + !data_root.join("install.json").exists(), + "setup analytics opt-out should not create an install identity" + ); + assert!( + !expected_device_path(&home, &state).exists(), + "setup analytics opt-out should not create a device identity" + ); +} + +#[test] +fn setup_analytics_dry_run_suppresses_start_completion_and_identities() { + let temp = tempdir(); + let data_root = temp.path().join("ctx-data"); + let home = temp.path().join("home"); + let state = temp.path().join("state"); + let events_path = temp.path().join("analytics.jsonl"); + fs::create_dir_all(home.join(".codex").join("sessions")).unwrap(); + + ctx(&temp) + .args(["setup", "--catalog-only", "--progress", "none"]) + .env("CTX_DATA_ROOT", &data_root) + .env("HOME", &home) + .env("XDG_STATE_HOME", &state) + .env("LOCALAPPDATA", &state) + .env_remove("CTX_ANALYTICS_OFF") + .env("CTX_ANALYTICS_DRY_RUN", "1") + .env("CTX_ANALYTICS_ENDPOINT", file_url(&events_path)) + .env("CTX_UPGRADE_OFF", "1") + .assert() + .success(); + + assert!( + !events_path.exists(), + "setup analytics dry run should suppress start and completion events" + ); + assert!( + !data_root.join("install.json").exists(), + "setup analytics dry run should not create an install identity" + ); + assert!( + !expected_device_path(&home, &state).exists(), + "setup analytics dry run should not create a device identity" + ); +} + #[test] fn analytics_config_opt_out_suppresses_delivery() { let temp = tempdir(); @@ -3278,6 +3515,7 @@ fn assert_analytics_properties_are_allowlisted(properties: &serde_json::Map Date: Fri, 3 Jul 2026 15:12:21 -0500 Subject: [PATCH 48/72] Fix Pi session directory imports Fixes #40 --- crates/ctx-cli/tests/cli.rs | 80 ++++--- crates/ctx-history-capture/src/lib.rs | 202 +++++++++++++----- .../src/provider_sources.rs | 26 ++- docs/cli-reference.md | 4 +- docs/first-10-minutes.md | 2 +- docs/limitations.md | 5 +- docs/provider-support-matrix.json | 5 +- docs/provider-support.md | 2 +- docs/providers.md | 3 +- 9 files changed, 237 insertions(+), 92 deletions(-) diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index d4d645e6c..503b08209 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1678,11 +1678,11 @@ fn import_all_discovers_and_imports_providers_together() { Path::new(&provider_history_fixture("codex-sessions")), &temp.path().join(".codex").join("sessions"), ); - let pi_home = temp.path().join(".pi"); + let pi_home = temp.path().join(".pi/agent/sessions/--workspace-example--"); fs::create_dir_all(&pi_home).unwrap(); fs::copy( provider_history_fixture("pi-session.jsonl"), - pi_home.join("sessions.jsonl"), + pi_home.join("2026-06-24T12-00-00-000Z_pi-session-docs-1.jsonl"), ) .unwrap(); @@ -5665,31 +5665,42 @@ fn install_default_claude_fixture(temp: &TempDir, query: &str) { copy_dir_all(&source, &temp.path().join(".claude").join("projects")); } -fn install_default_pi_fixture(temp: &TempDir, query: &str) { - let root = temp.path().join(".pi"); - fs::create_dir_all(&root).unwrap(); +fn write_pi_session_jsonl(path: &Path, id: &str, query: &str) { fs::write( - root.join("sessions.jsonl"), + path, format!( "{}\n{}\n", json!({ "type": "session", "version": 3, - "id": "pi-default-refresh", + "id": id, "timestamp": "2026-06-24T12:00:00.000Z", "cwd": "/workspace" }), json!({ "type": "message", - "id": "pi-default-refresh-user", + "id": format!("{id}-user"), "timestamp": "2026-06-24T12:00:01.000Z", - "message": {"role": "user", "content": query} + "message": { + "role": "user", + "content": [{"type": "text", "text": query}] + } }) ), ) .unwrap(); } +fn install_default_pi_fixture(temp: &TempDir, query: &str) { + let root = temp.path().join(".pi/agent/sessions/--workspace--"); + fs::create_dir_all(&root).unwrap(); + write_pi_session_jsonl( + &root.join("2026-06-24T12-00-00-000Z_pi-default-refresh.jsonl"), + "pi-default-refresh", + query, + ); +} + fn install_default_cursor_fixture(temp: &TempDir, query: &str) { let source = PathBuf::from(write_native_cursor_fixture(temp, query)); copy_dir_all(&source, &temp.path().join(".cursor").join("projects")); @@ -6649,26 +6660,41 @@ fn file_only_search_returns_touched_file_matches() { } #[test] -fn pi_cli_rejects_directory_import_path() { +fn pi_cli_imports_directory_tree_path() { let temp = tempdir(); let path = temp.path().join("pi-sessions-dir"); - fs::create_dir_all(&path).unwrap(); + let project = path.join("--workspace--"); + fs::create_dir_all(&project).unwrap(); + write_pi_session_jsonl( + &project.join("2026-06-24T12-00-00-000Z_pi-dir-alpha.jsonl"), + "pi-dir-alpha", + "pi directory alpha oracle", + ); + write_pi_session_jsonl( + &project.join("2026-06-24T12-01-00-000Z_pi-dir-beta.jsonl"), + "pi-dir-beta", + "pi directory beta oracle", + ); - ctx(&temp) - .args([ - "import", - "--provider", - "pi", - "--path", - path.to_str().unwrap(), - "--json", - ]) - .assert() - .failure() - .stderr( - predicate::str::contains("no importable pi history files") - .and(predicate::str::contains(path.to_str().unwrap())), - ); + let imported = json_output(ctx(&temp).args([ + "import", + "--provider", + "pi", + "--path", + path.to_str().unwrap(), + "--json", + ])); + assert_eq!(imported["totals"]["imported_sessions"], 2); + assert_eq!(imported["totals"]["imported_events"], 2); + + let search = json_output(ctx(&temp).args([ + "search", + "pi directory beta oracle", + "--provider", + "pi", + "--json", + ])); + assert_search_provider_oracle(&search, "pi", "pi directory beta oracle", 1, "message"); } #[test] @@ -6688,7 +6714,7 @@ fn pi_cli_rejects_wrong_file_import_path() { .assert() .failure() .stderr( - predicate::str::contains("no importable pi history files") + predicate::str::contains("no importable pi history files found") .and(predicate::str::contains(path.to_str().unwrap())), ); } diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 22761fff3..43a7e3fac 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -1375,68 +1375,105 @@ impl ProviderCaptureAdapter for PiSessionJsonlAdapter { path: &Path, context: &ProviderAdapterContext, ) -> Result { - ensure_regular_provider_transcript_file(path)?; - let file = File::open(path)?; - let reader = BufReader::new(file); - let mut result = ProviderNormalizationResult::default(); - let mut header = None; + normalize_pi_session_jsonl_path(path, context) + } +} - for (index, line) in reader.lines().enumerate() { - let line_number = index + 1; - let line = line?; - if line.trim().is_empty() { +fn normalize_pi_session_jsonl_path( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + if fs::symlink_metadata(path)?.file_type().is_file() { + return normalize_pi_session_jsonl_file(path, context); + } + + let mut paths = Vec::new(); + collect_jsonl_paths(path, &mut paths)?; + paths.sort(); + if paths.is_empty() { + return Err(CaptureError::InvalidProviderTranscriptPath { + path: path.to_path_buf(), + reason: native_jsonl_missing_reason(CaptureProvider::Pi), + }); + } + + let mut merged = ProviderNormalizationResult::default(); + for path in paths { + let mut file_context = context.clone(); + file_context.source_path = Some(path.clone()); + let mut result = normalize_pi_session_jsonl_file(&path, &file_context)?; + merged.summary.merge(result.summary); + merged.captures.append(&mut result.captures); + merged.files_touched.append(&mut result.files_touched); + } + Ok(merged) +} + +fn normalize_pi_session_jsonl_file( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + ensure_regular_provider_transcript_file(path)?; + let file = File::open(path)?; + let reader = BufReader::new(file); + let mut result = ProviderNormalizationResult::default(); + let mut header = None; + + for (index, line) in reader.lines().enumerate() { + let line_number = index + 1; + let line = line?; + if line.trim().is_empty() { + continue; + } + + let value: Value = match serde_json::from_str(&line) { + Ok(value) => value, + Err(err) => { + result.summary.failed += 1; + result.summary.failures.push(ProviderImportFailure { + line: line_number, + error: err.to_string(), + }); continue; } - - let value: Value = match serde_json::from_str(&line) { - Ok(value) => value, + }; + let entry_type = value + .get("type") + .and_then(Value::as_str) + .unwrap_or("unknown"); + if entry_type == "session" { + match pi_session_header(value) { + Ok(parsed) => { + let capture = pi_session_capture(&parsed, None, line_number, context); + header = Some(parsed); + result.captures.push((line_number, capture)); + } Err(err) => { result.summary.failed += 1; result.summary.failures.push(ProviderImportFailure { line: line_number, error: err.to_string(), }); - continue; - } - }; - let entry_type = value - .get("type") - .and_then(Value::as_str) - .unwrap_or("unknown"); - if entry_type == "session" { - match pi_session_header(value) { - Ok(parsed) => { - let capture = pi_session_capture(&parsed, None, line_number, context); - header = Some(parsed); - result.captures.push((line_number, capture)); - } - Err(err) => { - result.summary.failed += 1; - result.summary.failures.push(ProviderImportFailure { - line: line_number, - error: err.to_string(), - }); - } } - continue; } - - let Some(header) = header.as_ref() else { - result.summary.failed += 1; - result.summary.failures.push(ProviderImportFailure { - line: line_number, - error: "pi session entry appeared before session header".to_owned(), - }); - continue; - }; - result.captures.push(( - line_number, - pi_session_capture(header, Some(value), line_number, context), - )); + continue; } - Ok(result) + let Some(header) = header.as_ref() else { + result.summary.failed += 1; + result.summary.failures.push(ProviderImportFailure { + line: line_number, + error: "pi session entry appeared before session header".to_owned(), + }); + continue; + }; + result.captures.push(( + line_number, + pi_session_capture(header, Some(value), line_number, context), + )); } + + Ok(result) } impl ProviderCaptureAdapter for ClaudeProjectsJsonlAdapter { @@ -8877,6 +8914,7 @@ fn normalize_jsonl_tree( fn native_jsonl_missing_reason(provider: CaptureProvider) -> &'static str { match provider { + CaptureProvider::Pi => "no Pi session JSONL files found", CaptureProvider::Antigravity => { "no Antigravity transcript JSONL files found under brain/*/.system_generated/logs" } @@ -9614,7 +9652,7 @@ fn pi_session_capture( line_number: usize, context: &ProviderAdapterContext, ) -> ProviderCaptureEnvelope { - let event = entry.map(|entry| pi_session_event(&entry, line_number)); + let event = entry.map(|entry| pi_session_event(header, &entry, line_number)); let cursor = event.as_ref().and_then(|event| { event.cursor.as_ref().map(|cursor| ProviderCursorRange { before: None, @@ -9680,7 +9718,11 @@ fn pi_session_capture( } } -fn pi_session_event(entry: &Value, line_number: usize) -> ProviderEventEnvelope { +fn pi_session_event( + header: &PiSessionHeader, + entry: &Value, + line_number: usize, +) -> ProviderEventEnvelope { let entry_type = entry .get("type") .and_then(Value::as_str) @@ -9708,7 +9750,7 @@ fn pi_session_event(entry: &Value, line_number: usize) -> ProviderEventEnvelope occurred_at, fidelity: Fidelity::Imported, redaction_state: RedactionState::LocalPreview, - idempotency_key: Some(format!("provider-event:pi:{line_number}")), + idempotency_key: Some(format!("provider-event:pi:{}:{line_number}", header.id)), artifacts: Vec::new(), payload: json!({ "entry_type": entry_type, @@ -11957,6 +11999,64 @@ mod tests { assert!(!events[3].payload.to_string().contains("[REDACTED]")); } + #[test] + fn pi_session_import_replays_default_session_directory_tree() { + let temp = tempdir(); + let root = temp.path().join(".pi/agent/sessions/--workspace--"); + fs::create_dir_all(&root).unwrap(); + fs::write( + root.join("2026-06-24T12-00-00-000Z_pi-dir-alpha.jsonl"), + concat!( + "{\"type\":\"session\",\"version\":3,\"id\":\"pi-dir-alpha\",\"timestamp\":\"2026-06-24T12:00:00Z\",\"cwd\":\"/workspace\"}\n", + "{\"type\":\"message\",\"id\":\"pi-dir-alpha-user\",\"timestamp\":\"2026-06-24T12:00:01Z\",\"message\":{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"alpha directory import\"}]}}\n", + ), + ) + .unwrap(); + fs::write( + root.join("2026-06-24T12-01-00-000Z_pi-dir-beta.jsonl"), + concat!( + "{\"type\":\"session\",\"version\":3,\"id\":\"pi-dir-beta\",\"timestamp\":\"2026-06-24T12:01:00Z\",\"cwd\":\"/workspace\"}\n", + "{\"type\":\"message\",\"id\":\"pi-dir-beta-user\",\"timestamp\":\"2026-06-24T12:01:01Z\",\"message\":{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"beta directory import\"}]}}\n", + ), + ) + .unwrap(); + let sessions_root = temp.path().join(".pi/agent/sessions"); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let first = import_pi_session_jsonl( + &sessions_root, + &mut store, + PiSessionImportOptions { + source_path: Some(sessions_root.clone()), + imported_at: "2026-06-24T16:00:00Z".parse().unwrap(), + ..PiSessionImportOptions::default() + }, + ) + .unwrap(); + assert_eq!(first.failed, 0, "{:?}", first.failures); + assert_eq!(first.imported_sessions, 2); + assert_eq!(first.imported_events, 2); + + let second = import_pi_session_jsonl( + &sessions_root, + &mut store, + PiSessionImportOptions { + source_path: Some(sessions_root.clone()), + imported_at: "2026-06-24T16:00:00Z".parse().unwrap(), + ..PiSessionImportOptions::default() + }, + ) + .unwrap(); + assert_eq!(second.failed, 0, "{:?}", second.failures); + assert_eq!(second.imported_events, 0); + assert_eq!(second.skipped_events, 2); + + let alpha = provider_session_uuid(CaptureProvider::Pi, "pi-dir-alpha"); + let beta = provider_session_uuid(CaptureProvider::Pi, "pi-dir-beta"); + assert_eq!(store.events_for_session(alpha).unwrap().len(), 1); + assert_eq!(store.events_for_session(beta).unwrap().len(), 1); + } + #[test] fn codex_session_tree_imports_messages_and_subagent_edges() { let temp = tempdir(); diff --git a/crates/ctx-history-capture/src/provider_sources.rs b/crates/ctx-history-capture/src/provider_sources.rs index 0ec12c234..13711a733 100644 --- a/crates/ctx-history-capture/src/provider_sources.rs +++ b/crates/ctx-history-capture/src/provider_sources.rs @@ -105,7 +105,7 @@ const CODEX_DEFAULTS: &[ProviderDefaultLocation] = &[ ]; const PI_DEFAULTS: &[ProviderDefaultLocation] = &[ProviderDefaultLocation { - path_components: &[".pi", "sessions.jsonl"], + path_components: &[".pi", "agent", "sessions"], source_format: "pi_session_jsonl", source_kind: ProviderSourceKind::NativeHistory, }]; @@ -586,7 +586,7 @@ fn provider_source_from_location( fn empty_source_reason(provider: CaptureProvider) -> Option<&'static str> { match provider { CaptureProvider::Codex => Some("path exists but no Codex JSONL sessions were found"), - CaptureProvider::Pi => Some("path exists but no Pi session JSONL file was found"), + CaptureProvider::Pi => Some("path exists but no Pi session JSONL files were found"), CaptureProvider::Claude => { Some("path exists but no Claude project JSONL transcripts were found") } @@ -623,6 +623,9 @@ fn unknown_source_reason(provider: CaptureProvider) -> Option<&'static str> { CaptureProvider::Codex => { Some("path exists but the Codex session transcript probe hit its scan budget") } + CaptureProvider::Pi => { + Some("path exists but the Pi session transcript probe hit its scan budget") + } CaptureProvider::Claude => { Some("path exists but the Claude transcript probe hit its scan budget") } @@ -654,7 +657,7 @@ fn probe_io_error_reason(provider: CaptureProvider) -> Option<&'static str> { Some("path exists but Codex session transcripts could not be read; check permissions") } CaptureProvider::Pi => { - Some("path exists but the Pi session file could not be read; check permissions") + Some("path exists but Pi session transcripts could not be read; check permissions") } CaptureProvider::Claude => { Some("path exists but Claude project transcripts could not be read; check permissions") @@ -703,7 +706,7 @@ fn default_location_import_probe( path_is_file_probe(path) } CaptureProvider::Codex => has_jsonl_file_under_matching(path, 10_000, |_| true), - CaptureProvider::Pi => path_is_file_probe(path), + CaptureProvider::Pi => has_jsonl_file_under_matching(path, 10_000, |_| true), CaptureProvider::OpenCode => path_is_file_probe(path), CaptureProvider::Claude => has_jsonl_file_under_matching(path, 10_000, |_| true), CaptureProvider::OpenClaw => has_openclaw_session_jsonl(path, 10_000), @@ -957,6 +960,21 @@ mod tests { fn native_provider_default_discovery_uses_importer_specific_file_predicates() { let temp = tempfile::tempdir().unwrap(); + let pi = temp.path().join(".pi/agent/sessions"); + std::fs::create_dir_all(pi.join("--workspace--")).unwrap(); + assert_source_status( + temp.path(), + CaptureProvider::Pi, + ProviderSourceStatus::Empty, + ); + std::fs::write(pi.join("--workspace--/session.jsonl"), "{}\n").unwrap(); + let pi_source = discover_provider_sources(temp.path()) + .into_iter() + .find(|source| source.provider == CaptureProvider::Pi) + .unwrap(); + assert_eq!(pi_source.status, ProviderSourceStatus::Available); + assert_eq!(pi_source.path, temp.path().join(".pi/agent/sessions")); + let antigravity = temp.path().join(".gemini/antigravity-cli/brain"); std::fs::create_dir_all(antigravity.join("session/.system_generated/logs")).unwrap(); std::fs::write( diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 7acd83e98..55a4a9923 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -57,7 +57,7 @@ machine. Current rows include: - Codex session trees at `~/.codex/sessions`; - Codex prompt history at `~/.codex/history.jsonl`; -- Pi session JSONL at `~/.pi/sessions.jsonl`; +- Pi session JSONL files under `~/.pi/agent/sessions`; - native rows for supported Antigravity, Claude, OpenCode, OpenClaw, Hermes, Gemini, Cursor, Copilot CLI, and Factory AI Droid local history locations; - preview rows for NanoClaw project roots and AstrBot SQLite history when those @@ -94,7 +94,7 @@ ctx import --provider cursor ctx import --provider copilot-cli ctx import --provider factory-ai-droid ctx import --provider codex --path ~/.codex/sessions -ctx import --provider pi --path ~/.pi/sessions.jsonl +ctx import --provider pi --path ~/.pi/agent/sessions ctx import --format ctx-history-jsonl-v1 --path ./history.jsonl ctx import --history-source example-agent/default ctx import --history-source-manifest ./ctx-history-plugin.json diff --git a/docs/first-10-minutes.md b/docs/first-10-minutes.md index f45862a40..1ec93d850 100644 --- a/docs/first-10-minutes.md +++ b/docs/first-10-minutes.md @@ -61,7 +61,7 @@ you want to repair, re-run, resume, or pass an explicit path: ```bash ctx import --provider codex --path ~/.codex/sessions -ctx import --provider pi --path ~/.pi/sessions.jsonl +ctx import --provider pi --path ~/.pi/agent/sessions ctx import --provider cursor --path ~/.cursor/projects ctx import --provider hermes --path ~/.hermes/state.db ctx import --provider nanoclaw --path /path/to/nanoclaw-project diff --git a/docs/limitations.md b/docs/limitations.md index 3d2796e5a..f74ffde75 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -7,8 +7,9 @@ shipped. ## Provider Coverage - Codex local import is supported for documented local JSONL sources. -- Pi local import is supported only when a matching local `sessions.jsonl` file - exists. +- Pi local import is supported when matching local session JSONL files exist + under `~/.pi/agent/sessions`, or when an explicit Pi session JSONL file is + passed with `--path`. - Antigravity, Claude, OpenCode, OpenClaw, Hermes, Gemini, Cursor, Copilot CLI, and Factory AI Droid local import is supported only when their documented local history paths exist and match the supported native formats in the diff --git a/docs/provider-support-matrix.json b/docs/provider-support-matrix.json index a11a5ff82..3dc79e183 100644 --- a/docs/provider-support-matrix.json +++ b/docs/provider-support-matrix.json @@ -87,12 +87,13 @@ "ctx import --provider pi" ], "notes": [ - "Reads ~/.pi/sessions.jsonl when that file exists and matches the supported JSONL format." + "Reads Pi session JSONL files under ~/.pi/agent/sessions, including per-cwd session directories produced by current Pi releases.", + "An explicit Pi session JSONL file path remains supported with ctx import --provider pi --path." ] } ], "history_locations": [ - "~/.pi/sessions.jsonl" + "~/.pi/agent/sessions" ], "imports_existing_history": true, "captures_new_runs_passively": false, diff --git a/docs/provider-support.md b/docs/provider-support.md index 5f9ba1b35..9aa906012 100644 --- a/docs/provider-support.md +++ b/docs/provider-support.md @@ -23,7 +23,7 @@ is: | Provider | Status | Public import path | Public smoke | | --- | --- | --- | --- | | Codex | `local_import` | `~/.codex/sessions`, `~/.codex/history.jsonl`, or an explicit Codex path. | Static local-history fixture smoke. | -| Pi | `local_import_when_supported` | `~/.pi/sessions.jsonl` or an explicit Pi JSONL path. | Static local-history fixture smoke. | +| Pi | `local_import_when_supported` | `~/.pi/agent/sessions` or an explicit Pi session JSONL path. | Static local-history fixture smoke. | | Claude | `local_import_when_supported` | `~/.claude/projects` or an explicit Claude projects JSONL tree. | Static local-history fixture smoke. | | OpenCode | `local_import_when_supported` | `~/.local/share/opencode/opencode.db` or an explicit OpenCode SQLite DB. | Static local-history fixture smoke. | | OpenClaw | `local_import_when_supported` | `OPENCLAW_STATE_DIR`, `~/.openclaw`, legacy `~/.clawdbot`/`~/.moltbot`, or an explicit OpenClaw state tree. | Static local-history fixture smoke; beta storage-contract notes in the matrix. | diff --git a/docs/providers.md b/docs/providers.md index b6d18692b..f8bbeb731 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -10,8 +10,7 @@ The current CLI imports local history for: - Codex session JSONL trees under `~/.codex/sessions`; - Codex `~/.codex/history.jsonl`; -- Pi `~/.pi/sessions.jsonl` when that local file exists and matches the - supported JSONL format; +- Pi session JSONL files under `~/.pi/agent/sessions`; - Claude Code project JSONL transcripts under `~/.claude/projects`; - OpenCode SQLite history under `~/.local/share/opencode/opencode.db`; - OpenClaw session JSONL trees under `OPENCLAW_STATE_DIR`, `~/.openclaw`, From 1c7c21d57db0291f9c861ec7bd1366ff82505047 Mon Sep 17 00:00:00 2001 From: Luca King Date: Fri, 3 Jul 2026 15:30:34 -0500 Subject: [PATCH 49/72] Prepare 0.19.0 release Bump ctx to 0.19.0 for the Pi session directory release. --- Cargo.lock | 10 +++++----- crates/ctx-cli/Cargo.toml | 2 +- crates/ctx-history-capture/Cargo.toml | 2 +- crates/ctx-history-core/Cargo.toml | 2 +- crates/ctx-history-search/Cargo.toml | 2 +- crates/ctx-history-store/Cargo.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f793d8fec..8c50263bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,7 +256,7 @@ dependencies = [ [[package]] name = "ctx" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "assert_cmd", @@ -282,7 +282,7 @@ dependencies = [ [[package]] name = "ctx-history-capture" -version = "0.18.0" +version = "0.19.0" dependencies = [ "chrono", "ctx-history-core", @@ -297,7 +297,7 @@ dependencies = [ [[package]] name = "ctx-history-core" -version = "0.18.0" +version = "0.19.0" dependencies = [ "chrono", "directories", @@ -310,7 +310,7 @@ dependencies = [ [[package]] name = "ctx-history-search" -version = "0.18.0" +version = "0.19.0" dependencies = [ "chrono", "ctx-history-core", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "ctx-history-store" -version = "0.18.0" +version = "0.19.0" dependencies = [ "chrono", "ctx-history-core", diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index 537b062c7..8dbefb451 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx" -version = "0.18.0" +version = "0.19.0" description = "Local CLI for indexing and searching agent session history" edition.workspace = true autobins = false diff --git a/crates/ctx-history-capture/Cargo.toml b/crates/ctx-history-capture/Cargo.toml index 2435adb0c..688ea4fc3 100644 --- a/crates/ctx-history-capture/Cargo.toml +++ b/crates/ctx-history-capture/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-capture" -version = "0.18.0" +version = "0.19.0" description = "Internal provider import adapters for ctx local agent history" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-core/Cargo.toml b/crates/ctx-history-core/Cargo.toml index a80d8ab5f..4ae07ac59 100644 --- a/crates/ctx-history-core/Cargo.toml +++ b/crates/ctx-history-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-core" -version = "0.18.0" +version = "0.19.0" description = "Internal core types for ctx local agent history indexing" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-search/Cargo.toml b/crates/ctx-history-search/Cargo.toml index c6b5453cb..12dd4b78d 100644 --- a/crates/ctx-history-search/Cargo.toml +++ b/crates/ctx-history-search/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-search" -version = "0.18.0" +version = "0.19.0" description = "Internal search projection and ranking helpers for ctx" edition.workspace = true license.workspace = true diff --git a/crates/ctx-history-store/Cargo.toml b/crates/ctx-history-store/Cargo.toml index ae1927697..3557c20a1 100644 --- a/crates/ctx-history-store/Cargo.toml +++ b/crates/ctx-history-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ctx-history-store" -version = "0.18.0" +version = "0.19.0" description = "Internal SQLite storage layer for ctx local agent history" edition.workspace = true license.workspace = true From 34e29f93c4e0dc725d253be3e314b77ec3da3195 Mon Sep 17 00:00:00 2001 From: Atharva! Date: Sat, 4 Jul 2026 02:42:18 +0530 Subject: [PATCH 50/72] fix(cli): prevent overflow panic in --since parsing (#38) --- crates/ctx-cli/src/main.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 60d2aa0f9..6c717ad88 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -6428,7 +6428,12 @@ fn parse_since_filter(value: &str) -> Result> { let days: i64 = days .parse() .with_context(|| format!("invalid --since day window: {value}"))?; - return Ok(utc_now() - Duration::days(days)); + let duration = + Duration::try_days(days).ok_or_else(|| anyhow!("invalid --since day window: {value}: value too large"))?; + let since = utc_now() + .checked_sub_signed(duration) + .ok_or_else(|| anyhow!("invalid --since day window: {value}: value too large"))?; + return Ok(since); } Ok(chrono::DateTime::parse_from_rfc3339(trimmed) .with_context(|| format!("invalid --since value: {value}"))? @@ -6457,7 +6462,7 @@ fn home_dir() -> Option { #[cfg(test)] mod tests { - use super::{catalog_import_checkpoint_matches, sha256_file_prefix_hex, shell_quote_arg}; + use super::{catalog_import_checkpoint_matches, parse_since_filter, sha256_file_prefix_hex, shell_quote_arg}; use std::{fs, io::Write}; use tempfile::tempdir; @@ -6470,6 +6475,16 @@ mod tests { ); } + #[test] + fn parse_since_filter_rejects_large_day_window() { + let err = parse_since_filter("500000000d").unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("invalid --since day window"), + "expected error about invalid day window, got: {msg}" + ); + } + #[test] fn catalog_import_checkpoint_requires_matching_hash() { let temp = tempdir().unwrap(); From 4c71625cc4d573c6bb97f241fd9a9d6c4ad913dd Mon Sep 17 00:00:00 2001 From: Luca King Date: Fri, 3 Jul 2026 18:21:06 -0500 Subject: [PATCH 51/72] Add Shelley native history provider Add Shelley as a native ctx history/import provider using the shelley_sqlite source format. Supports explicit import and default discovery, normalizes Shelley SQLite conversations/messages into ctx sessions/events with stable cursors, metadata, citations, robust JSON text extraction, and read-only SQLite handling. Includes provider docs, support matrix updates, discovery tests, corrupt database coverage, idempotent re-import tests, and provider filtering/search coverage. --- crates/ctx-cli/src/main.rs | 20 +- crates/ctx-cli/src/mcp.rs | 2 + crates/ctx-cli/tests/cli.rs | 141 +- crates/ctx-history-capture/src/lib.rs | 1510 +++++++++++++++-- .../src/provider_sources.rs | 43 + crates/ctx-history-core/src/lib.rs | 1 + crates/ctx-history-core/src/provider.rs | 5 +- crates/ctx-history-store/src/lib.rs | 12 +- docs/cli-reference.md | 5 +- docs/first-10-minutes.md | 1 + docs/provider-support-matrix.json | 56 + docs/provider-support.md | 1 + docs/providers.md | 6 +- docs/search.md | 4 +- 14 files changed, 1655 insertions(+), 152 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 6c717ad88..9ab5e4c5b 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -36,7 +36,7 @@ use ctx_history_capture::{ import_custom_history_jsonl_v1_reader, import_factory_ai_droid_sessions, import_gemini_cli_history, import_hermes_sqlite, import_nanoclaw_project, import_openclaw_history, import_opencode_sqlite, import_pi_session_jsonl, - provider_source_for_path, provider_source_spec, stable_capture_uuid, + import_shelley_sqlite, provider_source_for_path, provider_source_spec, stable_capture_uuid, validate_custom_history_jsonl_v1, validate_custom_history_jsonl_v1_reader, AntigravityCliImportOptions, AstrBotSqliteImportOptions, CatalogSummary, ClaudeProjectsImportOptions, CodexEventImportMode, CodexHistoryImportOptions, @@ -46,6 +46,7 @@ use ctx_history_capture::{ GeminiCliImportOptions, HermesSqliteImportOptions, NanoClawImportOptions, OpenClawImportOptions, OpenCodeSqliteImportOptions, PiSessionImportOptions, ProviderImportSummary, ProviderImportSupport, ProviderSource, ProviderSourceStatus, + ShelleySqliteImportOptions, }; use ctx_history_core::{ database_path, default_data_root, utc_now, CaptureProvider, ContextCitation, @@ -681,6 +682,7 @@ enum NativeProviderArg { NanoClaw, #[value(name = "astrbot", alias = "astr-bot", alias = "astr_bot")] AstrBot, + Shelley, } #[derive(Debug, Clone, Copy, ValueEnum)] @@ -711,6 +713,7 @@ enum ProviderArg { NanoClaw, #[value(name = "astrbot", alias = "astr-bot", alias = "astr_bot")] AstrBot, + Shelley, Custom, } @@ -752,6 +755,7 @@ impl NativeProviderArg { Self::Hermes => CaptureProvider::Hermes, Self::NanoClaw => CaptureProvider::NanoClaw, Self::AstrBot => CaptureProvider::AstrBot, + Self::Shelley => CaptureProvider::Shelley, } } } @@ -772,6 +776,7 @@ impl ProviderArg { Self::Hermes => CaptureProvider::Hermes, Self::NanoClaw => CaptureProvider::NanoClaw, Self::AstrBot => CaptureProvider::AstrBot, + Self::Shelley => CaptureProvider::Shelley, Self::Custom => CaptureProvider::Custom, } } @@ -791,6 +796,7 @@ impl ProviderArg { Self::Hermes => "hermes", Self::NanoClaw => "nanoclaw", Self::AstrBot => "astrbot", + Self::Shelley => "shelley", Self::Custom => "custom", } } @@ -5505,6 +5511,17 @@ fn import_one_source_inner( }, ) .map_err(anyhow::Error::from), + CaptureProvider::Shelley => import_shelley_sqlite( + &source.path, + store, + ShelleySqliteImportOptions { + source_path: Some(source.path.clone()), + history_record_id: Some(record_id), + allow_partial_failures: true, + ..ShelleySqliteImportOptions::default() + }, + ) + .map_err(anyhow::Error::from), CaptureProvider::Gemini => import_gemini_cli_history( &source.path, store, @@ -5668,6 +5685,7 @@ fn source_uses_import_file_manifest(source: &SourceInfo) -> bool { | "hermes_state_sqlite" | "nanoclaw_project" | "astrbot_data_v4_sqlite" + | "shelley_sqlite" ) } diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index 60b807dc9..4d3614fc7 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -602,6 +602,7 @@ fn provider_names() -> Vec<&'static str> { ProviderArg::Hermes.cli_name(), ProviderArg::NanoClaw.cli_name(), ProviderArg::AstrBot.cli_name(), + ProviderArg::Shelley.cli_name(), ProviderArg::Custom.cli_name(), ]; names.sort_unstable(); @@ -687,6 +688,7 @@ fn optional_provider(arguments: &Value, key: &str) -> Result "hermes" => Ok(Some(ProviderArg::Hermes)), "nanoclaw" => Ok(Some(ProviderArg::NanoClaw)), "astrbot" => Ok(Some(ProviderArg::AstrBot)), + "shelley" => Ok(Some(ProviderArg::Shelley)), "custom" => Ok(Some(ProviderArg::Custom)), _ => Err(anyhow!( "provider must be one of {}", diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 503b08209..c027df05a 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1752,12 +1752,14 @@ fn sources_lists_personal_agent_provider_defaults() { install_default_openclaw_fixture(&temp, "openclaw-sources-oracle"); install_default_hermes_fixture(&temp, "hermes-sources-oracle"); install_default_astrbot_fixture(&temp, "astrbot-sources-oracle"); + install_default_shelley_fixture(&temp, "shelley-sources-oracle"); let sources = json_output(ctx(&temp).args(["sources", "--json"])); for (provider, source_format, import_support, native_import) in [ ("openclaw", "openclaw_session_jsonl_tree", "native", true), ("hermes", "hermes_state_sqlite", "native", true), ("astrbot", "astrbot_data_v4_sqlite", "preview", false), + ("shelley", "shelley_sqlite", "native", true), ] { let source = sources["sources"] .as_array() @@ -1775,6 +1777,31 @@ fn sources_lists_personal_agent_provider_defaults() { } } +#[test] +fn sources_discovers_shelley_db_env_override() { + let temp = tempdir(); + let db_path = temp.path().join("custom-shelley.db"); + fs::write(&db_path, b"sqlite fixture marker").unwrap(); + + let sources = json_output( + ctx(&temp) + .env("SHELLEY_DB", &db_path) + .args(["sources", "--json"]), + ); + let source = sources["sources"] + .as_array() + .unwrap() + .iter() + .find(|source| { + source["provider"] == "shelley" && source["path"] == db_path.to_str().unwrap() + }) + .unwrap_or_else(|| panic!("missing Shelley source in {sources:#}")); + assert_eq!(source["source_format"], "shelley_sqlite"); + assert_eq!(source["status"], "available"); + assert_eq!(source["import_support"], "native"); + assert_eq!(source["path"], db_path.to_str().unwrap()); +} + #[test] fn preview_native_sources_are_listed_but_not_auto_imported() { let temp = tempdir(); @@ -2041,7 +2068,7 @@ fn public_subcommand_help_is_golden_enough_for_session_retrieval() { vec![ "Usage: ctx import", "--provider ", - "[possible values: codex, pi, claude, opencode, antigravity, gemini, cursor, copilot-cli, factory-ai-droid, openclaw, hermes, nanoclaw, astrbot]", + "[possible values: codex, pi, claude, opencode, antigravity, gemini, cursor, copilot-cli, factory-ai-droid, openclaw, hermes, nanoclaw, astrbot, shelley]", "--path ", "--format ", "--resume", @@ -5154,6 +5181,7 @@ fn search_refresh_auto_imports_discovered_top_provider_sources() { ("cursor", "cursor", install_default_cursor_fixture), ("openclaw", "openclaw", install_default_openclaw_fixture), ("hermes", "hermes", install_default_hermes_fixture), + ("shelley", "shelley", install_default_shelley_fixture), ] { let temp = tempdir(); let query = format!("{stored_provider}-default-refresh-oracle"); @@ -5520,6 +5548,12 @@ fn native_provider_cli_flow_imports_new_supported_provider_paths() { "astrbot_data_v4_sqlite", write_native_astrbot_fixture, ), + ( + "shelley", + "shelley", + "shelley_sqlite", + write_native_shelley_fixture, + ), ] { let temp = tempdir(); let query = format!("{stored_provider}-native-cli-oracle"); @@ -5609,6 +5643,12 @@ fn personal_agent_provider_imports_are_idempotent_and_incremental() { write_native_astrbot_fixture, append_native_astrbot_event, ), + ( + "shelley", + "shelley", + write_native_shelley_fixture, + append_native_shelley_event, + ), ] { let temp = tempdir(); let initial_query = format!("{stored_provider}-incremental-initial-oracle"); @@ -5725,6 +5765,13 @@ fn install_default_astrbot_fixture(temp: &TempDir, query: &str) { fs::copy(source, target.join("data_v4.db")).unwrap(); } +fn install_default_shelley_fixture(temp: &TempDir, query: &str) { + let source = PathBuf::from(write_native_shelley_fixture(temp, query)); + let target = temp.path().join(".config/shelley"); + fs::create_dir_all(&target).unwrap(); + fs::copy(source, target.join("shelley.db")).unwrap(); +} + fn write_native_claude_fixture(temp: &TempDir, query: &str) -> String { let root = temp.path().join("native-claude/projects/-workspace"); fs::create_dir_all(&root).unwrap(); @@ -6232,6 +6279,82 @@ fn write_native_astrbot_fixture(temp: &TempDir, query: &str) -> String { path.to_str().unwrap().to_owned() } +fn write_native_shelley_fixture(temp: &TempDir, query: &str) -> String { + let path = temp.path().join("native-shelley.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "create table conversations ( + conversation_id text primary key, + slug text, + user_initiated boolean not null default true, + created_at datetime not null default current_timestamp, + updated_at datetime not null default current_timestamp, + cwd text, + archived boolean not null default false, + parent_conversation_id text, + model text, + conversation_options text not null default '{}', + current_generation integer not null default 1, + agent_working boolean not null default false, + tags text not null default '[]', + is_draft boolean not null default false, + draft text not null default '' + ); + create table messages ( + message_id text primary key, + conversation_id text not null, + sequence_id integer not null, + type text not null, + llm_data text, + user_data text, + usage_data text, + created_at datetime not null default current_timestamp, + display_data text, + excluded_from_context boolean not null default false, + generation integer not null default 1, + llm_api_url text, + model_name text, + forked_from_message_id text + );", + ) + .unwrap(); + conn.execute( + "insert into conversations values ( + 'shelley-cli-native', 'native shelley', 1, '2026-06-24 12:00:00', + '2026-06-24 12:00:01', '/workspace', 0, null, 'claude-opus-4-7', + '{}', 1, 0, '[]', 0, '' + )", + [], + ) + .unwrap(); + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, user_data, created_at + ) values ( + 'shelley-cli-native-user', 'shelley-cli-native', 1, 'user', ?1, + '2026-06-24 12:00:01' + )", + [json!({"Content": [{"Type": 2, "Text": query}]}).to_string()], + ) + .unwrap(); + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, llm_data, usage_data, + created_at, llm_api_url, model_name + ) values ( + 'shelley-cli-native-agent', 'shelley-cli-native', 2, 'agent', ?1, ?2, + '2026-06-24 12:00:02', 'https://api.anthropic.com/v1/messages', + 'claude-opus-4-7' + )", + [ + json!({"Content": [{"Type": 2, "Text": "native Shelley import ok"}]}).to_string(), + json!({"input_tokens": 12, "output_tokens": 8, "cost_usd": 0.001}).to_string(), + ], + ) + .unwrap(); + path.to_str().unwrap().to_owned() +} + fn append_native_openclaw_event(path: &str, query: &str) { let transcript = Path::new(path).join("agents/personal-agent/sessions/openclaw-cli-native.jsonl"); @@ -6300,6 +6423,20 @@ fn append_native_astrbot_event(path: &str, query: &str) { .unwrap(); } +fn append_native_shelley_event(path: &str, query: &str) { + let conn = Connection::open(path).unwrap(); + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, user_data, created_at + ) values ( + 'shelley-cli-native-user-2', 'shelley-cli-native', 3, 'user', ?1, + '2026-06-24 12:00:03' + )", + [json!({"Content": [{"Type": 2, "Text": query}]}).to_string()], + ) + .unwrap(); +} + #[test] fn openclaw_import_accepts_explicit_session_jsonl_file() { let temp = tempdir(); @@ -6377,6 +6514,7 @@ fn personal_agent_sqlite_imports_report_corrupt_databases() { for (provider, path) in [ ("hermes", "corrupt-hermes-state.db"), ("astrbot", "corrupt-astrbot-data_v4.db"), + ("shelley", "corrupt-shelley.db"), ] { let temp = tempdir(); let db_path = temp.path().join(path); @@ -6438,6 +6576,7 @@ fn native_provider_cli_requires_existing_history_or_explicit_path() { ("hermes", "no importable hermes history found"), ("nanoclaw", "no importable nanoclaw history found"), ("astrbot", "no importable astrbot history found"), + ("shelley", "no importable shelley history found"), ] { let temp = tempdir(); let stderr = diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 43a7e3fac..83000969c 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -10,7 +10,7 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDateTime, Utc}; use ctx_history_core::{ inbox_dir as core_inbox_dir, new_id, utc_now, AgentType, CaptureEnvelope, CaptureProvider, CaptureSource, CaptureSourceDescriptor, CaptureSourceKind, Confidence, @@ -537,6 +537,27 @@ impl Default for AstrBotSqliteImportOptions { } } +#[derive(Debug, Clone)] +pub struct ShelleySqliteImportOptions { + pub machine_id: String, + pub source_path: Option, + pub imported_at: DateTime, + pub history_record_id: Option, + pub allow_partial_failures: bool, +} + +impl Default for ShelleySqliteImportOptions { + fn default() -> Self { + Self { + machine_id: default_machine_id(), + source_path: None, + imported_at: utc_now(), + history_record_id: None, + allow_partial_failures: false, + } + } +} + #[derive(Debug, Clone)] pub struct AntigravityCliImportOptions { pub machine_id: String, @@ -858,6 +879,9 @@ pub struct NanoClawProjectAdapter; #[derive(Debug, Clone, Copy, Default)] pub struct AstrBotSqliteAdapter; +#[derive(Debug, Clone, Copy, Default)] +pub struct ShelleySqliteAdapter; + #[derive(Debug, Clone, Copy, Default)] pub struct AntigravityCliJsonlAdapter; @@ -1602,6 +1626,24 @@ impl ProviderCaptureAdapter for AstrBotSqliteAdapter { } } +impl ProviderCaptureAdapter for ShelleySqliteAdapter { + fn provider(&self) -> CaptureProvider { + CaptureProvider::Shelley + } + + fn source_format(&self) -> &str { + SHELLEY_SQLITE_SOURCE_FORMAT + } + + fn normalize_path( + &self, + path: &Path, + context: &ProviderAdapterContext, + ) -> Result { + normalize_shelley_sqlite(path, context) + } +} + impl ProviderCaptureAdapter for AntigravityCliJsonlAdapter { fn provider(&self) -> CaptureProvider { CaptureProvider::Antigravity @@ -3569,6 +3611,40 @@ pub fn import_astrbot_sqlite( ) } +pub fn import_shelley_sqlite( + path: impl AsRef, + store: &mut Store, + options: ShelleySqliteImportOptions, +) -> Result { + let path = path.as_ref(); + let source_path = options + .source_path + .clone() + .unwrap_or_else(|| path.to_path_buf()); + let normalization = ShelleySqliteAdapter.normalize_path( + path, + &ProviderAdapterContext { + machine_id: options.machine_id, + source_path: Some(source_path), + imported_at: options.imported_at, + tool_output_mode: CodexToolOutputMode::Full, + event_mode: CodexEventImportMode::Rich, + include_notices: true, + }, + )?; + import_normalized_provider_captures( + store, + normalization, + NormalizedProviderImportOptions { + history_record_id: options.history_record_id, + allow_partial_failures: options.allow_partial_failures, + persist_cursors: true, + wrap_transaction: true, + fast_event_inserts: true, + }, + ) +} + pub fn import_antigravity_cli_history( path: impl AsRef, store: &mut Store, @@ -3725,6 +3801,7 @@ const OPENCLAW_SOURCE_FORMAT: &str = "openclaw_session_jsonl_tree"; const HERMES_SQLITE_SOURCE_FORMAT: &str = "hermes_state_sqlite"; const NANOCLAW_SOURCE_FORMAT: &str = "nanoclaw_project"; const ASTRBOT_SQLITE_SOURCE_FORMAT: &str = "astrbot_data_v4_sqlite"; +const SHELLEY_SQLITE_SOURCE_FORMAT: &str = "shelley_sqlite"; const ANTIGRAVITY_CLI_SOURCE_FORMAT: &str = "antigravity_cli_transcript_jsonl_tree"; const GEMINI_CLI_SOURCE_FORMAT: &str = "gemini_cli_chat_recording_jsonl"; const CURSOR_AGENT_TRANSCRIPT_SOURCE_FORMAT: &str = "cursor_agent_transcript_jsonl"; @@ -6486,6 +6563,45 @@ struct OpenCodeMessageRow { data: String, } +#[derive(Debug, Clone)] +struct ShelleyConversationRow { + conversation_id: String, + slug: Option, + user_initiated: bool, + created_at: Option, + updated_at: Option, + cwd: Option, + archived: bool, + parent_conversation_id: Option, + model: Option, + conversation_options: Option, + current_generation: Option, + agent_working: bool, + tags: Option, + is_draft: bool, + draft: Option, + queued_messages: Option, +} + +#[derive(Debug, Clone)] +struct ShelleyMessageRow { + rowid: i64, + message_id: String, + conversation_id: String, + sequence_id: i64, + entry_type: String, + llm_data: Option, + user_data: Option, + usage_data: Option, + created_at: Option, + display_data: Option, + excluded_from_context: bool, + generation: Option, + llm_api_url: Option, + model_name: Option, + forked_from_message_id: Option, +} + struct NativeSessionDraft { provider: CaptureProvider, source_format: &'static str, @@ -7821,155 +7937,737 @@ fn nanoclaw_outbound_messages(path: &Path) -> Result> { .map_err(CaptureError::from) } -#[derive(Debug, Clone)] -struct AstrBotConversationRow { - row_id: i64, - inner_conversation_id: Option, - conversation_id: String, - platform_id: Option, - user_id: Option, - content: String, - title: Option, - persona_id: Option, - token_usage: Option, - created_at: Option, - updated_at: Option, -} - -#[derive(Debug, Clone)] -struct AstrBotPlatformMessageRow { - id: i64, - platform_id: Option, - user_id: Option, - sender_id: Option, - sender_name: Option, - content: Option, - llm_checkpoint_id: Option, - created_at: Option, -} - -fn normalize_astrbot_sqlite( +fn normalize_shelley_sqlite( path: &Path, context: &ProviderAdapterContext, ) -> Result { let conn = open_provider_sqlite_readonly(path)?; let user_version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?; let schema_fingerprint = opencode_schema_fingerprint(&conn)?; - let conversations = astrbot_conversations(&conn)?; - let platform_messages = astrbot_platform_messages(&conn)?; - let selected_conversation = astrbot_selected_conversation(&conn).ok().flatten(); + let conversations = shelley_conversations(&conn)?; + let messages = shelley_messages(&conn)?; + let conversations_by_id = conversations + .iter() + .map(|conversation| (conversation.conversation_id.clone(), conversation)) + .collect::>(); + let mut seen_message_conversations = BTreeSet::new(); + let raw_source_path = path.display().to_string(); let mut result = ProviderNormalizationResult::default(); - let mut checkpoint_sessions = BTreeMap::::new(); - for conversation in &conversations { - let provider_session_id = astrbot_provider_session_id(conversation); - let started_at = provider_timestamp_millis(conversation.created_at, context.imported_at); + for message in messages { + let Some(conversation) = conversations_by_id.get(&message.conversation_id) else { + result.summary.failed += 1; + result.summary.failures.push(ProviderImportFailure { + line: message.sequence_id.max(0) as usize, + error: format!( + "Shelley message {} references missing conversation {}", + message.message_id, message.conversation_id + ), + }); + continue; + }; + seen_message_conversations.insert(message.conversation_id.clone()); + let started_at = shelley_timestamp(conversation.created_at.as_deref(), context.imported_at); let ended_at = conversation .updated_at - .map(|timestamp| provider_timestamp_millis(Some(timestamp), context.imported_at)); - let content = provider_json_text(&conversation.content); - if let Value::Array(items) = &content { - for (index, item) in items.iter().enumerate() { - if let Some(checkpoint) = astrbot_checkpoint_id(item) { - checkpoint_sessions.insert(checkpoint, provider_session_id.clone()); - continue; - } - let role = astrbot_role(item); - let text = astrbot_item_text(item) - .unwrap_or_else(|| "AstrBot conversation item".to_owned()); - let event = native_event(NativeEventDraft { - provider: CaptureProvider::AstrBot, - source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, - provider_session_id: provider_session_id.clone(), - provider_event_index: index as u64, - provider_event_hash: astrbot_item_id(item) - .map(|id| format!("conversation:{id}")), - cursor: format!("conversation:{}:item:{index}", conversation.conversation_id), - event_type: EventType::Message, - role, - occurred_at: started_at, - text, - body: item.clone(), - metadata: json!({ - "source": "astrbot_conversations", - "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, - "conversation_id": conversation.conversation_id, - "inner_conversation_id": conversation.inner_conversation_id, - "item_index": index, - }), - }); - result.captures.push(( - index + 1, - astrbot_capture( - AstrBotCaptureDraft { - conversation, - provider_session_id: &provider_session_id, - started_at, - ended_at, - path, - user_version, - schema_fingerprint: &schema_fingerprint, - selected_conversation: selected_conversation.as_deref(), - event: Some(event), - }, - context, - ), - )); - } - } else { - let text = - provider_value_text(&content).unwrap_or_else(|| "AstrBot conversation".to_owned()); - let event = native_event(NativeEventDraft { - provider: CaptureProvider::AstrBot, - source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, - provider_session_id: provider_session_id.clone(), - provider_event_index: 0, - provider_event_hash: Some(format!("conversation-row:{}", conversation.row_id)), - cursor: format!("conversation:{}:content", conversation.conversation_id), - event_type: EventType::Message, - role: None, - occurred_at: started_at, - text, - body: content.clone(), - metadata: json!({ - "source": "astrbot_conversations", - "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, - "conversation_id": conversation.conversation_id, - }), - }); - result.captures.push(( - conversation.row_id.max(0) as usize, - astrbot_capture( - AstrBotCaptureDraft { - conversation, - provider_session_id: &provider_session_id, - started_at, - ended_at, - path, - user_version, - schema_fingerprint: &schema_fingerprint, - selected_conversation: selected_conversation.as_deref(), - event: Some(event), - }, - context, - ), - )); + .as_deref() + .map(|timestamp| shelley_timestamp(Some(timestamp), context.imported_at)); + let occurred_at = shelley_timestamp(message.created_at.as_deref(), started_at); + let body = shelley_message_body(&message); + let text = shelley_message_text(&message, &body) + .unwrap_or_else(|| format!("Shelley {} message", message.entry_type)); + let event_type = shelley_event_type(&message, &body); + let role = shelley_event_role(&message.entry_type); + let event = native_event(NativeEventDraft { + provider: CaptureProvider::Shelley, + source_format: SHELLEY_SQLITE_SOURCE_FORMAT, + provider_session_id: conversation.conversation_id.clone(), + provider_event_index: shelley_event_index(&message), + provider_event_hash: Some(message.message_id.clone()), + cursor: format!( + "conversation:{}:sequence:{}:message:{}", + message.conversation_id, message.sequence_id, message.message_id + ), + event_type, + role, + occurred_at, + text, + body, + metadata: json!({ + "source": "shelley_messages", + "source_format": SHELLEY_SQLITE_SOURCE_FORMAT, + "message_id": message.message_id, + "conversation_id": message.conversation_id, + "sequence_id": message.sequence_id, + "rowid": message.rowid, + "message_type": message.entry_type, + "generation": message.generation, + "excluded_from_context": message.excluded_from_context, + "usage": message.usage_data.as_deref().map(provider_json_text), + "llm_api_url": message.llm_api_url, + "model_name": message.model_name, + "forked_from_message_id": message.forked_from_message_id, + }), + }); + result.captures.push(( + message.rowid.max(0) as usize, + shelley_capture( + ShelleyCaptureDraft { + conversation, + started_at, + ended_at, + raw_source_path: &raw_source_path, + user_version, + schema_fingerprint: &schema_fingerprint, + event: Some(event), + }, + context, + ), + )); + } + + for conversation in conversations { + if seen_message_conversations.contains(&conversation.conversation_id) { + continue; } + let started_at = shelley_timestamp(conversation.created_at.as_deref(), context.imported_at); + let ended_at = conversation + .updated_at + .as_deref() + .map(|timestamp| shelley_timestamp(Some(timestamp), context.imported_at)); + result.captures.push(( + 0, + shelley_capture( + ShelleyCaptureDraft { + conversation: &conversation, + started_at, + ended_at, + raw_source_path: &raw_source_path, + user_version, + schema_fingerprint: &schema_fingerprint, + event: None, + }, + context, + ), + )); } - let conversations_by_id = conversations - .iter() - .map(|conversation| (astrbot_provider_session_id(conversation), conversation)) - .collect::>(); - for message in platform_messages { - let provider_session_id = message - .llm_checkpoint_id - .as_ref() - .and_then(|checkpoint| checkpoint_sessions.get(checkpoint)) - .cloned() - .unwrap_or_else(|| { - format!( - "platform/{}/{}", + Ok(result) +} + +struct ShelleyCaptureDraft<'a> { + conversation: &'a ShelleyConversationRow, + started_at: DateTime, + ended_at: Option>, + raw_source_path: &'a str, + user_version: i64, + schema_fingerprint: &'a str, + event: Option, +} + +fn shelley_capture( + draft: ShelleyCaptureDraft<'_>, + context: &ProviderAdapterContext, +) -> ProviderCaptureEnvelope { + let ShelleyCaptureDraft { + conversation, + started_at, + ended_at, + raw_source_path, + user_version, + schema_fingerprint, + event, + } = draft; + let is_subagent = conversation.parent_conversation_id.is_some() || !conversation.user_initiated; + let conversation_options = conversation + .conversation_options + .as_deref() + .map(provider_json_text) + .unwrap_or(Value::Null); + let tags = conversation + .tags + .as_deref() + .map(provider_json_text) + .unwrap_or(Value::Null); + let queued_messages = conversation + .queued_messages + .as_deref() + .map(provider_json_text) + .unwrap_or(Value::Null); + native_provider_capture( + NativeSessionDraft { + provider: CaptureProvider::Shelley, + source_format: SHELLEY_SQLITE_SOURCE_FORMAT, + provider_session_id: conversation.conversation_id.clone(), + parent_provider_session_id: conversation.parent_conversation_id.clone(), + root_provider_session_id: conversation.parent_conversation_id.clone(), + external_agent_id: None, + agent_type: if is_subagent { + AgentType::Subagent + } else { + AgentType::Primary + }, + role_hint: Some(if is_subagent { "subagent" } else { "primary" }.to_owned()), + is_primary: !is_subagent, + started_at, + ended_at, + cwd: conversation.cwd.clone(), + fidelity: Fidelity::Imported, + raw_source_path: raw_source_path.to_owned(), + trust: ProviderSourceTrust::ProviderNative, + source_metadata: json!({ + "adapter": SHELLEY_SQLITE_SOURCE_FORMAT, + "sqlite_user_version": user_version, + "schema_fingerprint": schema_fingerprint, + "source_path": raw_source_path, + }), + session_metadata: json!({ + "source_format": SHELLEY_SQLITE_SOURCE_FORMAT, + "conversation_id": conversation.conversation_id, + "slug": conversation.slug, + "title": conversation.slug, + "user_initiated": conversation.user_initiated, + "archived": conversation.archived, + "parent_conversation_id": conversation.parent_conversation_id, + "model": conversation.model, + "conversation_options": conversation_options, + "current_generation": conversation.current_generation, + "agent_working": conversation.agent_working, + "tags": tags, + "is_draft": conversation.is_draft, + "draft": conversation.draft, + "queued_messages": queued_messages, + }), + }, + context, + event, + ) +} + +fn shelley_conversations(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "conversations")? { + return Err(CaptureError::InvalidPayload( + "Shelley shelley.db is missing required conversations table".into(), + )); + } + let columns = sqlite_table_columns(conn, "conversations")?; + ensure_sqlite_table_columns( + &columns, + "Shelley conversations table", + &["conversation_id"], + )?; + let slug = optional_column_expr(&columns, "slug", "NULL"); + let user_initiated = optional_column_expr(&columns, "user_initiated", "1"); + let created_at = optional_column_expr(&columns, "created_at", "NULL"); + let updated_at = optional_column_expr(&columns, "updated_at", "NULL"); + let cwd = optional_column_expr(&columns, "cwd", "NULL"); + let archived = optional_column_expr(&columns, "archived", "0"); + let parent_conversation_id = optional_column_expr(&columns, "parent_conversation_id", "NULL"); + let model = optional_column_expr(&columns, "model", "NULL"); + let conversation_options = optional_column_expr(&columns, "conversation_options", "NULL"); + let current_generation = optional_column_expr(&columns, "current_generation", "NULL"); + let agent_working = optional_column_expr(&columns, "agent_working", "0"); + let tags = optional_column_expr(&columns, "tags", "NULL"); + let is_draft = optional_column_expr(&columns, "is_draft", "0"); + let draft = optional_column_expr(&columns, "draft", "NULL"); + let queued_messages = optional_column_expr(&columns, "queued_messages", "NULL"); + let sql = format!( + "select conversation_id, {slug}, {user_initiated}, {created_at}, {updated_at}, \ + {cwd}, {archived}, {parent_conversation_id}, {model}, {conversation_options}, \ + {current_generation}, {agent_working}, {tags}, {is_draft}, {draft}, \ + {queued_messages} \ + from conversations order by {created_at}, conversation_id" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(ShelleyConversationRow { + conversation_id: row.get(0)?, + slug: row.get(1)?, + user_initiated: sqlite_bool(row.get::<_, Option>(2)?), + created_at: row.get(3)?, + updated_at: row.get(4)?, + cwd: row.get(5)?, + archived: sqlite_bool(row.get::<_, Option>(6)?), + parent_conversation_id: row.get(7)?, + model: row.get(8)?, + conversation_options: row.get(9)?, + current_generation: row.get(10)?, + agent_working: sqlite_bool(row.get::<_, Option>(11)?), + tags: row.get(12)?, + is_draft: sqlite_bool(row.get::<_, Option>(13)?), + draft: row.get(14)?, + queued_messages: row.get(15)?, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +fn shelley_messages(conn: &Connection) -> Result> { + if !sqlite_table_exists(conn, "messages")? { + return Err(CaptureError::InvalidPayload( + "Shelley shelley.db is missing required messages table".into(), + )); + } + let columns = sqlite_table_columns(conn, "messages")?; + ensure_sqlite_table_columns( + &columns, + "Shelley messages table", + &["message_id", "conversation_id", "type"], + )?; + let sequence_id = optional_column_expr(&columns, "sequence_id", "rowid"); + let llm_data = optional_column_expr(&columns, "llm_data", "NULL"); + let user_data = optional_column_expr(&columns, "user_data", "NULL"); + let usage_data = optional_column_expr(&columns, "usage_data", "NULL"); + let created_at = optional_column_expr(&columns, "created_at", "NULL"); + let display_data = optional_column_expr(&columns, "display_data", "NULL"); + let excluded_from_context = optional_column_expr(&columns, "excluded_from_context", "0"); + let generation = optional_column_expr(&columns, "generation", "NULL"); + let llm_api_url = optional_column_expr(&columns, "llm_api_url", "NULL"); + let model_name = optional_column_expr(&columns, "model_name", "NULL"); + let forked_from_message_id = optional_column_expr(&columns, "forked_from_message_id", "NULL"); + let sql = format!( + "select rowid, message_id, conversation_id, {sequence_id}, type, {llm_data}, \ + {user_data}, {usage_data}, {created_at}, {display_data}, \ + {excluded_from_context}, {generation}, {llm_api_url}, {model_name}, \ + {forked_from_message_id} from messages order by conversation_id, {sequence_id}, rowid" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(ShelleyMessageRow { + rowid: row.get(0)?, + message_id: row.get(1)?, + conversation_id: row.get(2)?, + sequence_id: row.get(3)?, + entry_type: row.get(4)?, + llm_data: row.get(5)?, + user_data: row.get(6)?, + usage_data: row.get(7)?, + created_at: row.get(8)?, + display_data: row.get(9)?, + excluded_from_context: sqlite_bool(row.get::<_, Option>(10)?), + generation: row.get(11)?, + llm_api_url: row.get(12)?, + model_name: row.get(13)?, + forked_from_message_id: row.get(14)?, + }) + })?; + rows.collect::, _>>() + .map_err(CaptureError::from) +} + +fn sqlite_bool(value: Option) -> bool { + value.unwrap_or(0) != 0 +} + +fn shelley_timestamp(raw: Option<&str>, fallback: DateTime) -> DateTime { + let Some(raw) = raw.map(str::trim).filter(|raw| !raw.is_empty()) else { + return fallback; + }; + parse_rfc3339_utc(raw) + .or_else(|| { + NaiveDateTime::parse_from_str(raw, "%Y-%m-%d %H:%M:%S%.f") + .ok() + .map(|naive| DateTime::::from_naive_utc_and_offset(naive, Utc)) + }) + .unwrap_or(fallback) +} + +fn shelley_message_body(message: &ShelleyMessageRow) -> Value { + json!({ + "message_id": message.message_id, + "conversation_id": message.conversation_id, + "sequence_id": message.sequence_id, + "type": message.entry_type, + "llm_data": message.llm_data.as_deref().map(provider_json_text), + "user_data": message.user_data.as_deref().map(provider_json_text), + "display_data": message.display_data.as_deref().map(provider_json_text), + "usage_data": message.usage_data.as_deref().map(provider_json_text), + }) +} + +fn shelley_message_text(message: &ShelleyMessageRow, body: &Value) -> Option { + let mut parts = Vec::new(); + for pointer in ["/user_data", "/llm_data", "/display_data"] { + if let Some(text) = body.pointer(pointer).and_then(shelley_value_text) { + parts.push(text); + } + } + if parts.is_empty() && message.entry_type == "system" { + Some("Shelley system message".to_owned()) + } else if parts.is_empty() { + None + } else { + Some(parts.join("\n")) + } +} + +fn shelley_event_role(entry_type: &str) -> Option { + Some(match entry_type { + "user" => EventRole::User, + "agent" | "assistant" => EventRole::Assistant, + "tool" => EventRole::Tool, + "system" | "error" | "gitinfo" | "warning" | "modelchange" => EventRole::System, + _ => EventRole::Unknown, + }) +} + +fn shelley_event_type(message: &ShelleyMessageRow, body: &Value) -> EventType { + match message.entry_type.as_str() { + "tool" => EventType::ToolOutput, + "gitinfo" => EventType::VcsChange, + "system" | "error" | "warning" | "modelchange" => EventType::Notice, + "agent" | "assistant" if shelley_value_has_tool_use(body) => EventType::ToolCall, + "user" | "agent" | "assistant" if shelley_value_has_tool_result(body) => { + EventType::ToolOutput + } + "user" | "agent" | "assistant" => EventType::Message, + _ => EventType::Notice, + } +} + +fn shelley_event_index(message: &ShelleyMessageRow) -> u64 { + let sequence = message.sequence_id.max(0) as u64; + let bucket = text_id_index( + &format!("{}:{}", message.conversation_id, message.message_id), + 4_096, + ); + sequence.saturating_mul(4_096).saturating_add(bucket) +} + +fn shelley_value_has_tool_use(value: &Value) -> bool { + match value { + Value::Array(items) => items.iter().any(shelley_value_has_tool_use), + Value::Object(object) => { + let content_type = shelley_content_type(value); + matches!( + content_type.as_deref(), + Some("tool_use" | "server_tool_use") + ) || object.values().any(shelley_value_has_tool_use) + } + _ => false, + } +} + +fn shelley_value_has_tool_result(value: &Value) -> bool { + match value { + Value::Array(items) => items.iter().any(shelley_value_has_tool_result), + Value::Object(object) => { + let content_type = shelley_content_type(value); + matches!( + content_type.as_deref(), + Some("tool_result" | "web_search_tool_result" | "web_search_result") + ) || object.values().any(shelley_value_has_tool_result) + } + _ => false, + } +} + +fn shelley_value_text(value: &Value) -> Option { + let mut parts = Vec::new(); + shelley_collect_text(value, &mut parts); + (!parts.is_empty()).then(|| parts.join("\n")) +} + +fn shelley_collect_text(value: &Value, parts: &mut Vec) { + match value { + Value::String(text) => shelley_push_text(parts, text), + Value::Array(items) => { + for item in items { + if shelley_text_budget_remaining(parts) == 0 { + break; + } + shelley_collect_text(item, parts); + } + } + Value::Object(object) => { + if let Some(kind) = shelley_content_type(value) { + let handled = match kind.as_str() { + "text" => { + if let Some(text) = object.get("Text").and_then(Value::as_str) { + shelley_push_text(parts, text); + } + true + } + "thinking" | "redacted_thinking" => { + if let Some(text) = object.get("Thinking").and_then(Value::as_str) { + shelley_push_text(parts, text); + } + true + } + "tool_use" | "server_tool_use" => { + let name = object + .get("ToolName") + .and_then(Value::as_str) + .unwrap_or("tool"); + shelley_push_text(parts, &format!("tool call: {name}")); + if let Some(input) = object.get("ToolInput") { + if !input.is_null() { + let input = provider_capped_json(input, PROVIDER_MAX_PREVIEW_CHARS); + shelley_push_text(parts, &format!("tool input: {input}")); + } + } + true + } + "tool_result" | "web_search_tool_result" => { + shelley_push_text(parts, "tool result"); + if let Some(results) = object.get("ToolResult") { + shelley_collect_text(results, parts); + } + if let Some(display) = object.get("Display") { + shelley_collect_text(display, parts); + } + true + } + "web_search_result" => { + for key in ["Title", "URL", "PageAge"] { + if let Some(text) = object.get(key).and_then(Value::as_str) { + shelley_push_text(parts, text); + } + } + true + } + _ => false, + }; + if handled { + return; + } + } + + for key in [ + "Text", + "text", + "Thinking", + "thinking", + "content", + "Content", + "output", + "Output", + "summary", + "Summary", + "message", + "Message", + "error", + "Error", + "LLMContent", + "ToolResult", + "Display", + ] { + if shelley_text_budget_remaining(parts) == 0 { + break; + } + if let Some(child) = object.get(key) { + shelley_collect_text(child, parts); + } + } + } + Value::Number(_) | Value::Bool(_) | Value::Null => {} + } +} + +fn shelley_push_text(parts: &mut Vec, text: &str) { + let text = text.trim(); + if !text.is_empty() { + let remaining = shelley_text_budget_remaining(parts); + if remaining == 0 { + return; + } + let separator_budget = usize::from(!parts.is_empty()); + if remaining <= separator_budget { + return; + } + let (text, _) = capped_text(text, remaining - separator_budget); + parts.push(text); + } +} + +fn shelley_text_budget_remaining(parts: &[String]) -> usize { + let used = parts.iter().map(|part| part.chars().count()).sum::() + + parts.len().saturating_sub(1); + (PROVIDER_MAX_TEXT_CHARS + 1).saturating_sub(used) +} + +fn shelley_content_type(value: &Value) -> Option { + let raw = value.get("Type")?; + if let Some(text) = raw.as_str() { + let normalized = text.trim().to_ascii_lowercase(); + return match normalized.as_str() { + "contenttypetext" => Some("text".to_owned()), + "contenttypethinking" => Some("thinking".to_owned()), + "contenttyperedactedthinking" => Some("redacted_thinking".to_owned()), + "contenttypetooluse" => Some("tool_use".to_owned()), + "contenttypetoolresult" => Some("tool_result".to_owned()), + "contenttypeservertooluse" => Some("server_tool_use".to_owned()), + "contenttypewebsearchtoolresult" => Some("web_search_tool_result".to_owned()), + "contenttypewebsearchresult" => Some("web_search_result".to_owned()), + _ => Some(normalized), + }; + } + raw.as_i64().and_then(|kind| { + match kind { + 2 => Some("text"), + 3 => Some("thinking"), + 4 => Some("redacted_thinking"), + 5 => Some("tool_use"), + 6 => Some("tool_result"), + 7 => Some("server_tool_use"), + 8 => Some("web_search_tool_result"), + 9 => Some("web_search_result"), + _ => None, + } + .map(str::to_owned) + }) +} + +#[derive(Debug, Clone)] +struct AstrBotConversationRow { + row_id: i64, + inner_conversation_id: Option, + conversation_id: String, + platform_id: Option, + user_id: Option, + content: String, + title: Option, + persona_id: Option, + token_usage: Option, + created_at: Option, + updated_at: Option, +} + +#[derive(Debug, Clone)] +struct AstrBotPlatformMessageRow { + id: i64, + platform_id: Option, + user_id: Option, + sender_id: Option, + sender_name: Option, + content: Option, + llm_checkpoint_id: Option, + created_at: Option, +} + +fn normalize_astrbot_sqlite( + path: &Path, + context: &ProviderAdapterContext, +) -> Result { + let conn = open_provider_sqlite_readonly(path)?; + let user_version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?; + let schema_fingerprint = opencode_schema_fingerprint(&conn)?; + let conversations = astrbot_conversations(&conn)?; + let platform_messages = astrbot_platform_messages(&conn)?; + let selected_conversation = astrbot_selected_conversation(&conn).ok().flatten(); + let mut result = ProviderNormalizationResult::default(); + let mut checkpoint_sessions = BTreeMap::::new(); + + for conversation in &conversations { + let provider_session_id = astrbot_provider_session_id(conversation); + let started_at = provider_timestamp_millis(conversation.created_at, context.imported_at); + let ended_at = conversation + .updated_at + .map(|timestamp| provider_timestamp_millis(Some(timestamp), context.imported_at)); + let content = provider_json_text(&conversation.content); + if let Value::Array(items) = &content { + for (index, item) in items.iter().enumerate() { + if let Some(checkpoint) = astrbot_checkpoint_id(item) { + checkpoint_sessions.insert(checkpoint, provider_session_id.clone()); + continue; + } + let role = astrbot_role(item); + let text = astrbot_item_text(item) + .unwrap_or_else(|| "AstrBot conversation item".to_owned()); + let event = native_event(NativeEventDraft { + provider: CaptureProvider::AstrBot, + source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + provider_event_index: index as u64, + provider_event_hash: astrbot_item_id(item) + .map(|id| format!("conversation:{id}")), + cursor: format!("conversation:{}:item:{index}", conversation.conversation_id), + event_type: EventType::Message, + role, + occurred_at: started_at, + text, + body: item.clone(), + metadata: json!({ + "source": "astrbot_conversations", + "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, + "conversation_id": conversation.conversation_id, + "inner_conversation_id": conversation.inner_conversation_id, + "item_index": index, + }), + }); + result.captures.push(( + index + 1, + astrbot_capture( + AstrBotCaptureDraft { + conversation, + provider_session_id: &provider_session_id, + started_at, + ended_at, + path, + user_version, + schema_fingerprint: &schema_fingerprint, + selected_conversation: selected_conversation.as_deref(), + event: Some(event), + }, + context, + ), + )); + } + } else { + let text = + provider_value_text(&content).unwrap_or_else(|| "AstrBot conversation".to_owned()); + let event = native_event(NativeEventDraft { + provider: CaptureProvider::AstrBot, + source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, + provider_session_id: provider_session_id.clone(), + provider_event_index: 0, + provider_event_hash: Some(format!("conversation-row:{}", conversation.row_id)), + cursor: format!("conversation:{}:content", conversation.conversation_id), + event_type: EventType::Message, + role: None, + occurred_at: started_at, + text, + body: content.clone(), + metadata: json!({ + "source": "astrbot_conversations", + "source_format": ASTRBOT_SQLITE_SOURCE_FORMAT, + "conversation_id": conversation.conversation_id, + }), + }); + result.captures.push(( + conversation.row_id.max(0) as usize, + astrbot_capture( + AstrBotCaptureDraft { + conversation, + provider_session_id: &provider_session_id, + started_at, + ended_at, + path, + user_version, + schema_fingerprint: &schema_fingerprint, + selected_conversation: selected_conversation.as_deref(), + event: Some(event), + }, + context, + ), + )); + } + } + + let conversations_by_id = conversations + .iter() + .map(|conversation| (astrbot_provider_session_id(conversation), conversation)) + .collect::>(); + for message in platform_messages { + let provider_session_id = message + .llm_checkpoint_id + .as_ref() + .and_then(|checkpoint| checkpoint_sessions.get(checkpoint)) + .cloned() + .unwrap_or_else(|| { + format!( + "platform/{}/{}", message.platform_id.as_deref().unwrap_or("unknown"), message.user_id.as_deref().unwrap_or("unknown") ) @@ -13317,6 +14015,297 @@ mod tests { .contains("OpenCode SQLite message table missing required column(s): data")); } + #[test] + fn native_shelley_imports_sessions_messages_metadata_and_citations() { + let temp = tempdir(); + let fixture = write_shelley_smoke_db(&temp); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_shelley_sqlite( + &fixture, + &mut store, + ShelleySqliteImportOptions { + machine_id: "test-machine".into(), + source_path: Some(fixture.clone()), + imported_at: DateTime::parse_from_rfc3339("2026-06-24T12:00:00Z") + .unwrap() + .with_timezone(&Utc), + allow_partial_failures: true, + ..ShelleySqliteImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_sessions, 3); + assert_eq!(summary.imported_events, 4); + assert_eq!(summary.imported_edges, 1); + + let parent_id = provider_session_uuid(CaptureProvider::Shelley, "shelley-root"); + let child_id = provider_session_uuid(CaptureProvider::Shelley, "shelley-child"); + assert_eq!( + store.get_session(child_id).unwrap().parent_session_id, + Some(parent_id) + ); + assert!(store + .get_session(parent_id) + .unwrap() + .sync + .metadata + .to_string() + .contains("queued oracle")); + + let source = store + .capture_source_by_external_session(CaptureProvider::Shelley, "shelley-root") + .unwrap() + .unwrap(); + assert_eq!( + source.descriptor.raw_source_path.as_deref(), + fixture.to_str() + ); + assert_eq!(source.descriptor.provider, CaptureProvider::Shelley); + + let events = store.events_for_session(parent_id).unwrap(); + assert_eq!(events.len(), 3); + let agent_event = events + .iter() + .find(|event| { + event.sync.metadata["metadata"]["message_id"].as_str() == Some("msg-agent") + }) + .expect("Shelley agent event imported"); + let tool_result_event = events + .iter() + .find(|event| { + event.sync.metadata["metadata"]["message_id"].as_str() == Some("msg-tool-result") + }) + .expect("Shelley tool-result event imported"); + assert_eq!(agent_event.event_type, EventType::ToolCall); + assert_eq!(tool_result_event.event_type, EventType::ToolOutput); + let rendered = serde_json::to_string(&events).unwrap(); + assert!(rendered.contains("shelley search oracle")); + assert!(rendered.contains("thinking through the search")); + assert!(rendered.contains("tool call: bash")); + assert!(rendered.contains("tool output oracle")); + assert!(rendered.contains("claude-opus-4-7")); + assert!(rendered.contains("https://api.anthropic.com/v1/messages")); + let user_event = events + .iter() + .find(|event| { + event.sync.metadata["metadata"]["message_id"].as_str() == Some("msg-user") + }) + .expect("Shelley user event imported"); + assert!(user_event + .sync + .metadata + .to_string() + .contains("conversation:shelley-root:sequence:1:message:msg-user")); + + let cursor = store + .get_sync_cursor( + None, + "test-machine", + &provider_cursor_stream(CaptureProvider::Shelley, SHELLEY_SQLITE_SOURCE_FORMAT), + ) + .unwrap() + .unwrap(); + assert!(cursor + .cursor + .contains("conversation:shelley-root:sequence:3:message:msg-tool-result")); + } + + #[test] + fn native_shelley_reimport_is_idempotent() { + let temp = tempdir(); + let fixture = write_shelley_smoke_db(&temp); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let first = import_shelley_sqlite( + &fixture, + &mut store, + ShelleySqliteImportOptions { + allow_partial_failures: true, + ..ShelleySqliteImportOptions::default() + }, + ) + .unwrap(); + assert_eq!(first.imported_events, 4); + + let second = import_shelley_sqlite( + &fixture, + &mut store, + ShelleySqliteImportOptions { + allow_partial_failures: true, + ..ShelleySqliteImportOptions::default() + }, + ) + .unwrap(); + assert_eq!(second.failed, 0, "{:?}", second.failures); + assert_eq!(second.imported_sessions, 0); + assert_eq!(second.imported_events, 0); + assert_eq!(second.imported_edges, 0); + assert_eq!(second.skipped_sessions, 3); + assert_eq!(second.skipped_events, 4); + assert_eq!(second.skipped_edges, 1); + } + + #[test] + fn native_shelley_handles_duplicate_sequences_and_nonchat_rows() { + let temp = tempdir(); + let fixture = write_shelley_adversarial_db(&temp); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_shelley_sqlite( + &fixture, + &mut store, + ShelleySqliteImportOptions { + allow_partial_failures: true, + ..ShelleySqliteImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0, "{:?}", summary.failures); + assert_eq!(summary.imported_sessions, 1); + assert_eq!(summary.imported_events, 5); + + let session_id = provider_session_uuid(CaptureProvider::Shelley, "shelley-adversarial"); + let events = store.events_for_session(session_id).unwrap(); + assert_eq!(events.len(), 5); + assert_eq!( + events + .iter() + .map(|event| event.id) + .collect::>() + .len(), + 5 + ); + let rendered = serde_json::to_string(&events).unwrap(); + assert!(rendered.contains("duplicate sequence first")); + assert!(rendered.contains("duplicate sequence second")); + assert!(events + .iter() + .any(|event| event.event_type == EventType::VcsChange)); + assert!(events + .iter() + .any( + |event| event.sync.metadata["metadata"]["message_type"].as_str() == Some("warning") + )); + + let large = events + .iter() + .find(|event| { + event.sync.metadata["metadata"]["message_id"].as_str() == Some("msg-large") + }) + .expect("large Shelley event imported"); + assert_eq!(large.payload["body"]["truncated"].as_bool(), Some(true)); + assert!( + large.payload["body"]["text"] + .as_str() + .unwrap() + .chars() + .count() + <= PROVIDER_MAX_TEXT_CHARS + ); + } + + #[test] + fn native_shelley_text_extraction_is_not_duplicate_or_unbounded() { + let text = shelley_value_text(&json!({ + "Content": [ + {"Type": 2, "Text": "once"} + ] + })) + .unwrap(); + assert_eq!(text, "once"); + + let huge = "x".repeat(PROVIDER_MAX_TEXT_CHARS + 200); + let text = shelley_value_text(&json!({ + "Content": [ + {"Type": 2, "Text": huge}, + {"Type": 2, "Text": "after cap"} + ] + })) + .unwrap(); + assert_eq!(text.chars().count(), PROVIDER_MAX_TEXT_CHARS + 1); + assert!(!text.contains("after cap")); + } + + #[test] + fn native_shelley_event_index_uses_stable_message_identity() { + let message = ShelleyMessageRow { + rowid: 1, + message_id: "msg-stable".to_owned(), + conversation_id: "conv-stable".to_owned(), + sequence_id: 42, + entry_type: "user".to_owned(), + llm_data: None, + user_data: None, + usage_data: None, + created_at: None, + display_data: None, + excluded_from_context: false, + generation: None, + llm_api_url: None, + model_name: None, + forked_from_message_id: None, + }; + let mut moved_row = message.clone(); + moved_row.rowid = 999; + let mut duplicate_sequence = message.clone(); + duplicate_sequence.message_id = "msg-stable-other".to_owned(); + + assert_eq!( + shelley_event_index(&message), + shelley_event_index(&moved_row) + ); + assert_ne!( + shelley_event_index(&message), + shelley_event_index(&duplicate_sequence) + ); + } + + #[test] + fn native_shelley_reports_malformed_and_corrupt_db() { + let temp = tempdir(); + let malformed = write_shelley_malformed_db(&temp); + let corrupt = temp.path().join("corrupt-shelley.db"); + fs::write(&corrupt, b"not sqlite").unwrap(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let err = import_shelley_sqlite( + &malformed, + &mut store, + ShelleySqliteImportOptions::default(), + ) + .unwrap_err(); + assert!(err + .to_string() + .contains("Shelley messages table missing required column(s): type")); + + let err = + import_shelley_sqlite(&corrupt, &mut store, ShelleySqliteImportOptions::default()) + .unwrap_err(); + assert!(err.to_string().contains("not a database")); + } + + #[test] + fn provider_sources_discovers_shelley_default_db() { + let temp = tempdir(); + let db = temp.path().join(".config/shelley/shelley.db"); + fs::create_dir_all(db.parent().unwrap()).unwrap(); + fs::write(&db, b"not inspected by source probe").unwrap(); + + let sources = discover_provider_sources_for_provider(temp.path(), CaptureProvider::Shelley); + let source = sources + .iter() + .find(|source| source.source_format == SHELLEY_SQLITE_SOURCE_FORMAT) + .unwrap_or_else(|| panic!("missing Shelley source in {sources:#?}")); + assert_eq!(source.provider, CaptureProvider::Shelley); + assert_eq!(source.status, ProviderSourceStatus::Available); + assert_eq!(source.import_support, ProviderImportSupport::Native); + assert_eq!(source.path, db); + } + #[test] fn native_jsonl_tree_imports_gemini_droid_and_copilot_smokes() { let temp = tempdir(); @@ -13629,6 +14618,253 @@ mod tests { path } + fn write_shelley_smoke_db(temp: &TempDir) -> PathBuf { + let path = temp.path().join("shelley.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "create table conversations ( + conversation_id text primary key, + slug text, + user_initiated boolean not null default true, + created_at datetime not null default current_timestamp, + updated_at datetime not null default current_timestamp, + cwd text, + archived boolean not null default false, + parent_conversation_id text, + model text, + conversation_options text not null default '{}', + current_generation integer not null default 1, + agent_working boolean not null default false, + tags text not null default '[]', + is_draft boolean not null default false, + draft text not null default '', + queued_messages text not null default '[]' + ); + create table messages ( + message_id text primary key, + conversation_id text not null, + sequence_id integer not null, + type text not null, + llm_data text, + user_data text, + usage_data text, + created_at datetime not null default current_timestamp, + display_data text, + excluded_from_context boolean not null default false, + generation integer not null default 1, + llm_api_url text, + model_name text, + forked_from_message_id text + );", + ) + .unwrap(); + conn.execute( + "insert into conversations values ( + 'shelley-root', 'root-slug', 1, '2026-06-24 12:00:00', + '2026-06-24 12:05:00', '/workspace/shelley', 0, null, + 'claude-opus-4-7', ?1, 2, 0, ?2, 0, '', ?3 + )", + [ + r#"{"thinking_level":"high","subagent_backend":"shelley"}"#, + r#"["native","ctx"]"#, + r#"[{"id":"queued-1","llm":{"Content":[{"Type":2,"Text":"queued oracle"}]},"created_at":"2026-06-24T12:00:04Z","model":"claude-opus-4-7"}]"#, + ], + ) + .unwrap(); + conn.execute( + "insert into conversations values ( + 'shelley-child', 'child-slug', 0, '2026-06-24 12:01:00', + '2026-06-24 12:02:00', '/workspace/shelley', 0, 'shelley-root', + 'claude-sonnet-4-5', '{}', 1, 0, '[]', 0, '', '[]' + )", + [], + ) + .unwrap(); + conn.execute( + "insert into conversations values ( + 'shelley-draft', 'old-draft', 1, '2026-06-24 11:00:00', + '2026-06-24 11:01:00', '/workspace/archive', 1, null, + null, '{}', 1, 0, '[]', 1, 'draft body', '[]' + )", + [], + ) + .unwrap(); + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, user_data, created_at + ) values ('msg-user', 'shelley-root', 1, 'user', ?1, '2026-06-24 12:00:01')", + [json!({ + "Content": [ + {"Type": 2, "Text": "please run shelley search oracle"} + ] + }) + .to_string()], + ) + .unwrap(); + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, llm_data, usage_data, + created_at, generation, llm_api_url, model_name + ) values ( + 'msg-agent', 'shelley-root', 2, 'agent', ?1, ?2, + '2026-06-24 12:00:02', 2, 'https://api.anthropic.com/v1/messages', + 'claude-opus-4-7' + )", + [ + json!({ + "Role": 1, + "Content": [ + {"Type": 3, "Thinking": "thinking through the search"}, + {"Type": 2, "Text": "I will inspect the source."}, + {"Type": 5, "ID": "toolu_1", "ToolName": "bash", "ToolInput": {"command": "rg shelley"}} + ], + "EndOfTurn": false + }) + .to_string(), + json!({ + "input_tokens": 100, + "cache_read_input_tokens": 25, + "output_tokens": 40, + "cost_usd": 0.0123, + "model": "claude-opus-4-7", + "url": "https://api.anthropic.com/v1/messages" + }) + .to_string(), + ], + ) + .unwrap(); + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, user_data, display_data, + created_at, forked_from_message_id + ) values ( + 'msg-tool-result', 'shelley-root', 3, 'user', ?1, ?2, + '2026-06-24 12:00:03', 'source-msg-tool-result' + )", + [ + json!({ + "Role": 0, + "Content": [ + {"Type": 6, "ToolUseID": "toolu_1", "ToolResult": [{"Type": 2, "Text": "tool output oracle"}]} + ] + }) + .to_string(), + json!({"stdout": "tool output oracle", "exit_code": 0}).to_string(), + ], + ) + .unwrap(); + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, llm_data, created_at + ) values ('msg-child', 'shelley-child', 1, 'agent', ?1, '2026-06-24 12:01:01')", + [json!({ + "Content": [ + {"Type": 2, "Text": "subagent result from Shelley"} + ] + }) + .to_string()], + ) + .unwrap(); + path + } + + fn write_shelley_adversarial_db(temp: &TempDir) -> PathBuf { + let path = temp.path().join("shelley-adversarial.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "create table conversations ( + conversation_id text primary key, + slug text, + user_initiated boolean not null default true, + created_at datetime not null default current_timestamp, + updated_at datetime not null default current_timestamp, + cwd text, + archived boolean not null default false, + parent_conversation_id text, + model text, + conversation_options text not null default '{}', + current_generation integer not null default 1, + agent_working boolean not null default false, + tags text not null default '[]', + is_draft boolean not null default false, + draft text not null default '', + queued_messages text not null default '[]' + ); + create table messages ( + message_id text primary key, + conversation_id text not null, + sequence_id integer not null, + type text not null, + llm_data text, + user_data text, + usage_data text, + created_at datetime not null default current_timestamp, + display_data text, + excluded_from_context boolean not null default false, + generation integer not null default 1, + llm_api_url text, + model_name text, + forked_from_message_id text + );", + ) + .unwrap(); + conn.execute( + "insert into conversations values ( + 'shelley-adversarial', 'adversarial', 1, '2026-06-24 12:00:00', + '2026-06-24 12:05:00', '/workspace/shelley', 0, null, + 'claude-opus-4-7', '{}', 1, 0, '[]', 0, '', '[]' + )", + [], + ) + .unwrap(); + for (message_id, sequence_id, message_type, text) in [ + ("msg-dup-a", 1, "user", "duplicate sequence first"), + ("msg-dup-b", 1, "user", "duplicate sequence second"), + ("msg-git", 2, "gitinfo", "commit abc touched shelley.rs"), + ("msg-warning", 3, "warning", "warning message for Shelley"), + ] { + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, user_data, created_at + ) values (?1, 'shelley-adversarial', ?2, ?3, ?4, '2026-06-24 12:00:01')", + rusqlite::params![ + message_id, + sequence_id, + message_type, + json!({"Content": [{"Type": 2, "Text": text}]}).to_string(), + ], + ) + .unwrap(); + } + conn.execute( + "insert into messages ( + message_id, conversation_id, sequence_id, type, llm_data, created_at + ) values ('msg-large', 'shelley-adversarial', 4, 'agent', ?1, '2026-06-24 12:00:04')", + [json!({ + "Content": [ + {"Type": 2, "Text": "x".repeat(PROVIDER_MAX_TEXT_CHARS + 200)} + ] + }) + .to_string()], + ) + .unwrap(); + path + } + + fn write_shelley_malformed_db(temp: &TempDir) -> PathBuf { + let path = temp.path().join("shelley-malformed.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "create table conversations (conversation_id text primary key); + create table messages ( + message_id text primary key, + conversation_id text not null + );", + ) + .unwrap(); + path + } + fn write_gemini_smoke_fixture(temp: &TempDir) -> PathBuf { let chats = temp.path().join("gemini/.gemini/tmp/project/chats"); let child_dir = chats.join("gemini-root"); diff --git a/crates/ctx-history-capture/src/provider_sources.rs b/crates/ctx-history-capture/src/provider_sources.rs index 13711a733..27da1b859 100644 --- a/crates/ctx-history-capture/src/provider_sources.rs +++ b/crates/ctx-history-capture/src/provider_sources.rs @@ -184,6 +184,12 @@ const ASTRBOT_DEFAULTS: &[ProviderDefaultLocation] = &[ProviderDefaultLocation { source_kind: ProviderSourceKind::NativeHistory, }]; +const SHELLEY_DEFAULTS: &[ProviderDefaultLocation] = &[ProviderDefaultLocation { + path_components: &[".config", "shelley", "shelley.db"], + source_format: "shelley_sqlite", + source_kind: ProviderSourceKind::NativeHistory, +}]; + const PROVIDER_SPECS: &[ProviderSourceSpec] = &[ ProviderSourceSpec { provider: CaptureProvider::Codex, @@ -315,6 +321,16 @@ const PROVIDER_SPECS: &[ProviderSourceSpec] = &[ redaction_boundary: ProviderRedactionBoundary::BeforeExport, unsupported_reason: None, }, + ProviderSourceSpec { + provider: CaptureProvider::Shelley, + display_name: "Shelley", + default_locations: SHELLEY_DEFAULTS, + import_support: ProviderImportSupport::Native, + catalog_support: ProviderCatalogSupport::None, + raw_retention: ProviderRawRetention::PathReference, + redaction_boundary: ProviderRedactionBoundary::BeforeExport, + unsupported_reason: None, + }, ]; pub fn provider_source_specs() -> &'static [ProviderSourceSpec] { @@ -417,6 +433,16 @@ fn discover_provider_sources_for_spec( )); } } + CaptureProvider::Shelley => { + if let Some(path) = env_path("SHELLEY_DB") { + sources.push(provider_source_from_parts( + spec, + path, + "shelley_sqlite", + ProviderSourceKind::NativeHistory, + )); + } + } _ => {} } @@ -502,6 +528,7 @@ pub fn provider_source_for_path(provider: CaptureProvider, path: PathBuf) -> Pro CaptureProvider::Hermes => "hermes_state_sqlite", CaptureProvider::NanoClaw => "nanoclaw_project", CaptureProvider::AstrBot => "astrbot_data_v4_sqlite", + CaptureProvider::Shelley => "shelley_sqlite", _ => "unsupported", }; let explicit_import_support = spec.import_support; @@ -614,6 +641,7 @@ fn empty_source_reason(provider: CaptureProvider) -> Option<&'static str> { Some("path exists but no NanoClaw data/v2.db and data/v2-sessions store was found") } CaptureProvider::AstrBot => Some("path exists but no AstrBot data/data_v4.db was found"), + CaptureProvider::Shelley => Some("path exists but no Shelley SQLite database was found"), _ => None, } } @@ -692,6 +720,9 @@ fn probe_io_error_reason(provider: CaptureProvider) -> Option<&'static str> { CaptureProvider::AstrBot => { Some("path exists but the AstrBot data database could not be read; check permissions") } + CaptureProvider::Shelley => { + Some("path exists but the Shelley database could not be read; check permissions") + } _ => None, } } @@ -713,6 +744,7 @@ fn default_location_import_probe( CaptureProvider::Hermes => path_is_file_probe(path), CaptureProvider::NanoClaw => has_nanoclaw_project(path), CaptureProvider::AstrBot => path_is_file_probe(path), + CaptureProvider::Shelley => path_is_file_probe(path), CaptureProvider::Antigravity => has_jsonl_file_under_matching(path, 10_000, |candidate| { matches!( candidate.file_name().and_then(|name| name.to_str()), @@ -1071,6 +1103,17 @@ mod tests { ); assert!(astrbot_source.import_support.is_importable()); assert!(!astrbot_source.import_support.is_auto_importable()); + + let shelley = temp.path().join(".config/shelley"); + std::fs::create_dir_all(&shelley).unwrap(); + std::fs::write(shelley.join("shelley.db"), b"sqlite fixture marker").unwrap(); + let shelley_source = discover_provider_sources(temp.path()) + .into_iter() + .find(|source| source.provider == CaptureProvider::Shelley) + .unwrap(); + assert_eq!(shelley_source.status, ProviderSourceStatus::Available); + assert_eq!(shelley_source.import_support, ProviderImportSupport::Native); + assert!(shelley_source.import_support.is_auto_importable()); } #[test] diff --git a/crates/ctx-history-core/src/lib.rs b/crates/ctx-history-core/src/lib.rs index 6a0d7663a..3ed7e2bef 100644 --- a/crates/ctx-history-core/src/lib.rs +++ b/crates/ctx-history-core/src/lib.rs @@ -195,6 +195,7 @@ text_enum! { Hermes => "hermes", NanoClaw => "nanoclaw", AstrBot => "astrbot", + Shelley => "shelley", Shell => "shell", Git => "git", Jj => "jj", diff --git a/crates/ctx-history-core/src/provider.rs b/crates/ctx-history-core/src/provider.rs index 5010d3b6e..699b2bfc1 100644 --- a/crates/ctx-history-core/src/provider.rs +++ b/crates/ctx-history-core/src/provider.rs @@ -59,6 +59,7 @@ pub enum ProviderId { NanoClaw, #[serde(rename = "astrbot", alias = "astr_bot")] AstrBot, + Shelley, Goose, #[serde(rename = "openhands")] OpenHands, @@ -76,7 +77,7 @@ pub enum ProviderId { } impl ProviderId { - pub const ALL: [Self; 31] = [ + pub const ALL: [Self; 32] = [ Self::Codex, Self::ClaudeCode, Self::ClaudeCliCrp, @@ -95,6 +96,7 @@ impl ProviderId { Self::Hermes, Self::NanoClaw, Self::AstrBot, + Self::Shelley, Self::Goose, Self::OpenHands, Self::Cagent, @@ -416,6 +418,7 @@ mod tests { ProviderId::OpenCode, ProviderId::OpenClaw, ProviderId::Pi, + ProviderId::Shelley, ] .into_iter() .collect::>(); diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 2537851ac..46554dd28 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -476,7 +476,7 @@ const CREATE_TABLES_SQL: &str = r#" CREATE TABLE IF NOT EXISTS capture_sources ( id TEXT PRIMARY KEY NOT NULL, kind TEXT NOT NULL CHECK (kind IN ('provider_import', 'provider_hook', 'direct_cli', 'manual')), - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shelley', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), machine_id TEXT NOT NULL, process_id INTEGER, cwd TEXT, @@ -493,7 +493,7 @@ CREATE TABLE IF NOT EXISTS capture_sources ( CREATE TABLE IF NOT EXISTS catalog_sessions ( source_path TEXT PRIMARY KEY NOT NULL, - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shelley', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, external_session_id TEXT, @@ -522,7 +522,7 @@ CREATE TABLE IF NOT EXISTS catalog_sessions ( ); CREATE TABLE IF NOT EXISTS source_import_files ( - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shelley', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, source_path TEXT NOT NULL, @@ -5129,7 +5129,7 @@ fn rebuild_capture_sources_provider_check(conn: &Connection) -> Result<()> { CREATE TABLE capture_sources_new ( id TEXT PRIMARY KEY NOT NULL, kind TEXT NOT NULL CHECK (kind IN ('provider_import', 'provider_hook', 'direct_cli', 'manual')), - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shelley', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), machine_id TEXT NOT NULL, process_id INTEGER, cwd TEXT, @@ -5177,7 +5177,7 @@ fn rebuild_catalog_sessions_provider_check(conn: &Connection) -> Result<()> { DROP TABLE IF EXISTS catalog_sessions_new; CREATE TABLE catalog_sessions_new ( source_path TEXT PRIMARY KEY NOT NULL, - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shelley', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, external_session_id TEXT, @@ -5232,7 +5232,7 @@ fn rebuild_source_import_files_provider_check(conn: &Connection) -> Result<()> { r#" DROP TABLE IF EXISTS source_import_files_new; CREATE TABLE source_import_files_new ( - provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), + provider TEXT NOT NULL CHECK (provider IN ('codex', 'claude', 'pi', 'opencode', 'antigravity', 'gemini', 'cursor', 'copilot_cli', 'factory_ai_droid', 'openclaw', 'hermes', 'nanoclaw', 'astrbot', 'shelley', 'shell', 'git', 'jj', 'gh', 'custom', 'unknown')), source_format TEXT NOT NULL, source_root TEXT NOT NULL, source_path TEXT NOT NULL, diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 55a4a9923..5eed80ad6 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -89,6 +89,7 @@ ctx import --provider openclaw ctx import --provider hermes ctx import --provider nanoclaw --path /path/to/nanoclaw-project ctx import --provider astrbot --path /path/to/data/data_v4.db +ctx import --provider shelley --path ~/.config/shelley/shelley.db ctx import --provider gemini ctx import --provider cursor ctx import --provider copilot-cli @@ -251,7 +252,7 @@ optimized for agent reading; use `--verbose` for expanded text diagnostics. Filters: -- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|antigravity|gemini|cursor|copilot-cli|factory-ai-droid|custom`; +- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|shelley|antigravity|gemini|cursor|copilot-cli|factory-ai-droid|custom`; - `--workspace `, substring match over stored workspace, cwd, source path, or repository-name text; - `--since d`, for example `2026-06-01T00:00:00Z` or `30d`; @@ -271,7 +272,7 @@ Filters: CLI provider filters use kebab-case names. JSON output and stable SQL views use provider IDs in ctx output; multiword IDs may be snake_case, such as `copilot_cli` or `factory_ai_droid`, while compact IDs such as `openclaw`, -`nanoclaw`, and `astrbot` stay compact. +`nanoclaw`, `astrbot`, and `shelley` stay compact. `search` reads discovered native provider files and runs enabled auto history-source plugin commands for pre-search refresh, then queries SQLite. It diff --git a/docs/first-10-minutes.md b/docs/first-10-minutes.md index 1ec93d850..a9af2223e 100644 --- a/docs/first-10-minutes.md +++ b/docs/first-10-minutes.md @@ -66,6 +66,7 @@ ctx import --provider cursor --path ~/.cursor/projects ctx import --provider hermes --path ~/.hermes/state.db ctx import --provider nanoclaw --path /path/to/nanoclaw-project ctx import --provider astrbot --path /path/to/data/data_v4.db +ctx import --provider shelley --path ~/.config/shelley/shelley.db ``` Preview providers such as NanoClaw and AstrBot are explicit-import only. Use diff --git a/docs/provider-support-matrix.json b/docs/provider-support-matrix.json index 3dc79e183..b1324271a 100644 --- a/docs/provider-support-matrix.json +++ b/docs/provider-support-matrix.json @@ -434,6 +434,62 @@ "crates/ctx-history-capture/src/lib.rs" ] }, + { + "id": "shelley", + "display_name": "Shelley", + "priority": "p1", + "status": "local_import_when_supported", + "capture_provider": "shelley", + "implemented_paths": [ + { + "kind": "native_import", + "source_format": "shelley_sqlite", + "fidelity": "imported", + "proof": [ + "ctx sources", + "ctx import --provider shelley", + "ctx import --provider shelley --path " + ], + "notes": [ + "Reads Shelley SQLite history from SHELLEY_DB or ~/.config/shelley/shelley.db using a read-only SQLite connection.", + "Normalizes conversations as sessions and messages as events with stable conversation/sequence/message cursors.", + "Indexes text from Shelley user/agent/tool JSON, including thinking, tool calls, and tool results when present." + ] + } + ], + "history_locations": [ + "SHELLEY_DB", + "~/.config/shelley/shelley.db" + ], + "imports_existing_history": true, + "captures_new_runs_passively": false, + "child_sessions_supported": true, + "fidelity": { + "user_prompts": true, + "assistant_messages": true, + "tool_calls": true, + "tool_output": true, + "command_output": false, + "files_touched": false, + "artifacts": false, + "model_identity": true, + "costs": true, + "token_usage": true, + "parent_child_session_edges": true + }, + "redaction_notes": [ + "Reads the provider SQLite database read-only; imported conversation text, tool output, cwd, model, usage, and local source paths remain in the local ctx index." + ], + "blockers": [ + "Full GA needs ongoing validation against upstream Shelley schema drift." + ], + "public_docs": "docs/providers.md", + "fixture_paths": [], + "tests": [ + "crates/ctx-cli/tests/cli.rs", + "crates/ctx-history-capture/src/lib.rs" + ] + }, { "id": "antigravity_cli", "display_name": "Antigravity", diff --git a/docs/provider-support.md b/docs/provider-support.md index 9aa906012..b9e96df90 100644 --- a/docs/provider-support.md +++ b/docs/provider-support.md @@ -30,6 +30,7 @@ is: | Hermes Agent | `local_import_when_supported` | `HERMES_HOME/state.db`, `~/.hermes/state.db`, or an explicit Hermes SQLite DB. | Static local-history fixture smoke. | | NanoClaw | `local_import_when_supported` | Preview/manual import from a NanoClaw project root or `data/v2.db`; cwd/ancestor discovery only. | Static local-history fixture smoke; excluded from `ctx import --all` and pre-search refresh until promoted. | | AstrBot | `local_import_when_supported` | Preview/manual import from `ASTRBOT_ROOT/data/data_v4.db`, `~/.astrbot/data/data_v4.db`, cwd/ancestor project DBs, or an explicit DB path. | Static local-history fixture smoke; imports LLM context plus available platform history, not guaranteed complete IM transcripts. | +| Shelley | `local_import_when_supported` | `SHELLEY_DB`, `~/.config/shelley/shelley.db`, or an explicit Shelley SQLite DB. | Static local-history fixture smoke; imports conversations/messages read-only with tool text, usage/model metadata, and parent conversation links. | | Antigravity | `local_import_when_supported` | Antigravity `transcript_full.jsonl` or `transcript.jsonl` files under `~/.gemini/antigravity-cli/brain`, or an explicit Antigravity transcript JSONL tree. | Static local-history fixture smoke. | | Gemini | `local_import_when_supported` | Gemini chat JSONL files under `~/.gemini/tmp/**/chats`, or an explicit Gemini CLI history tree. | Static local-history fixture smoke. | | Cursor | `local_import_when_supported` | Cursor agent transcript JSONL files under `~/.cursor/projects/**/agent-transcripts`, or an explicit Cursor agent transcript path. | Static local-history fixture smoke. | diff --git a/docs/providers.md b/docs/providers.md index f8bbeb731..1852e9792 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -22,6 +22,8 @@ The current CLI imports local history for: - AstrBot local SQLite history from `ASTRBOT_ROOT/data/data_v4.db`, `~/.astrbot/data/data_v4.db`, or a project `data/data_v4.db` when imported explicitly; +- Shelley SQLite history from `SHELLEY_DB`, `~/.config/shelley/shelley.db`, or + an explicit Shelley DB path; - Antigravity transcript JSONL mirrors under `~/.gemini/antigravity-cli/brain/*/.system_generated/logs/transcript_full.jsonl` or `transcript.jsonl`; @@ -49,10 +51,10 @@ ctx sources --json ``` CLI provider flags use names such as `openclaw`, `hermes`, `nanoclaw`, -`astrbot`, `copilot-cli`, and `factory-ai-droid`. +`astrbot`, `shelley`, `copilot-cli`, and `factory-ai-droid`. Structured JSON and stable SQL views use provider IDs in ctx output; multiword IDs may be snake_case, such as `copilot_cli` or `factory_ai_droid`, while compact native -IDs such as `openclaw`, `nanoclaw`, and `astrbot` stay compact. +IDs such as `openclaw`, `nanoclaw`, `astrbot`, and `shelley` stay compact. `ctx sources --json` reports each known provider source with `import_support` and `importable` fields. A native source is marked available/importable only diff --git a/docs/search.md b/docs/search.md index c9078190e..6ca4a94f3 100644 --- a/docs/search.md +++ b/docs/search.md @@ -54,7 +54,7 @@ that support it. Search filters narrow both human output and JSON: -- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; +- `--provider codex|pi|claude|opencode|openclaw|hermes|nanoclaw|astrbot|shelley|antigravity|gemini|cursor|copilot-cli|factory-ai-droid`; - `--history-source `, for custom history imports; - `--provider-key `, `--source-id `, and @@ -79,7 +79,7 @@ Search filters narrow both human output and JSON: CLI provider filters use the kebab-case names above. JSON output and stable SQL views use provider IDs in ctx output; multiword provider IDs may be snake_case, such as `copilot_cli` or `factory_ai_droid`, while compact IDs such as -`openclaw`, `nanoclaw`, and `astrbot` stay compact. +`openclaw`, `nanoclaw`, `astrbot`, and `shelley` stay compact. `--since` accepts RFC 3339 timestamps such as `2026-06-01T00:00:00Z` or a day window such as `30d`. From e14d4f3f51f2e05b1f21682843b6c07d3d096b44 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:30:46 -0500 Subject: [PATCH 52/72] Harden CLI parser boundaries --- Cargo.lock | 147 +++++++++++++++++++++++- Cargo.toml | 1 + crates/ctx-cli/Cargo.toml | 1 + crates/ctx-cli/src/main.rs | 117 ++++++++++++++++--- crates/ctx-cli/src/mcp.rs | 12 +- crates/ctx-cli/src/parser_prop_tests.rs | 48 ++++++++ crates/ctx-cli/tests/cli.rs | 50 ++++++++ 7 files changed, 354 insertions(+), 22 deletions(-) create mode 100644 crates/ctx-cli/src/parser_prop_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 8c50263bc..36d52de21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.0" @@ -270,6 +285,7 @@ dependencies = [ "ctx-history-store", "libc", "predicates", + "proptest", "ring", "rusqlite", "serde", @@ -457,6 +473,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -511,6 +533,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -519,7 +553,7 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", ] [[package]] @@ -792,6 +826,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "predicates" version = "3.1.4" @@ -831,6 +874,31 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.46" @@ -840,12 +908,56 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + [[package]] name = "redox_users" version = "0.4.6" @@ -974,6 +1086,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "serde" version = "1.0.228" @@ -1147,6 +1271,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1238,6 +1368,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.125" @@ -1455,6 +1594,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index bd6f49189..ae516e886 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ directories = "5.0" ed25519-dalek = "2.1" libc = "0.2" predicates = "3.1" +proptest = "1" regex = "1.10" ring = "0.17" rusqlite = { version = "0.32", features = ["bundled", "hooks", "limits"] } diff --git a/crates/ctx-cli/Cargo.toml b/crates/ctx-cli/Cargo.toml index 8dbefb451..64aba4c2a 100644 --- a/crates/ctx-cli/Cargo.toml +++ b/crates/ctx-cli/Cargo.toml @@ -38,5 +38,6 @@ libc.workspace = true [dev-dependencies] assert_cmd.workspace = true predicates.workspace = true +proptest.workspace = true rusqlite.workspace = true tempfile.workspace = true diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 9ab5e4c5b..5704d01c0 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -25,6 +25,9 @@ mod mcp; mod net; mod upgrade; +#[cfg(test)] +mod parser_prop_tests; + use analytics::{AnalyticsEvent, AnalyticsProperties}; use config::{AppConfig, CONFIG_FILE}; use ctx_history_capture::{ @@ -57,7 +60,7 @@ use ctx_history_store::{ CatalogSession, CatalogSourceIndexUpdate, RawSqlOptions, RawSqlResult, RawSqlValue, SourceImportFile, SourceImportFileIndexUpdate, Store, StoreError, RAW_SQL_DEFAULT_MAX_COLUMNS, RAW_SQL_DEFAULT_MAX_ROWS, RAW_SQL_DEFAULT_MAX_SQL_BYTES, RAW_SQL_DEFAULT_MAX_VALUE_BYTES, - RAW_SQL_MAX_TIMEOUT, + RAW_SQL_MAX_SQL_BYTES_CAP, RAW_SQL_MAX_TIMEOUT, }; use history_source_plugins::{ discover_history_source_plugins, discover_history_source_plugins_with_diagnostics, @@ -69,6 +72,7 @@ const WAL_TRUNCATE_MIN_BYTES: u64 = 64 * 1024 * 1024; const LARGE_IMPORT_SOURCE_FILES_WARNING: usize = 10_000; const LARGE_IMPORT_SOURCE_BYTES_WARNING: u64 = 1024 * 1024 * 1024; const MAX_SEARCH_LIMIT: usize = 200; +pub(crate) const MAX_EVENT_WINDOW: usize = 50; #[derive(Debug, Parser)] #[command(name = "ctx", version, about = "Search local agent history")] @@ -202,11 +206,11 @@ struct ShowSessionArgs { struct ShowEventArgs { #[arg(help = "ctx event id or unambiguous id prefix")] id: String, - #[arg(long, default_value_t = 0)] + #[arg(long, default_value_t = 0, value_parser = parse_event_window_limit)] before: usize, - #[arg(long, default_value_t = 0)] + #[arg(long, default_value_t = 0, value_parser = parse_event_window_limit)] after: usize, - #[arg(long)] + #[arg(long, value_parser = parse_event_window_limit)] window: Option, #[arg(long, value_enum, default_value_t = OutputFormat::Text)] format: OutputFormat, @@ -3124,7 +3128,10 @@ fn event_window( .map(|window| (window, window)) .unwrap_or((before, after)); let start = index.saturating_sub(before); - let end = (index + after + 1).min(events.len()); + let end = index + .saturating_add(after) + .saturating_add(1) + .min(events.len()); Ok(events[start..end].to_vec()) } @@ -4001,6 +4008,18 @@ fn parse_search_limit(value: &str) -> std::result::Result { Ok(limit) } +fn parse_event_window_limit(value: &str) -> std::result::Result { + let limit = value + .parse::() + .map_err(|err| format!("invalid event window: {err}"))?; + if limit > MAX_EVENT_WINDOW { + return Err(format!( + "event window must be between 0 and {MAX_EVENT_WINDOW}" + )); + } + Ok(limit) +} + fn parse_sql_timeout(value: &str) -> std::result::Result { let trimmed = value.trim(); if trimmed.is_empty() { @@ -4091,17 +4110,16 @@ fn run_sql(args: SqlArgs, data_root: PathBuf) -> Result<()> { } fn read_sql_input(args: &SqlArgs) -> Result { + let max_sql_bytes = args.max_sql_bytes.min(RAW_SQL_MAX_SQL_BYTES_CAP); match (&args.sql, &args.file) { (Some(sql), None) if sql == "-" => { - let mut input = String::new(); - std::io::stdin() - .read_to_string(&mut input) - .context("read SQL from stdin")?; - Ok(input) + read_sql_limited(std::io::stdin().lock(), max_sql_bytes, "stdin") } (Some(sql), None) => Ok(sql.clone()), (None, Some(path)) => { - fs::read_to_string(path).with_context(|| format!("read SQL from {}", path.display())) + let file = fs::File::open(path) + .with_context(|| format!("read SQL from {}", path.display()))?; + read_sql_limited(file, max_sql_bytes, &path.display().to_string()) } (None, None) => Err(anyhow!( "SQL is required; pass a statement, --file , or '-' for stdin" @@ -4110,6 +4128,21 @@ fn read_sql_input(args: &SqlArgs) -> Result { } } +fn read_sql_limited(mut reader: impl Read, max_sql_bytes: usize, label: &str) -> Result { + let mut input = String::new(); + reader + .by_ref() + .take((max_sql_bytes as u64).saturating_add(1)) + .read_to_string(&mut input) + .with_context(|| format!("read SQL from {label}"))?; + if input.len() > max_sql_bytes { + return Err(anyhow!( + "SQL input from {label} exceeds max_sql_bytes ({max_sql_bytes})" + )); + } + Ok(input) +} + fn print_sql_table(result: &RawSqlResult) -> Result<()> { let rows = result .rows @@ -6446,8 +6479,8 @@ fn parse_since_filter(value: &str) -> Result> { let days: i64 = days .parse() .with_context(|| format!("invalid --since day window: {value}"))?; - let duration = - Duration::try_days(days).ok_or_else(|| anyhow!("invalid --since day window: {value}: value too large"))?; + let duration = Duration::try_days(days) + .ok_or_else(|| anyhow!("invalid --since day window: {value}: value too large"))?; let since = utc_now() .checked_sub_signed(duration) .ok_or_else(|| anyhow!("invalid --since day window: {value}: value too large"))?; @@ -6480,8 +6513,12 @@ fn home_dir() -> Option { #[cfg(test)] mod tests { - use super::{catalog_import_checkpoint_matches, parse_since_filter, sha256_file_prefix_hex, shell_quote_arg}; - use std::{fs, io::Write}; + use super::{ + catalog_import_checkpoint_matches, normalize_uuid_prefix, parse_event_window_limit, + parse_search_limit, parse_since_filter, parse_sql_timeout, sha256_file_prefix_hex, + shell_quote_arg, + }; + use std::{fs, io::Write, panic}; use tempfile::tempdir; #[test] @@ -6503,6 +6540,56 @@ mod tests { ); } + #[test] + fn cli_value_parsers_do_not_panic_on_adversarial_inputs() { + let inputs = [ + "", + " ", + "0", + "-1", + "1", + "30d", + "500000000d", + "9223372036854775807d", + "-9223372036854775808d", + "999999999999999999999999999999d", + "NaN", + "inf", + "1e309", + "1.5d", + "1970-01-01T00:00:00Z", + "999999-99-99T99:99:99Z", + "zzzzzzzz", + "ffffffff", + "ffffffff-ffff-ffff-ffff-ffffffffffff", + "\0", + "１２３", + ]; + + for input in inputs { + assert!( + panic::catch_unwind(|| parse_since_filter(input)).is_ok(), + "parse_since_filter panicked for {input:?}" + ); + assert!( + panic::catch_unwind(|| parse_search_limit(input)).is_ok(), + "parse_search_limit panicked for {input:?}" + ); + assert!( + panic::catch_unwind(|| parse_event_window_limit(input)).is_ok(), + "parse_event_window_limit panicked for {input:?}" + ); + assert!( + panic::catch_unwind(|| parse_sql_timeout(input)).is_ok(), + "parse_sql_timeout panicked for {input:?}" + ); + assert!( + panic::catch_unwind(|| normalize_uuid_prefix(input, "test")).is_ok(), + "normalize_uuid_prefix panicked for {input:?}" + ); + } + } + #[test] fn catalog_import_checkpoint_requires_matching_hash() { let temp = tempdir().unwrap(); diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index 4d3614fc7..010c46400 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -21,11 +21,11 @@ use super::{ event_window, event_window_json, indexed_history_item_count, mark_share_safe, raw_sql_result_json, search_filters, search_has_intent, session_transcript_json, sources_json, OutputFormat, ProviderArg, RefreshArg, SearchDto, SearchFilterInput, SearchIntentInput, - SearchRefreshReport, SourceIdentityFilterArgs, TranscriptMode, MAX_SEARCH_LIMIT, + SearchRefreshReport, SourceIdentityFilterArgs, TranscriptMode, MAX_EVENT_WINDOW, + MAX_SEARCH_LIMIT, }; const MCP_PROTOCOL_VERSION: &str = "2025-11-25"; -const MCP_MAX_EVENT_WINDOW: usize = 50; #[derive(Debug, Args)] pub(crate) struct McpArgs { @@ -435,12 +435,12 @@ fn tool_show_event(arguments: &Value, data_root: &Path) -> Result { let before = optional_usize(arguments, "before")?.unwrap_or(0); let after = optional_usize(arguments, "after")?.unwrap_or(0); let window = optional_usize(arguments, "window")?; - if before > MCP_MAX_EVENT_WINDOW - || after > MCP_MAX_EVENT_WINDOW - || window.is_some_and(|window| window > MCP_MAX_EVENT_WINDOW) + if before > MAX_EVENT_WINDOW + || after > MAX_EVENT_WINDOW + || window.is_some_and(|window| window > MAX_EVENT_WINDOW) { return Err(anyhow!( - "show_event before/after/window must be {MCP_MAX_EVENT_WINDOW} or less" + "show_event before/after/window must be {MAX_EVENT_WINDOW} or less" )); } let event = store.get_event(event_id)?; diff --git a/crates/ctx-cli/src/parser_prop_tests.rs b/crates/ctx-cli/src/parser_prop_tests.rs new file mode 100644 index 000000000..970bcb92f --- /dev/null +++ b/crates/ctx-cli/src/parser_prop_tests.rs @@ -0,0 +1,48 @@ +use super::{ + normalize_uuid_prefix, parse_event_window_limit, parse_search_limit, parse_since_filter, + parse_sql_timeout, MAX_EVENT_WINDOW, MAX_SEARCH_LIMIT, +}; +use proptest::prelude::*; +use std::panic; + +proptest! { + #[test] + fn cli_value_parsers_never_panic_for_generated_strings(input in ".{0,256}") { + prop_assert!(panic::catch_unwind(|| parse_since_filter(&input)).is_ok()); + prop_assert!(panic::catch_unwind(|| parse_search_limit(&input)).is_ok()); + prop_assert!(panic::catch_unwind(|| parse_event_window_limit(&input)).is_ok()); + prop_assert!(panic::catch_unwind(|| parse_sql_timeout(&input)).is_ok()); + prop_assert!(panic::catch_unwind(|| normalize_uuid_prefix(&input, "test")).is_ok()); + } + + #[test] + fn parse_search_limit_accepts_only_public_limit_range(limit in 1usize..=MAX_SEARCH_LIMIT) { + prop_assert_eq!(parse_search_limit(&limit.to_string()), Ok(limit)); + } + + #[test] + fn parse_search_limit_rejects_values_above_public_limit(limit in (MAX_SEARCH_LIMIT + 1)..=usize::MAX) { + prop_assert!(parse_search_limit(&limit.to_string()).is_err()); + } + + #[test] + fn parse_event_window_limit_accepts_only_public_window_range(limit in 0usize..=MAX_EVENT_WINDOW) { + prop_assert_eq!(parse_event_window_limit(&limit.to_string()), Ok(limit)); + } + + #[test] + fn parse_event_window_limit_rejects_values_above_public_window(limit in (MAX_EVENT_WINDOW + 1)..=usize::MAX) { + prop_assert!(parse_event_window_limit(&limit.to_string()).is_err()); + } + + #[test] + fn parse_since_filter_rejects_unrepresentable_day_windows(days in any::()) { + if chrono::Duration::try_days(days) + .and_then(|duration| crate::utc_now().checked_sub_signed(duration)) + .is_none() + { + let input = format!("{days}d"); + prop_assert!(parse_since_filter(&input).is_err()); + } + } +} diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index c027df05a..835060053 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -2202,6 +2202,32 @@ fn sql_reads_existing_store_and_supports_formats_and_input_sources() { "value,n\n\"a,b\",2\n" ); + let oversized_file_stderr = failure_stderr( + ctx(&temp) + .arg("sql") + .arg("--file") + .arg(&query_file) + .args(["--max-sql-bytes", "4"]), + ); + assert!( + oversized_file_stderr.contains("exceeds max_sql_bytes (4)"), + "{oversized_file_stderr}" + ); + + let oversized_stdin_stderr = ctx(&temp) + .args(["sql", "-", "--max-sql-bytes", "4"]) + .write_stdin("SELECT 1") + .assert() + .failure() + .get_output() + .stderr + .clone(); + let oversized_stdin_stderr = String::from_utf8(oversized_stdin_stderr).unwrap(); + assert!( + oversized_stdin_stderr.contains("exceeds max_sql_bytes (4)"), + "{oversized_stdin_stderr}" + ); + let raw_output = ctx(&temp) .args(["sql", "-", "--format", "raw"]) .write_stdin("SELECT 'abc' AS value") @@ -3838,6 +3864,30 @@ fn fresh_home_search_mvp_flow() { ])); assert_eq!(show_event_prefix["event"]["ctx_event_id"], ctx_event_id); + let oversized_after = failure_stderr(ctx(&temp).args([ + "show", + "event", + &ctx_event_id, + "--after", + "18446744073709551615", + ])); + assert!( + oversized_after.contains("event window must be between 0 and 50"), + "{oversized_after}" + ); + + let oversized_window = failure_stderr(ctx(&temp).args([ + "show", + "event", + &ctx_event_id, + "--window", + "18446744073709551615", + ])); + assert!( + oversized_window.contains("event window must be between 0 and 50"), + "{oversized_window}" + ); + let show_session = json_output(ctx(&temp).args(["show", "session", &ctx_session_id, "--format", "json"])); assert_eq!(show_session["schema_version"], 1); From 4f5db825eb49dd629963188a7ed78b73b674b170 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:39:34 -0500 Subject: [PATCH 53/72] Bound MCP and SQL result resources --- crates/ctx-cli/src/mcp.rs | 92 ++++++++++++++++++++++++++--- crates/ctx-cli/tests/cli.rs | 48 +++++++++++++++ crates/ctx-history-store/src/lib.rs | 46 +++++++++++++++ 3 files changed, 179 insertions(+), 7 deletions(-) diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index 010c46400..c2751785c 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -26,6 +26,12 @@ use super::{ }; const MCP_PROTOCOL_VERSION: &str = "2025-11-25"; +const MCP_MAX_LINE_BYTES: usize = 1024 * 1024; + +enum McpInputLine { + Line(String), + TooLarge, +} #[derive(Debug, Args)] pub(crate) struct McpArgs { @@ -54,16 +60,29 @@ pub(crate) fn run(args: McpArgs, data_root: PathBuf) -> Result<()> { fn serve_stdio(data_root: PathBuf) -> Result<()> { let stdin = io::stdin(); let stdout = io::stdout(); + let mut stdin = stdin.lock(); let mut stdout = stdout.lock(); let mut initialized = false; - for line in stdin.lock().lines() { - let line = line?; - let line = line.trim(); - if line.is_empty() { - continue; - } - if let Some(response) = handle_line(line, &data_root, &mut initialized) { + while let Some(input) = read_mcp_input_line(&mut stdin)? { + let response = match input { + McpInputLine::Line(line) => { + let line = line.trim(); + if line.is_empty() { + continue; + } + handle_line(line, &data_root, &mut initialized) + } + McpInputLine::TooLarge => Some(error_response( + Value::Null, + -32700, + "Parse error", + Some(json!({ + "error": format!("MCP message exceeds max line bytes ({MCP_MAX_LINE_BYTES})") + })), + )), + }; + if let Some(response) = response { writeln!(stdout, "{}", serde_json::to_string(&response)?)?; stdout.flush()?; } @@ -71,6 +90,65 @@ fn serve_stdio(data_root: PathBuf) -> Result<()> { Ok(()) } +fn read_mcp_input_line(reader: &mut impl BufRead) -> Result> { + let mut buffer = Vec::new(); + loop { + let available = reader.fill_buf()?; + if available.is_empty() { + if buffer.is_empty() { + return Ok(None); + } + break; + } + if let Some(newline_index) = available.iter().position(|byte| *byte == b'\n') { + let bytes_to_consume = newline_index + 1; + if buffer.len().saturating_add(bytes_to_consume) > MCP_MAX_LINE_BYTES { + reader.consume(bytes_to_consume); + return Ok(Some(McpInputLine::TooLarge)); + } + buffer.extend_from_slice(&available[..bytes_to_consume]); + reader.consume(bytes_to_consume); + break; + } + + let bytes_to_consume = available.len(); + if buffer.len().saturating_add(bytes_to_consume) > MCP_MAX_LINE_BYTES { + reader.consume(bytes_to_consume); + discard_until_newline(reader)?; + return Ok(Some(McpInputLine::TooLarge)); + } + buffer.extend_from_slice(available); + reader.consume(bytes_to_consume); + } + + Ok(Some(McpInputLine::Line( + String::from_utf8(buffer) + .map_err(|err| anyhow!("read MCP JSON-RPC line as UTF-8: {err}"))?, + ))) +} + +fn discard_until_newline(reader: &mut impl BufRead) -> Result<()> { + loop { + let available = reader.fill_buf()?; + if available.is_empty() { + return Ok(()); + } + let bytes_to_consume = available + .iter() + .position(|byte| *byte == b'\n') + .map(|index| index + 1) + .unwrap_or(available.len()); + let found_newline = bytes_to_consume <= available.len() + && available + .get(bytes_to_consume.saturating_sub(1)) + .is_some_and(|byte| *byte == b'\n'); + reader.consume(bytes_to_consume); + if found_newline { + return Ok(()); + } + } +} + fn handle_line(line: &str, data_root: &Path, initialized: &mut bool) -> Option { let message = match serde_json::from_str::(line) { Ok(message) => message, diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 835060053..87a8f1ec7 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -509,6 +509,22 @@ fn mcp_roundtrip_with_env(temp: &TempDir, messages: &[Value], envs: &[(&str, &st .collect() } +fn mcp_raw_roundtrip(temp: &TempDir, stdin: String) -> Vec { + let output = ctx(temp) + .args(["mcp", "serve"]) + .write_stdin(stdin) + .assert() + .success() + .get_output() + .stdout + .clone(); + String::from_utf8(output) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + fn assert_omits_keys(value: &Value, forbidden_keys: &[&str]) { match value { Value::Object(map) => { @@ -4092,6 +4108,38 @@ fn mcp_status_and_tools_list_are_read_only_without_initialized_store() { ); } +#[test] +fn mcp_rejects_oversized_input_line_and_continues() { + let temp = tempdir(); + let initialize = json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { "name": "ctx-test", "version": "0" } + } + }); + let mut stdin = "x".repeat(1024 * 1024 + 1); + stdin.push('\n'); + stdin.push_str(&serde_json::to_string(&initialize).unwrap()); + stdin.push('\n'); + + let responses = mcp_raw_roundtrip(&temp, stdin); + assert_eq!(responses.len(), 2); + assert_eq!(responses[0]["error"]["code"], -32700); + assert!( + responses[0]["error"]["data"]["error"] + .as_str() + .unwrap() + .contains("exceeds max line bytes"), + "{:#}", + responses[0] + ); + assert_eq!(responses[1]["result"]["serverInfo"]["name"], "ctx"); +} + #[test] fn mcp_sql_tool_returns_structured_json_and_rejects_writes() { let temp = tempdir(); diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 46554dd28..b558d2705 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -87,6 +87,11 @@ pub enum StoreError { min: usize, max: usize, }, + #[error("SQL result preview budget {estimated_bytes} bytes exceeds maximum {max_result_bytes}; lower max_rows, max_columns, or max_value_bytes")] + RawSqlResultBudgetTooLarge { + estimated_bytes: usize, + max_result_bytes: usize, + }, #[error("SQL query timed out after {timeout_ms}ms")] RawSqlTimedOut { timeout_ms: u64 }, } @@ -106,6 +111,7 @@ pub const RAW_SQL_DEFAULT_MAX_COLUMNS: usize = 64; pub const RAW_SQL_MAX_COLUMNS_CAP: usize = 256; pub const RAW_SQL_DEFAULT_MAX_VALUE_BYTES: usize = 512; pub const RAW_SQL_MAX_VALUE_BYTES_CAP: usize = 1_048_576; +pub const RAW_SQL_MAX_RESULT_PREVIEW_BYTES: usize = 64 * 1024 * 1024; const RAW_SQL_MIN_SQLITE_LENGTH_LIMIT_BYTES: usize = 64 * 1024; const RAW_SQL_VALUE_LENGTH_MARGIN_BYTES: usize = 1024; pub const RAW_SQL_DEFAULT_MAX_SQL_BYTES: usize = 64 * 1024; @@ -3946,6 +3952,7 @@ fn validate_raw_sql_options(options: &RawSqlOptions) -> Result<()> { max: usize::try_from(duration_ms(RAW_SQL_MAX_TIMEOUT)).unwrap_or(usize::MAX), }); } + validate_raw_sql_result_preview_budget(options)?; Ok(()) } @@ -3953,6 +3960,21 @@ fn validate_raw_sql_statement_bytes(sql: &str, options: &RawSqlOptions) -> Resul validate_raw_sql_usize("sql_bytes", sql.len(), 1, options.max_sql_bytes) } +fn validate_raw_sql_result_preview_budget(options: &RawSqlOptions) -> Result<()> { + let per_cell_bytes = options.max_value_bytes.saturating_mul(2).max(32); + let estimated_bytes = options + .max_rows + .saturating_mul(options.max_columns) + .saturating_mul(per_cell_bytes); + if estimated_bytes > RAW_SQL_MAX_RESULT_PREVIEW_BYTES { + return Err(StoreError::RawSqlResultBudgetTooLarge { + estimated_bytes, + max_result_bytes: RAW_SQL_MAX_RESULT_PREVIEW_BYTES, + }); + } + Ok(()) +} + struct RawSqlLimitGuard<'a> { conn: &'a Connection, length: i32, @@ -8618,6 +8640,30 @@ mod catalog_tests { assert!(result.truncated.values); } + #[test] + fn raw_sql_query_rejects_excessive_result_preview_budget() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let err = store + .raw_sql_query( + "SELECT 1", + RawSqlOptions { + max_rows: RAW_SQL_MAX_ROWS_CAP, + max_columns: RAW_SQL_MAX_COLUMNS_CAP, + max_value_bytes: 32, + ..RawSqlOptions::default() + }, + ) + .unwrap_err(); + assert!(matches!( + err, + StoreError::RawSqlResultBudgetTooLarge { + max_result_bytes: RAW_SQL_MAX_RESULT_PREVIEW_BYTES, + .. + } + )); + } + #[test] fn raw_sql_query_times_out_long_running_queries() { let temp = tempdir(); From 0d7978da760f279fc1e2d7808968415c9780ea80 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:41:48 -0500 Subject: [PATCH 54/72] Reject corrupt unsigned store fields --- crates/ctx-history-store/src/lib.rs | 120 ++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 6 deletions(-) diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index b558d2705..a65e603a7 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -3759,7 +3759,7 @@ impl Store { history_record_id: parse_optional_uuid(row.get(1)?)?, session_id: parse_optional_uuid(row.get(2)?)?, run_id: parse_optional_uuid(row.get(3)?)?, - seq: row.get::<_, i64>(4)? as u64, + seq: nonnegative_i64_to_u64(row.get(4)?)?, event_type: parse_text_enum::(row.get::<_, String>(5)?)?, role: parse_optional_text_enum::(row.get(6)?)?, occurred_at: ms_to_time(row.get(7)?)?, @@ -5540,6 +5540,10 @@ fn nonnegative_i64_to_u64(value: i64) -> rusqlite::Result { u64::try_from(value).map_err(|err| rusqlite::Error::ToSqlConversionFailure(Box::new(err))) } +fn nonnegative_i64_to_u32(value: i64) -> rusqlite::Result { + u32::try_from(value).map_err(|err| rusqlite::Error::ToSqlConversionFailure(Box::new(err))) +} + fn time_ms(value: i64) -> DateTime { DateTime::::from_timestamp_millis(value).unwrap_or(DateTime::::UNIX_EPOCH) } @@ -6940,7 +6944,10 @@ fn capture_source_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result(row.get::<_, String>(1)?)?, provider: parse_text_enum::(row.get::<_, String>(2)?)?, machine_id: row.get(3)?, - process_id: row.get::<_, Option>(4)?.map(|value| value as u32), + process_id: row + .get::<_, Option>(4)? + .map(nonnegative_i64_to_u32) + .transpose()?, cwd: row.get(5)?, raw_source_path: row.get(6)?, external_session_id: row.get(7)?, @@ -6951,7 +6958,7 @@ fn capture_source_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result(row.get::<_, String>(10)?)?, visibility: parse_text_enum::(row.get::<_, String>(11)?)?, sync_state: parse_text_enum::(row.get::<_, String>(12)?)?, - sync_version: row.get::<_, i64>(13)? as u64, + sync_version: nonnegative_i64_to_u64(row.get(13)?)?, deleted_at: None, metadata: parse_json(row.get::<_, String>(14)?)?, }, @@ -7114,7 +7121,7 @@ fn event_select_sql(tail: &str) -> String { fn event_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { Ok(Event { id: parse_uuid(row.get::<_, String>(0)?)?, - seq: row.get::<_, i64>(1)? as u64, + seq: nonnegative_i64_to_u64(row.get(1)?)?, history_record_id: parse_optional_uuid(row.get(2)?)?, session_id: parse_optional_uuid(row.get(3)?)?, run_id: parse_optional_uuid(row.get(4)?)?, @@ -7145,7 +7152,7 @@ fn artifact_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { kind: parse_text_enum::(row.get::<_, String>(1)?)?, blob_hash: row.get(2)?, blob_path: row.get(3)?, - byte_size: row.get::<_, i64>(4)? as u64, + byte_size: nonnegative_i64_to_u64(row.get(4)?)?, media_type: row.get(5)?, preview_text: row.get(6)?, redaction_state: parse_text_enum::(row.get::<_, String>(7)?)?, @@ -7321,7 +7328,7 @@ fn sync_metadata_from_row( visibility: parse_text_enum::(row.get::<_, String>(visibility_index)?)?, fidelity: parse_text_enum::(row.get::<_, String>(fidelity_index)?)?, sync_state: parse_text_enum::(row.get::<_, String>(sync_state_index)?)?, - sync_version: row.get::<_, i64>(sync_version_index)? as u64, + sync_version: nonnegative_i64_to_u64(row.get(sync_version_index)?)?, deleted_at: optional_ms_to_time(row.get(deleted_at_index)?)?, metadata: parse_json(row.get::<_, String>(metadata_index)?)?, }) @@ -7910,6 +7917,29 @@ mod catalog_tests { } } + fn artifact_record(id: Uuid, byte_size: u64) -> Artifact { + Artifact { + id, + kind: ArtifactKind::Markdown, + blob_hash: format!("{:064x}", 1), + blob_path: format!("{OBJECTS_DIR}/00/test-artifact"), + byte_size, + media_type: Some("text/markdown".to_owned()), + preview_text: Some("artifact preview".to_owned()), + redaction_state: RedactionState::LocalPreview, + timestamps: timestamps(), + source_id: None, + sync: sync_metadata(), + } + } + + fn assert_sql_conversion_error(result: Result) { + assert!( + matches!(result, Err(StoreError::Sql(_))), + "expected sqlite conversion error, got {result:?}" + ); + } + #[test] fn catalog_session_upsert_skips_unchanged_rows() { let temp = tempdir(); @@ -8640,6 +8670,84 @@ mod catalog_tests { assert!(result.truncated.values); } + #[test] + fn row_readers_reject_negative_unsigned_columns() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let bad_process_id = new_id(); + store + .conn + .execute( + r#" + INSERT INTO capture_sources + ( + id, kind, provider, machine_id, process_id, cwd, raw_source_path, + external_session_id, started_at_ms, fidelity, sync_version + ) + VALUES (?1, 'provider_import', 'codex', 'test-machine', -1, '/repo', '/tmp/session.jsonl', 'session', 1, 'imported', 0) + "#, + params![bad_process_id.to_string()], + ) + .unwrap(); + assert_sql_conversion_error(store.get_capture_source(bad_process_id)); + + let bad_sync_version = new_id(); + store + .conn + .execute( + r#" + INSERT INTO capture_sources + ( + id, kind, provider, machine_id, cwd, raw_source_path, + external_session_id, started_at_ms, fidelity, sync_version + ) + VALUES (?1, 'provider_import', 'codex', 'test-machine', '/repo', '/tmp/session.jsonl', 'session', 1, 'imported', -1) + "#, + params![bad_sync_version.to_string()], + ) + .unwrap(); + assert_sql_conversion_error(store.get_capture_source(bad_sync_version)); + + let event = Event { + id: new_id(), + seq: 1, + history_record_id: None, + session_id: None, + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::Assistant), + occurred_at: fixed_time(), + capture_source_id: None, + payload: serde_json::json!({"text": "negative seq marker"}), + payload_blob_id: None, + dedupe_key: None, + redaction_state: RedactionState::LocalPreview, + sync: sync_metadata(), + }; + store.upsert_event(&event).unwrap(); + store + .conn + .execute( + "UPDATE events SET seq = -1 WHERE id = ?1", + params![event.id.to_string()], + ) + .unwrap(); + assert_sql_conversion_error(store.get_event(event.id)); + assert_sql_conversion_error(store.search_event_hits("negative seq marker", 1)); + + let artifact = artifact_record(new_id(), 1); + store.upsert_artifact(&artifact).unwrap(); + store + .conn + .execute( + "UPDATE artifacts SET byte_size = -1 WHERE id = ?1", + params![artifact.id.to_string()], + ) + .unwrap(); + assert_sql_conversion_error(store.list_artifacts()); + } + #[test] fn raw_sql_query_rejects_excessive_result_preview_budget() { let temp = tempdir(); From 54c452775ebe2e59c3e93be7efd2c378c29b442f Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:43:57 -0500 Subject: [PATCH 55/72] Guard OpenCode SQLite normalizer paths --- crates/ctx-history-capture/src/lib.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 83000969c..1981ebc41 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -9004,12 +9004,7 @@ fn normalize_opencode_sqlite( path: &Path, context: &ProviderAdapterContext, ) -> Result { - let conn = Connection::open_with_flags( - path, - OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, - )?; - conn.busy_timeout(std::time::Duration::from_secs(5))?; - conn.pragma_update(None, "query_only", true)?; + let conn = open_provider_sqlite_readonly(path)?; let user_version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?; let schema_fingerprint = opencode_schema_fingerprint(&conn)?; let legacy_message_rows = opencode_count(&conn, "message").unwrap_or(0); @@ -13890,6 +13885,25 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn native_opencode_normalizer_rejects_symlinked_sqlite() { + use std::os::unix::fs::symlink; + + let temp = tempdir(); + let fixture = write_opencode_smoke_db(&temp, false); + let link = temp.path().join("linked-opencode.db"); + symlink(&fixture, &link).unwrap(); + + let err = normalize_opencode_sqlite(&link, &ProviderAdapterContext::default()).unwrap_err(); + assert!(matches!( + err, + CaptureError::InvalidProviderTranscriptPath { path, reason } + if path.ends_with("linked-opencode.db") + && reason == "symlinked provider transcript files are rejected" + )); + } + #[test] fn native_opencode_synthesizes_session_message_seq_when_missing() { let temp = tempdir(); From 215371e63e6497dd3417e9916babd45c33ae7375 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:46:58 -0500 Subject: [PATCH 56/72] Tighten MCP framing and result caps --- crates/ctx-cli/src/mcp.rs | 42 ++++++--- crates/ctx-cli/tests/cli.rs | 131 ++++++++++++++++++++++++++++ crates/ctx-history-store/src/lib.rs | 61 +++++++++++-- 3 files changed, 216 insertions(+), 18 deletions(-) diff --git a/crates/ctx-cli/src/mcp.rs b/crates/ctx-cli/src/mcp.rs index c2751785c..728176265 100644 --- a/crates/ctx-cli/src/mcp.rs +++ b/crates/ctx-cli/src/mcp.rs @@ -27,9 +27,11 @@ use super::{ const MCP_PROTOCOL_VERSION: &str = "2025-11-25"; const MCP_MAX_LINE_BYTES: usize = 1024 * 1024; +const MCP_MAX_SESSION_EVENTS: usize = 200; enum McpInputLine { Line(String), + InvalidUtf8, TooLarge, } @@ -73,6 +75,12 @@ fn serve_stdio(data_root: PathBuf) -> Result<()> { } handle_line(line, &data_root, &mut initialized) } + McpInputLine::InvalidUtf8 => Some(error_response( + Value::Null, + -32700, + "Parse error", + Some(json!({ "error": "MCP message is not valid UTF-8" })), + )), McpInputLine::TooLarge => Some(error_response( Value::Null, -32700, @@ -121,10 +129,10 @@ fn read_mcp_input_line(reader: &mut impl BufRead) -> Result reader.consume(bytes_to_consume); } - Ok(Some(McpInputLine::Line( - String::from_utf8(buffer) - .map_err(|err| anyhow!("read MCP JSON-RPC line as UTF-8: {err}"))?, - ))) + Ok(Some(match String::from_utf8(buffer) { + Ok(line) => McpInputLine::Line(line), + Err(_) => McpInputLine::InvalidUtf8, + })) } fn discard_until_newline(reader: &mut impl BufRead) -> Result<()> { @@ -497,14 +505,24 @@ fn tool_show_session(arguments: &Value, data_root: &Path) -> Result { let session_id = required_uuid(arguments, "ctx_session_id")?; let mode = optional_transcript_mode(arguments, "mode")?.unwrap_or(TranscriptMode::Lite); let session = store.get_session(session_id)?; - let events = store.events_for_session(session.id)?; - Ok(session_transcript_json( - &store, - &session, - &events, - mode, - OutputFormat::Json, - )) + let mut events = store.events_for_session_limited(session.id, MCP_MAX_SESSION_EVENTS + 1)?; + let truncated = events.len() > MCP_MAX_SESSION_EVENTS; + if truncated { + events.truncate(MCP_MAX_SESSION_EVENTS); + } + let mut value = session_transcript_json(&store, &session, &events, mode, OutputFormat::Json); + if truncated { + if let Some(object) = value.as_object_mut() { + object.insert( + "truncated".to_owned(), + json!({ + "events": true, + "max_events": MCP_MAX_SESSION_EVENTS, + }), + ); + } + } + Ok(value) } fn tool_show_event(arguments: &Value, data_root: &Path) -> Result { diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 87a8f1ec7..f1e22002e 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -510,6 +510,10 @@ fn mcp_roundtrip_with_env(temp: &TempDir, messages: &[Value], envs: &[(&str, &st } fn mcp_raw_roundtrip(temp: &TempDir, stdin: String) -> Vec { + mcp_raw_roundtrip_bytes(temp, stdin.into_bytes()) +} + +fn mcp_raw_roundtrip_bytes(temp: &TempDir, stdin: Vec) -> Vec { let output = ctx(temp) .args(["mcp", "serve"]) .write_stdin(stdin) @@ -4140,6 +4144,33 @@ fn mcp_rejects_oversized_input_line_and_continues() { assert_eq!(responses[1]["result"]["serverInfo"]["name"], "ctx"); } +#[test] +fn mcp_rejects_invalid_utf8_input_line_and_continues() { + let temp = tempdir(); + let initialize = json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { "name": "ctx-test", "version": "0" } + } + }); + let mut stdin = vec![0xff, b'\n']; + stdin.extend_from_slice(serde_json::to_string(&initialize).unwrap().as_bytes()); + stdin.push(b'\n'); + + let responses = mcp_raw_roundtrip_bytes(&temp, stdin); + assert_eq!(responses.len(), 2); + assert_eq!(responses[0]["error"]["code"], -32700); + assert_eq!( + responses[0]["error"]["data"]["error"], + "MCP message is not valid UTF-8" + ); + assert_eq!(responses[1]["result"]["serverInfo"]["name"], "ctx"); +} + #[test] fn mcp_sql_tool_returns_structured_json_and_rejects_writes() { let temp = tempdir(); @@ -4184,6 +4215,23 @@ fn mcp_sql_tool_returns_structured_json_and_rejects_writes() { } } }), + json!({ + "jsonrpc": "2.0", + "id": "budget", + "method": "tools/call", + "params": { + "name": "sql", + "arguments": { + "sql": format!( + "SELECT {}", + (0..256).map(|index| format!("1 AS c{index}")).collect::>().join(", ") + ), + "max_rows": 10000, + "max_columns": 256, + "max_value_bytes": 32 + } + } + }), ], ); @@ -4200,6 +4248,89 @@ fn mcp_sql_tool_returns_structured_json_and_rejects_writes() { .as_str() .unwrap() .contains("SQL query must be read-only")); + + let budget = &responses[3]["result"]; + assert_eq!(budget["isError"], true); + assert!(budget["structuredContent"]["error"] + .as_str() + .unwrap() + .contains("SQL result preview budget")); +} + +#[test] +fn mcp_show_session_caps_transcript_events() { + let temp = tempdir(); + ctx(&temp) + .args(["setup", "--catalog-only", "--progress", "none"]) + .assert() + .success(); + + let session_id = "018f45d0-0000-7000-8000-000000010001"; + let conn = Connection::open(temp.path().join("work.sqlite")).unwrap(); + conn.execute( + r#" + INSERT INTO sessions + ( + id, provider, external_session_id, agent_type, is_primary, status, fidelity, + started_at_ms, created_at_ms, updated_at_ms + ) + VALUES (?1, 'codex', 'mcp-large-session', 'primary', 1, 'imported', 'imported', 1, 1, 1) + "#, + [session_id], + ) + .unwrap(); + for index in 0..201 { + let event_id = format!("018f45d0-0000-7000-8000-{index:012x}"); + conn.execute( + r#" + INSERT INTO events + (id, seq, session_id, event_type, role, occurred_at_ms, payload_json) + VALUES (?1, ?2, ?3, 'message', 'assistant', ?4, ?5) + "#, + params![ + event_id, + index, + session_id, + index + 1, + format!(r#"{{"text":"mcp transcript event {index}"}}"#) + ], + ) + .unwrap(); + } + drop(conn); + + let responses = mcp_roundtrip( + &temp, + &[ + json!({ + "jsonrpc": "2.0", + "id": "init", + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { "name": "ctx-test", "version": "0" } + } + }), + json!({ + "jsonrpc": "2.0", + "id": "show", + "method": "tools/call", + "params": { + "name": "show_session", + "arguments": { + "ctx_session_id": session_id, + "mode": "log" + } + } + }), + ], + ); + + let transcript = &responses[1]["result"]["structuredContent"]; + assert_eq!(transcript["truncated"]["events"], true); + assert_eq!(transcript["truncated"]["max_events"], 200); + assert_eq!(transcript["events"].as_array().unwrap().len(), 200); } #[test] diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index a65e603a7..6b72a1033 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -112,6 +112,7 @@ pub const RAW_SQL_MAX_COLUMNS_CAP: usize = 256; pub const RAW_SQL_DEFAULT_MAX_VALUE_BYTES: usize = 512; pub const RAW_SQL_MAX_VALUE_BYTES_CAP: usize = 1_048_576; pub const RAW_SQL_MAX_RESULT_PREVIEW_BYTES: usize = 64 * 1024 * 1024; +pub const RAW_SQL_MAX_RESULT_CELLS: usize = 262_144; const RAW_SQL_MIN_SQLITE_LENGTH_LIMIT_BYTES: usize = 64 * 1024; const RAW_SQL_VALUE_LENGTH_MARGIN_BYTES: usize = 1024; pub const RAW_SQL_DEFAULT_MAX_SQL_BYTES: usize = 64 * 1024; @@ -1215,6 +1216,7 @@ impl Store { max_columns: options.max_columns, }); } + validate_raw_sql_result_preview_budget(&options, column_count)?; let columns = stmt .column_names() @@ -2690,6 +2692,21 @@ impl Store { collect_rows(rows) } + pub fn events_for_session_limited(&self, session_id: Uuid, limit: usize) -> Result> { + let mut stmt = self.conn.prepare( + event_select_sql("WHERE session_id = ?1 ORDER BY seq, occurred_at_ms LIMIT ?2") + .as_str(), + )?; + let rows = stmt.query_map( + params![ + session_id.to_string(), + i64::try_from(limit).unwrap_or(i64::MAX) + ], + event_from_row, + )?; + collect_rows(rows) + } + pub fn events_for_record(&self, record_id: Uuid) -> Result> { let mut stmt = self.conn.prepare( event_select_sql( @@ -3952,7 +3969,6 @@ fn validate_raw_sql_options(options: &RawSqlOptions) -> Result<()> { max: usize::try_from(duration_ms(RAW_SQL_MAX_TIMEOUT)).unwrap_or(usize::MAX), }); } - validate_raw_sql_result_preview_budget(options)?; Ok(()) } @@ -3960,13 +3976,23 @@ fn validate_raw_sql_statement_bytes(sql: &str, options: &RawSqlOptions) -> Resul validate_raw_sql_usize("sql_bytes", sql.len(), 1, options.max_sql_bytes) } -fn validate_raw_sql_result_preview_budget(options: &RawSqlOptions) -> Result<()> { - let per_cell_bytes = options.max_value_bytes.saturating_mul(2).max(32); +fn validate_raw_sql_result_preview_budget( + options: &RawSqlOptions, + column_count: usize, +) -> Result<()> { + let estimated_cells = options.max_rows.saturating_mul(column_count); + let per_cell_bytes = options + .max_value_bytes + .saturating_mul(4) + .saturating_add(64) + .max(128); let estimated_bytes = options .max_rows - .saturating_mul(options.max_columns) + .saturating_mul(column_count) .saturating_mul(per_cell_bytes); - if estimated_bytes > RAW_SQL_MAX_RESULT_PREVIEW_BYTES { + if estimated_cells > RAW_SQL_MAX_RESULT_CELLS + || estimated_bytes > RAW_SQL_MAX_RESULT_PREVIEW_BYTES + { return Err(StoreError::RawSqlResultBudgetTooLarge { estimated_bytes, max_result_bytes: RAW_SQL_MAX_RESULT_PREVIEW_BYTES, @@ -8752,9 +8778,13 @@ mod catalog_tests { fn raw_sql_query_rejects_excessive_result_preview_budget() { let temp = tempdir(); let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let many_columns = (0..RAW_SQL_MAX_COLUMNS_CAP) + .map(|index| format!("1 AS c{index}")) + .collect::>() + .join(", "); let err = store .raw_sql_query( - "SELECT 1", + &format!("SELECT {many_columns}"), RawSqlOptions { max_rows: RAW_SQL_MAX_ROWS_CAP, max_columns: RAW_SQL_MAX_COLUMNS_CAP, @@ -8772,6 +8802,25 @@ mod catalog_tests { )); } + #[test] + fn raw_sql_query_budgets_against_actual_column_count() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let result = store + .raw_sql_query( + "SELECT 1", + RawSqlOptions { + max_rows: RAW_SQL_MAX_ROWS_CAP, + max_columns: RAW_SQL_MAX_COLUMNS_CAP, + max_value_bytes: 32, + ..RawSqlOptions::default() + }, + ) + .unwrap(); + assert_eq!(result.returned_rows, 1); + assert_eq!(result.rows[0][0], RawSqlValue::Integer(1)); + } + #[test] fn raw_sql_query_times_out_long_running_queries() { let temp = tempdir(); From e1469c977974884ae55f25f78d0901515ce80eda Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:50:39 -0500 Subject: [PATCH 57/72] Reject negative provider SQLite indexes --- crates/ctx-history-capture/src/lib.rs | 149 +++++++++++++++++++++----- 1 file changed, 122 insertions(+), 27 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 1981ebc41..6f1cddabf 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -6693,6 +6693,16 @@ fn open_provider_sqlite_readonly(path: &Path) -> Result { Ok(conn) } +fn provider_nonnegative_i64_to_u64(value: i64, field: &'static str) -> Result { + u64::try_from(value).map_err(|_| { + CaptureError::InvalidPayload(format!("{field} must be nonnegative, got {value}")) + }) +} + +fn provider_line_from_index(index: u64) -> usize { + index.min(usize::MAX as u64) as usize +} + fn provider_timestamp_seconds(value: Option, fallback: DateTime) -> DateTime { let Some(value) = value else { return fallback; @@ -7246,15 +7256,24 @@ fn normalize_hermes_sqlite( let mut result = ProviderNormalizationResult::default(); for row in messages { + let provider_event_index = + match provider_nonnegative_i64_to_u64(row.id, "Hermes message id") { + Ok(value) => value, + Err(err) => { + push_provider_import_failure(&mut result.summary, 0, err.to_string()); + continue; + } + }; + let line = provider_line_from_index(provider_event_index); let Some(session) = sessions_by_id.get(&row.session_id) else { - result.summary.failed += 1; - result.summary.failures.push(ProviderImportFailure { - line: row.id.max(0) as usize, - error: format!( + push_provider_import_failure( + &mut result.summary, + line, + format!( "Hermes message {} references missing session {}", row.id, row.session_id ), - }); + ); continue; }; let provider_session_id = session.id.clone(); @@ -7276,7 +7295,7 @@ fn normalize_hermes_sqlite( provider: CaptureProvider::Hermes, source_format: HERMES_SQLITE_SOURCE_FORMAT, provider_session_id: provider_session_id.clone(), - provider_event_index: row.id.max(0) as u64, + provider_event_index, provider_event_hash: Some(format!("message:{}", row.id)), cursor: format!("messages:id:{}", row.id), event_type, @@ -7309,7 +7328,7 @@ fn normalize_hermes_sqlite( }), }); result.captures.push(( - row.id.max(0) as usize, + line, native_provider_capture( NativeSessionDraft { provider: CaptureProvider::Hermes, @@ -7609,6 +7628,17 @@ fn normalize_nanoclaw_project( ) }); for message in messages { + let seq = match message + .seq + .map(|seq| provider_nonnegative_i64_to_u64(seq, "NanoClaw message seq")) + .transpose() + { + Ok(seq) => seq, + Err(err) => { + push_provider_import_failure(&mut result.summary, 0, err.to_string()); + continue; + } + }; let provider_session_id = format!("{}/{}", session.agent_group_id, session.id); let occurred_at = provider_timestamp_millis(message.timestamp, context.imported_at); let started_at = provider_timestamp_millis(session.created_at, occurred_at); @@ -7623,7 +7653,7 @@ fn normalize_nanoclaw_project( message.kind.as_deref().unwrap_or(message.source) ) }); - let event_index = nanoclaw_event_index(&message); + let event_index = nanoclaw_event_index(&message, seq); let role = if message.source == "inbound" { Some(EventRole::User) } else { @@ -7738,15 +7768,15 @@ fn nanoclaw_project_root(path: &Path) -> Result { }) } -fn nanoclaw_event_index(message: &NanoClawMessageRow) -> u64 { - if let Some(seq) = message.seq { +fn nanoclaw_event_index(message: &NanoClawMessageRow, seq: Option) -> u64 { + if let Some(seq) = seq { let source_bucket = if message.source == "outbound" { 500_000 } else { 0 }; let row_bucket = fnv1a64(format!("{}:{}", message.source, message.id).as_bytes()) % 500_000; - return (seq.max(0) as u64) + return seq .saturating_mul(1_000_000) .saturating_add(source_bucket) .saturating_add(row_bucket); @@ -8560,6 +8590,16 @@ fn normalize_astrbot_sqlite( let mut checkpoint_sessions = BTreeMap::::new(); for conversation in &conversations { + let conversation_line = match provider_nonnegative_i64_to_u64( + conversation.row_id, + "AstrBot conversation row id", + ) { + Ok(value) => provider_line_from_index(value), + Err(err) => { + push_provider_import_failure(&mut result.summary, 0, err.to_string()); + continue; + } + }; let provider_session_id = astrbot_provider_session_id(conversation); let started_at = provider_timestamp_millis(conversation.created_at, context.imported_at); let ended_at = conversation @@ -8636,7 +8676,7 @@ fn normalize_astrbot_sqlite( }), }); result.captures.push(( - conversation.row_id.max(0) as usize, + conversation_line, astrbot_capture( AstrBotCaptureDraft { conversation, @@ -8660,6 +8700,14 @@ fn normalize_astrbot_sqlite( .map(|conversation| (astrbot_provider_session_id(conversation), conversation)) .collect::>(); for message in platform_messages { + let message_id = + match provider_nonnegative_i64_to_u64(message.id, "AstrBot platform message id") { + Ok(value) => value, + Err(err) => { + push_provider_import_failure(&mut result.summary, 0, err.to_string()); + continue; + } + }; let provider_session_id = message .llm_checkpoint_id .as_ref() @@ -8689,7 +8737,7 @@ fn normalize_astrbot_sqlite( } else { Some(EventRole::Assistant) }; - let event_index = 1_000_000u64.saturating_add(message.id.max(0) as u64); + let event_index = 1_000_000u64.saturating_add(message_id); let event = native_event(NativeEventDraft { provider: CaptureProvider::AstrBot, source_format: ASTRBOT_SQLITE_SOURCE_FORMAT, @@ -9028,25 +9076,34 @@ fn normalize_opencode_sqlite( let raw_source_path = path.display().to_string(); for row in messages { + let provider_event_index = + match provider_nonnegative_i64_to_u64(row.seq, "OpenCode session_message seq") { + Ok(value) => value, + Err(err) => { + push_provider_import_failure(&mut result.summary, 0, err.to_string()); + continue; + } + }; + let line = provider_line_from_index(provider_event_index); let Some(session) = sessions_by_id.get(&row.session_id) else { - result.summary.failed += 1; - result.summary.failures.push(ProviderImportFailure { - line: row.seq.max(0) as usize, - error: format!( + push_provider_import_failure( + &mut result.summary, + line, + format!( "OpenCode session_message {} references missing session {}", row.id, row.session_id ), - }); + ); continue; }; let data: Value = match serde_json::from_str(&row.data) { Ok(data) => data, Err(err) => { - result.summary.failed += 1; - result.summary.failures.push(ProviderImportFailure { - line: row.seq.max(0) as usize, - error: format!("invalid JSON in session_message {}: {err}", row.id), - }); + push_provider_import_failure( + &mut result.summary, + line, + format!("invalid JSON in session_message {}: {err}", row.id), + ); continue; } }; @@ -9057,7 +9114,7 @@ fn normalize_opencode_sqlite( .get(&session.id) .copied() .unwrap_or(occurred_at); - let event = opencode_event(&row, &data, occurred_at); + let event = opencode_event(&row, &data, occurred_at, provider_event_index); result .files_touched .extend(provider_file_touches_from_raw_value( @@ -9067,11 +9124,11 @@ fn normalize_opencode_sqlite( Some(raw_source_path.as_str()), &data, &event, - row.seq.max(0) as usize, + line, )); let is_subagent = session.parent_id.is_some(); result.captures.push(( - row.seq.max(0) as usize, + line, ProviderCaptureEnvelope { schema_version: PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, provider: CaptureProvider::OpenCode, @@ -9465,13 +9522,14 @@ fn opencode_event( row: &OpenCodeMessageRow, data: &Value, occurred_at: DateTime, + provider_event_index: u64, ) -> ProviderEventEnvelope { let event_type = opencode_event_type(&row.entry_type, data); let role = Some(provider_role(Some(&row.entry_type))); let text = opencode_event_text(&row.entry_type, data, event_type); let (text, truncated) = provider_local_preview(&text, PROVIDER_MAX_TEXT_CHARS); ProviderEventEnvelope { - provider_event_index: row.seq.max(0) as u64, + provider_event_index, provider_event_hash: Some(row.id.clone()), cursor: Some(format!( "session_message:{}:seq:{}", @@ -13938,6 +13996,43 @@ mod tests { assert_ne!(events[0].id, events[1].id); } + #[test] + fn native_opencode_rejects_negative_session_message_seq() { + let temp = tempdir(); + let fixture = write_opencode_smoke_db(&temp, false); + let conn = Connection::open(&fixture).unwrap(); + conn.execute( + "update session_message set seq = -1 where id = 'msg-user'", + [], + ) + .unwrap(); + drop(conn); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_opencode_sqlite( + &fixture, + &mut store, + OpenCodeSqliteImportOptions { + allow_partial_failures: true, + ..OpenCodeSqliteImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 1); + assert!(summary.failures[0] + .error + .contains("OpenCode session_message seq must be nonnegative")); + assert_eq!(summary.imported_events, 2); + let session_id = provider_session_uuid(CaptureProvider::OpenCode, "opencode-root"); + let events = store.events_for_session(session_id).unwrap(); + assert!(events.iter().all(|event| { + event.payload["body"]["session_message_seq"] + .as_i64() + .is_some_and(|seq| seq >= 0) + })); + } + #[test] fn native_opencode_reports_malformed_and_corrupt_db() { let temp = tempdir(); From b8e588cbceeb2207ad6e85ea10b266a368b8d024 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:54:48 -0500 Subject: [PATCH 58/72] Bound provider JSONL line reads --- crates/ctx-history-capture/src/lib.rs | 222 ++++++++++++++++++-------- 1 file changed, 152 insertions(+), 70 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 6f1cddabf..343155715 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -40,6 +40,7 @@ pub use provider_sources::{ }; pub const CAPTURE_SCHEMA_VERSION: u32 = 1; +const MAX_PROVIDER_JSONL_LINE_BYTES: usize = 16 * 1024 * 1024; #[derive(Debug, Error)] pub enum CaptureError { #[error("io error: {0}")] @@ -913,17 +914,18 @@ impl ProviderCaptureAdapter for ProviderFixtureJsonlAdapter { ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let reader = BufReader::new(file); + let mut reader = BufReader::new(file); let mut result = ProviderNormalizationResult::default(); + let mut line = Vec::new(); + let mut line_number = 0usize; - for (index, line) in reader.lines().enumerate() { - let line_number = index + 1; - let line = line?; - if line.trim().is_empty() { + while read_provider_jsonl_line(&mut reader, &mut line)? { + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { continue; } - let fixture: ProviderFixtureLine = match serde_json::from_str(&line) { + let fixture: ProviderFixtureLine = match serde_json::from_slice(&line) { Ok(fixture) => fixture, Err(err) => { result.summary.failed += 1; @@ -975,19 +977,20 @@ impl ProviderCaptureAdapter for CodexHistoryJsonlAdapter { ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let reader = BufReader::new(file); + let mut reader = BufReader::new(file); let mut result = ProviderNormalizationResult::default(); let mut parsed = Vec::new(); let mut first_seen = BTreeMap::new(); + let mut line = Vec::new(); + let mut line_number = 0usize; - for (index, line) in reader.lines().enumerate() { - let line_number = index + 1; - let line = line?; - if line.trim().is_empty() { + while read_provider_jsonl_line(&mut reader, &mut line)? { + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { continue; } - let history: CodexHistoryLine = match serde_json::from_str(&line) { + let history: CodexHistoryLine = match serde_json::from_slice(&line) { Ok(history) => history, Err(err) => { result.summary.failed += 1; @@ -1167,12 +1170,7 @@ impl ProviderCaptureAdapter for CodexSessionJsonlAdapter { let mut line_number = 0usize; let mut line = Vec::new(); - loop { - line.clear(); - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { - break; - } + while read_provider_jsonl_line(&mut reader, &mut line)? { line_number += 1; if line.iter().all(u8::is_ascii_whitespace) { continue; @@ -1439,18 +1437,19 @@ fn normalize_pi_session_jsonl_file( ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let reader = BufReader::new(file); + let mut reader = BufReader::new(file); let mut result = ProviderNormalizationResult::default(); let mut header = None; + let mut line = Vec::new(); + let mut line_number = 0usize; - for (index, line) in reader.lines().enumerate() { - let line_number = index + 1; - let line = line?; - if line.trim().is_empty() { + while read_provider_jsonl_line(&mut reader, &mut line)? { + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { continue; } - let value: Value = match serde_json::from_str(&line) { + let value: Value = match serde_json::from_slice(&line) { Ok(value) => value, Err(err) => { result.summary.failed += 1; @@ -1853,18 +1852,20 @@ pub fn read_jsonl(path: impl AsRef) -> Result> { let path = path.as_ref(); ensure_regular_spool_file(path)?; let file = File::open(path)?; - let reader = BufReader::new(file); + let mut reader = BufReader::new(file); let mut envelopes = Vec::new(); + let mut line = Vec::new(); + let mut line_number = 0usize; - for (index, line) in reader.lines().enumerate() { - let line = line?; - if line.trim().is_empty() { + while read_provider_jsonl_line(&mut reader, &mut line)? { + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { continue; } let envelope: CaptureEnvelope = - serde_json::from_str(&line).map_err(|source| CaptureError::InvalidJsonLine { + serde_json::from_slice(&line).map_err(|source| CaptureError::InvalidJsonLine { path: path.to_path_buf(), - line: index + 1, + line: line_number, source, })?; validate_envelope(&envelope)?; @@ -2192,22 +2193,21 @@ pub fn import_codex_session_jsonl_tail( let mut line_number = 0usize; let mut position = 0u64; - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { + if !read_provider_jsonl_line(&mut reader, &mut line)? { return Ok(summary); } line_number += 1; + let read = line.len(); position = position.saturating_add(read as u64); let header_value: Value = serde_json::from_slice(&line)?; let header = codex_session_header(header_value)?; while position < start_offset { - line.clear(); - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { + if !read_provider_jsonl_line(&mut reader, &mut line)? { return Ok(summary); } line_number += 1; + let read = line.len(); position = position.saturating_add(read as u64); } @@ -2225,13 +2225,9 @@ pub fn import_codex_session_jsonl_tail( let mut call_contexts: BTreeMap = BTreeMap::new(); let mut completed_bytes = 0u64; - loop { - line.clear(); - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { - break; - } + while read_provider_jsonl_line(&mut reader, &mut line)? { line_number += 1; + let read = line.len(); completed_bytes = completed_bytes.saturating_add(read as u64); if line.iter().all(u8::is_ascii_whitespace) { continue; @@ -2729,12 +2725,7 @@ fn import_codex_session_path_fast( let mut call_contexts: BTreeMap = BTreeMap::new(); let mut line_number = 0usize; let mut line = Vec::new(); - loop { - line.clear(); - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { - break; - } + while read_provider_jsonl_line(&mut reader, &mut line)? { line_number += 1; if line.iter().all(u8::is_ascii_whitespace) { continue; @@ -3858,16 +3849,18 @@ fn normalize_custom_history_jsonl_v1_reader( reader: impl BufRead, context: &ProviderAdapterContext, ) -> Result { + let mut reader = reader; let mut summary = ProviderImportSummary::default(); let mut records = Vec::new(); + let mut line = Vec::new(); + let mut line_number = 0usize; - for (index, line) in reader.lines().enumerate() { - let line_number = index + 1; - let line = line?; - if line.trim().is_empty() { + while read_provider_jsonl_line(&mut reader, &mut line)? { + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { continue; } - match serde_json::from_str::(&line) { + match serde_json::from_slice::(&line) { Ok(record) => records.push((line_number, record)), Err(err) => push_provider_import_failure(&mut summary, line_number, err.to_string()), } @@ -4840,6 +4833,64 @@ fn ensure_regular_provider_transcript_file(path: &Path) -> Result<()> { Ok(()) } +fn read_provider_jsonl_line(reader: &mut impl BufRead, buffer: &mut Vec) -> Result { + buffer.clear(); + let mut total = 0usize; + loop { + let available = reader.fill_buf()?; + if available.is_empty() { + return Ok(total > 0); + } + if let Some(newline_index) = available.iter().position(|byte| *byte == b'\n') { + let bytes_to_consume = newline_index + 1; + if total.saturating_add(bytes_to_consume) > MAX_PROVIDER_JSONL_LINE_BYTES { + reader.consume(bytes_to_consume); + return Err(provider_jsonl_line_too_large()); + } + buffer.extend_from_slice(&available[..bytes_to_consume]); + reader.consume(bytes_to_consume); + return Ok(true); + } + + let bytes_to_consume = available.len(); + if total.saturating_add(bytes_to_consume) > MAX_PROVIDER_JSONL_LINE_BYTES { + reader.consume(bytes_to_consume); + discard_provider_jsonl_line(reader)?; + return Err(provider_jsonl_line_too_large()); + } + buffer.extend_from_slice(available); + reader.consume(bytes_to_consume); + total = total.saturating_add(bytes_to_consume); + } +} + +fn discard_provider_jsonl_line(reader: &mut impl BufRead) -> Result<()> { + loop { + let available = reader.fill_buf()?; + if available.is_empty() { + return Ok(()); + } + let bytes_to_consume = available + .iter() + .position(|byte| *byte == b'\n') + .map(|index| index + 1) + .unwrap_or(available.len()); + let found_newline = available + .get(bytes_to_consume.saturating_sub(1)) + .is_some_and(|byte| *byte == b'\n'); + reader.consume(bytes_to_consume); + if found_newline { + return Ok(()); + } + } +} + +fn provider_jsonl_line_too_large() -> CaptureError { + CaptureError::InvalidPayload(format!( + "provider JSONL line exceeds max bytes ({MAX_PROVIDER_JSONL_LINE_BYTES})" + )) +} + fn parse_rfc3339_utc(value: &str) -> Option> { DateTime::parse_from_rfc3339(value) .ok() @@ -6191,17 +6242,18 @@ fn normalize_claude_projects_jsonl_file( ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let reader = BufReader::new(file); + let mut reader = BufReader::new(file); let mut result = ProviderNormalizationResult::default(); let mut rows = Vec::new(); + let mut line = Vec::new(); + let mut line_number = 0usize; - for (index, line) in reader.lines().enumerate() { - let line_number = index + 1; - let line = line?; - if line.trim().is_empty() { + while read_provider_jsonl_line(&mut reader, &mut line)? { + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { continue; } - let value: Value = match serde_json::from_str(&line) { + let value: Value = match serde_json::from_slice(&line) { Ok(value) => value, Err(err) => { result.summary.failed += 1; @@ -6957,12 +7009,7 @@ fn normalize_openclaw_jsonl_file( let mut header_seen = false; let mut line_number = 0usize; let mut line = Vec::new(); - loop { - line.clear(); - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { - break; - } + while read_provider_jsonl_line(&mut reader, &mut line)? { line_number += 1; if line.iter().all(u8::is_ascii_whitespace) { continue; @@ -9730,17 +9777,18 @@ fn normalize_native_jsonl_session_file( ) -> Result { ensure_regular_provider_transcript_file(path)?; let file = File::open(path)?; - let reader = BufReader::new(file); + let mut reader = BufReader::new(file); let mut result = ProviderNormalizationResult::default(); let mut rows = Vec::new(); + let mut line = Vec::new(); + let mut line_number = 0usize; - for (index, line) in reader.lines().enumerate() { - let line_number = index + 1; - let line = line?; - if line.trim().is_empty() { + while read_provider_jsonl_line(&mut reader, &mut line)? { + line_number += 1; + if line.iter().all(u8::is_ascii_whitespace) { continue; } - let value: Value = match serde_json::from_str(&line) { + let value: Value = match serde_json::from_slice(&line) { Ok(value) => value, Err(err) => { result.summary.failed += 1; @@ -12154,6 +12202,10 @@ mod tests { materialized_fixture("custom-history-jsonl", name) } + fn write_oversized_jsonl_line(path: &Path) { + fs::write(path, vec![b'x'; MAX_PROVIDER_JSONL_LINE_BYTES + 1]).unwrap(); + } + fn materialized_fixture(category: &str, name: &str) -> PathBuf { let source = match category { "provider" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -13372,6 +13424,18 @@ mod tests { assert!(store.list_sessions().unwrap().is_empty()); } + #[test] + fn codex_session_jsonl_rejects_oversized_line() { + let temp = tempdir(); + let path = temp.path().join("oversized-codex.jsonl"); + write_oversized_jsonl_line(&path); + + let err = CodexSessionJsonlAdapter + .normalize_path(&path, &ProviderAdapterContext::default()) + .unwrap_err(); + assert!(err.to_string().contains("provider JSONL line exceeds")); + } + #[test] fn codex_session_tree_imports_rich_tool_outputs_and_preserves_previews() { let temp = tempdir(); @@ -15994,6 +16058,24 @@ mod tests { assert_eq!(events, 0); } + #[test] + fn custom_history_jsonl_rejects_oversized_line() { + let temp = tempdir(); + let path = temp.path().join("oversized-custom.jsonl"); + write_oversized_jsonl_line(&path); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let err = import_custom_history_jsonl_v1( + &path, + &mut store, + CustomHistoryJsonlV1ImportOptions::default(), + ) + .unwrap_err(); + + assert!(err.to_string().contains("provider JSONL line exceeds")); + assert_eq!(store.capture_source_count().unwrap(), 0); + } + #[test] fn custom_history_jsonl_preview_overrides_raw_payload_for_searchable_event_payload() { let temp = tempdir(); From 34cde3b898925e9dbd78e75c5e2a2b4ea3ace10a Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:56:12 -0500 Subject: [PATCH 59/72] Reject symlinked provider path parents --- crates/ctx-history-capture/src/lib.rs | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 343155715..7534a2e25 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -4791,6 +4791,7 @@ fn collect_jsonl_paths(root: &Path, paths: &mut Vec) -> Result<()> { reason: "symlinked provider transcript roots are rejected", }); } + ensure_provider_path_parents_are_not_symlinks(root)?; if file_type.is_file() { if root.extension().and_then(|ext| ext.to_str()) == Some("jsonl") { ensure_regular_provider_transcript_file(root)?; @@ -4830,6 +4831,28 @@ fn ensure_regular_provider_transcript_file(path: &Path) -> Result<()> { reason: "provider transcript paths must be regular files", }); } + ensure_provider_path_parents_are_not_symlinks(path)?; + Ok(()) +} + +fn ensure_provider_path_parents_are_not_symlinks(path: &Path) -> Result<()> { + let parent_count = path.components().count().saturating_sub(1); + let mut current = PathBuf::new(); + for component in path.components().take(parent_count) { + current.push(component.as_os_str()); + if current.as_os_str().is_empty() { + continue; + } + let Ok(metadata) = fs::symlink_metadata(¤t) else { + continue; + }; + if metadata.file_type().is_symlink() { + return Err(CaptureError::InvalidProviderTranscriptPath { + path: path.to_path_buf(), + reason: "symlinked provider transcript path components are rejected", + }); + } + } Ok(()) } @@ -13393,6 +13416,40 @@ mod tests { assert!(store.list_sessions().unwrap().is_empty()); } + #[cfg(unix)] + #[test] + fn codex_session_file_rejects_symlinked_parent_components() { + use std::os::unix::fs::symlink; + + let temp = tempdir(); + let real_dir = temp.path().join("real-parent"); + fs::create_dir_all(&real_dir).unwrap(); + let fixture = provider_history_fixture("codex-sessions").join("2026/06/23/root.jsonl"); + fs::copy(&fixture, real_dir.join("root.jsonl")).unwrap(); + let link_dir = temp.path().join("linked-parent"); + symlink(&real_dir, &link_dir).unwrap(); + let linked_file = link_dir.join("root.jsonl"); + + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let err = import_codex_session_jsonl( + &linked_file, + &mut store, + CodexSessionImportOptions { + imported_at: "2026-06-23T16:30:00Z".parse().unwrap(), + ..CodexSessionImportOptions::default() + }, + ) + .unwrap_err(); + + assert!(matches!( + err, + CaptureError::InvalidProviderTranscriptPath { path, reason } + if path.ends_with("linked-parent/root.jsonl") + && reason == "symlinked provider transcript path components are rejected" + )); + assert!(store.list_sessions().unwrap().is_empty()); + } + #[cfg(unix)] #[test] fn codex_session_tree_rejects_symlinked_jsonl_files() { From f3705585662dcd2df51da0360f189ec0a4d09501 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:09:03 -0500 Subject: [PATCH 60/72] Cap upgrade network downloads --- crates/ctx-cli/src/net.rs | 30 ++++++++++++++++++++++++++---- crates/ctx-cli/src/upgrade.rs | 12 ++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/ctx-cli/src/net.rs b/crates/ctx-cli/src/net.rs index e4eeb0c96..f228b1dfe 100644 --- a/crates/ctx-cli/src/net.rs +++ b/crates/ctx-cli/src/net.rs @@ -27,20 +27,33 @@ pub fn post_json(endpoint: &str, body: &[u8]) -> Result<()> { .map_err(|err| anyhow!("POST {endpoint}: {err}")) } -pub fn get_bytes(endpoint: &str) -> Result> { +pub fn get_bytes_limited(endpoint: &str, max_bytes: usize) -> Result> { if let Some(path) = file_url_path(endpoint)? { - return fs::read(&path).with_context(|| format!("read {}", path.display())); + let file = fs::File::open(&path).with_context(|| format!("read {}", path.display()))?; + return read_limited(file, max_bytes, &format!("read {}", path.display())); } require_https_or_localhost(endpoint)?; let response = ureq::get(endpoint) .timeout(std::time::Duration::from_secs(20)) .call() .map_err(|err| anyhow!("GET {endpoint}: {err}"))?; - let mut reader = response.into_reader(); + read_limited( + response.into_reader(), + max_bytes, + &format!("GET {endpoint}"), + ) +} + +fn read_limited(mut reader: impl Read, max_bytes: usize, label: &str) -> Result> { let mut bytes = Vec::new(); reader + .by_ref() + .take((max_bytes as u64).saturating_add(1)) .read_to_end(&mut bytes) - .map_err(|err| anyhow!("read GET {endpoint}: {err}"))?; + .map_err(|err| anyhow!("{label}: {err}"))?; + if bytes.len() > max_bytes { + return Err(anyhow!("{label} exceeds max bytes ({max_bytes})")); + } Ok(bytes) } @@ -109,4 +122,13 @@ mod tests { assert!(require_https_or_localhost("http://example.com/events").is_err()); assert!(require_https_or_localhost("http://example.com@localhost/events").is_err()); } + + #[test] + fn get_bytes_limited_rejects_oversized_file_urls() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("oversized.bin"); + fs::write(&path, b"12345").unwrap(); + let err = get_bytes_limited(&format!("file://{}", path.display()), 4).unwrap_err(); + assert!(err.to_string().contains("exceeds max bytes (4)")); + } } diff --git a/crates/ctx-cli/src/upgrade.rs b/crates/ctx-cli/src/upgrade.rs index b6a759f7c..8074f8b5a 100644 --- a/crates/ctx-cli/src/upgrade.rs +++ b/crates/ctx-cli/src/upgrade.rs @@ -21,6 +21,9 @@ use crate::{config::AppConfig, net}; const STATE_FILE: &str = "upgrade-state.json"; const LOCK_FILE: &str = "upgrade.lock"; const LOG_FILE: &str = "logs/upgrade.log"; +const RELEASE_METADATA_MAX_BYTES: usize = 1024 * 1024; +const RELEASE_METADATA_SIGNATURE_MAX_BYTES: usize = 64 * 1024; +const RELEASE_ARTIFACT_MAX_BYTES: usize = 128 * 1024 * 1024; const VERSION_PROBE_TIMEOUT: Duration = Duration::from_secs(2); const VERSION_PROBE_OUTPUT_LIMIT: usize = 4096; const STALE_UPGRADE_LOCK_AFTER: Duration = Duration::from_secs(30 * 60); @@ -380,7 +383,7 @@ fn apply_upgrade( warnings, }); } - let bytes = net::get_bytes(&plan.artifact_url) + let bytes = net::get_bytes_limited(&plan.artifact_url, RELEASE_ARTIFACT_MAX_BYTES) .with_context(|| format!("download {}", plan.artifact_url))?; verify_artifact_sha(&bytes, &plan.artifact_sha256)?; let apply_result = apply_artifact(&plan, &bytes)?; @@ -444,10 +447,11 @@ fn build_upgrade_plan( warnings.extend(path.warnings.clone()); let metadata_url = metadata_url(config, &channel); let signature_url = metadata_signature_url(&metadata_url); - let metadata_bytes = net::get_bytes(&metadata_url) + let metadata_bytes = net::get_bytes_limited(&metadata_url, RELEASE_METADATA_MAX_BYTES) .with_context(|| format!("download release metadata {metadata_url}"))?; - let signature_bytes = net::get_bytes(&signature_url) - .with_context(|| format!("download release metadata signature {signature_url}"))?; + let signature_bytes = + net::get_bytes_limited(&signature_url, RELEASE_METADATA_SIGNATURE_MAX_BYTES) + .with_context(|| format!("download release metadata signature {signature_url}"))?; verify_metadata_signature(&metadata_bytes, &signature_bytes)?; let metadata = parse_release_metadata(&metadata_bytes, &platform, &channel)?; let artifact_url = format!( From f366f7675e12ad841115f76c5b4fb0da0649f6e4 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:24:58 -0500 Subject: [PATCH 61/72] Reject malformed provider times --- crates/ctx-history-capture/src/lib.rs | 311 ++++++++++++++++++++++---- 1 file changed, 266 insertions(+), 45 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 7534a2e25..50a107209 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -1232,11 +1232,17 @@ impl ProviderCaptureAdapter for CodexSessionJsonlAdapter { }); continue; }; - let occurred_at = value - .get("timestamp") - .and_then(Value::as_str) - .and_then(parse_rfc3339_utc) - .unwrap_or(header.timestamp); + let occurred_at = match codex_session_line_timestamp(&value, header.timestamp) { + Ok(occurred_at) => occurred_at, + Err(err) => { + result.summary.failed += 1; + result.summary.failures.push(ProviderImportFailure { + line: line_number, + error: err.to_string(), + }); + continue; + } + }; let mut line_capture = codex_session_line_capture( header, &value, @@ -1467,7 +1473,7 @@ fn normalize_pi_session_jsonl_file( if entry_type == "session" { match pi_session_header(value) { Ok(parsed) => { - let capture = pi_session_capture(&parsed, None, line_number, context); + let capture = pi_session_capture(&parsed, None, line_number, context)?; header = Some(parsed); result.captures.push((line_number, capture)); } @@ -1490,10 +1496,16 @@ fn normalize_pi_session_jsonl_file( }); continue; }; - result.captures.push(( - line_number, - pi_session_capture(header, Some(value), line_number, context), - )); + match pi_session_capture(header, Some(value), line_number, context) { + Ok(capture) => result.captures.push((line_number, capture)), + Err(err) => { + result.summary.failed += 1; + result.summary.failures.push(ProviderImportFailure { + line: line_number, + error: err.to_string(), + }); + } + } } Ok(result) @@ -2262,11 +2274,20 @@ pub fn import_codex_session_jsonl_tail( { continue; } - let occurred_at = value - .get("timestamp") - .and_then(Value::as_str) - .and_then(parse_rfc3339_utc) - .unwrap_or(header.timestamp); + let occurred_at = match codex_session_line_timestamp(&value, header.timestamp) { + Ok(occurred_at) => occurred_at, + Err(err) => { + summary.failed += 1; + summary.failures.push(ProviderImportFailure { + line: line_number, + error: err.to_string(), + }); + if !options.allow_partial_failures { + return Ok(summary); + } + continue; + } + }; let mut line_capture = codex_session_line_capture( &header, &value, @@ -2803,11 +2824,20 @@ fn import_codex_session_path_fast( } continue; }; - let occurred_at = value - .get("timestamp") - .and_then(Value::as_str) - .and_then(parse_rfc3339_utc) - .unwrap_or(header.timestamp); + let occurred_at = match codex_session_line_timestamp(&value, header.timestamp) { + Ok(occurred_at) => occurred_at, + Err(err) => { + summary.failed += 1; + summary.failures.push(ProviderImportFailure { + line: line_number, + error: err.to_string(), + }); + if !options.allow_partial_failures { + return Ok(()); + } + continue; + } + }; let mut line_capture = codex_session_line_capture( header, &value, @@ -2886,7 +2916,7 @@ fn import_codex_provider_event_fast( event, payload: &payload, event_hash: &event_hash, - }); + })?; let normalized_event = Event { id: event_identity.id, seq: event_identity.seq, @@ -4920,6 +4950,27 @@ fn parse_rfc3339_utc(value: &str) -> Option> { .map(|time| time.with_timezone(&Utc)) } +fn parse_optional_rfc3339_field( + value: &Value, + field: &'static str, +) -> Result>> { + let Some(raw_value) = value.get(field) else { + return Ok(None); + }; + let raw = raw_value.as_str().ok_or_else(|| { + CaptureError::InvalidPayload(format!("{field} must be an RFC3339 string")) + })?; + parse_rfc3339_utc(raw) + .ok_or_else(|| { + CaptureError::InvalidPayload(format!("{field} is not a valid RFC3339 timestamp")) + }) + .map(Some) +} + +fn codex_session_line_timestamp(value: &Value, fallback: DateTime) -> Result> { + Ok(parse_optional_rfc3339_field(value, "timestamp")?.unwrap_or(fallback)) +} + fn codex_session_header(value: Value) -> Result { let payload = value .get("payload") @@ -10473,8 +10524,10 @@ fn pi_session_capture( entry: Option, line_number: usize, context: &ProviderAdapterContext, -) -> ProviderCaptureEnvelope { - let event = entry.map(|entry| pi_session_event(header, &entry, line_number)); +) -> Result { + let event = entry + .map(|entry| pi_session_event(header, &entry, line_number)) + .transpose()?; let cursor = event.as_ref().and_then(|event| { event.cursor.as_ref().map(|cursor| ProviderCursorRange { before: None, @@ -10486,7 +10539,7 @@ fn pi_session_capture( }) }); - ProviderCaptureEnvelope { + Ok(ProviderCaptureEnvelope { schema_version: PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, provider: CaptureProvider::Pi, source: ProviderSourceEnvelope { @@ -10537,14 +10590,14 @@ fn pi_session_capture( }), }, event, - } + }) } fn pi_session_event( header: &PiSessionHeader, entry: &Value, line_number: usize, -) -> ProviderEventEnvelope { +) -> Result { let entry_type = entry .get("type") .and_then(Value::as_str) @@ -10553,17 +10606,14 @@ fn pi_session_event( let message_role = message .and_then(|message| message.get("role")) .and_then(Value::as_str); - let occurred_at = entry - .get("timestamp") - .and_then(Value::as_str) - .and_then(|timestamp| DateTime::parse_from_rfc3339(timestamp).ok()) - .map(|time| time.with_timezone(&Utc)) - .unwrap_or_else(utc_now); + let occurred_at = parse_optional_rfc3339_field(entry, "timestamp")?.ok_or_else(|| { + CaptureError::InvalidPayload("pi session event missing timestamp".to_owned()) + })?; let event_type = pi_event_type(entry_type, message); let role = message_role.map(pi_event_role); let text = message.and_then(pi_message_text); - ProviderEventEnvelope { + Ok(ProviderEventEnvelope { provider_event_index: (line_number - 1) as u64, provider_event_hash: None, cursor: entry.get("id").and_then(Value::as_str).map(str::to_owned), @@ -10598,7 +10648,7 @@ fn pi_session_event( .and_then(Value::as_str), "usage": message.and_then(|message| message.get("usage")).cloned(), }), - } + }) } fn pi_event_type(entry_type: &str, message: Option<&Value>) -> EventType { @@ -11088,7 +11138,7 @@ fn import_provider_capture_line( event, payload: &payload, event_hash: &event_hash, - }); + })?; let normalized_event = Event { id: event_identity.id, seq: event_identity.seq, @@ -11899,7 +11949,7 @@ struct ProviderCommandRunInput<'a> { event_hash: &'a str, } -fn provider_command_run_from_event(input: ProviderCommandRunInput<'_>) -> Option { +fn provider_command_run_from_event(input: ProviderCommandRunInput<'_>) -> Result> { let ProviderCommandRunInput { provider, provider_session_id, @@ -11912,7 +11962,7 @@ fn provider_command_run_from_event(input: ProviderCommandRunInput<'_>) -> Option event_hash, } = input; if event.event_type != EventType::CommandOutput { - return None; + return Ok(None); } let command_preview = payload .get("command") @@ -11921,16 +11971,29 @@ fn provider_command_run_from_event(input: ProviderCommandRunInput<'_>) -> Option .map(str::to_owned); let call_id = payload.get("call_id").and_then(Value::as_str); let key = call_id.unwrap_or(event_hash); - let duration_ms = payload.get("duration_ms").and_then(Value::as_i64); + let duration_ms = provider_command_duration_ms(payload)?; let ended_at = Some(event.occurred_at); - let started_at = duration_ms - .and_then(|duration| { + let started_at = match duration_ms { + Some(duration) => { + let duration_value = duration; + let duration = chrono::Duration::try_milliseconds(duration_value).ok_or_else(|| { + CaptureError::InvalidPayload(format!( + "duration_ms is not representable as milliseconds: {duration_value}" + )) + })?; event .occurred_at - .checked_sub_signed(chrono::Duration::milliseconds(duration.max(0))) - }) - .unwrap_or(event.occurred_at); - Some(Run { + .checked_sub_signed(duration) + .ok_or_else(|| { + CaptureError::InvalidPayload(format!( + "duration_ms moves command start before representable time: {}", + duration_value + )) + })? + } + None => event.occurred_at, + }; + Ok(Some(Run { id: run_source_id .map(|source_id| provider_source_run_uuid(source_id, key)) .unwrap_or_else(|| provider_run_uuid(provider, provider_session_id, key)), @@ -11960,7 +12023,25 @@ fn provider_command_run_from_event(input: ProviderCommandRunInput<'_>) -> Option "source": "provider_command_output", }), ), - }) + })) +} + +fn provider_command_duration_ms(payload: &Value) -> Result> { + let Some(value) = payload.get("duration_ms") else { + return Ok(None); + }; + if value.is_null() { + return Ok(None); + } + let duration = value + .as_i64() + .ok_or_else(|| CaptureError::InvalidPayload("duration_ms must be an integer".to_owned()))?; + if duration < 0 { + return Err(CaptureError::InvalidPayload(format!( + "duration_ms must be nonnegative, got {duration}" + ))); + } + Ok(Some(duration)) } fn provider_command_run_status(payload: &Value) -> RunStatus { @@ -12229,6 +12310,27 @@ mod tests { fs::write(path, vec![b'x'; MAX_PROVIDER_JSONL_LINE_BYTES + 1]).unwrap(); } + fn jsonl_line(value: Value) -> String { + serde_json::to_string(&value).unwrap() + "\n" + } + + fn test_provider_event(event_type: EventType) -> ProviderEventEnvelope { + ProviderEventEnvelope { + provider_event_index: 0, + provider_event_hash: Some("event-hash".to_owned()), + cursor: None, + event_type, + role: Some(EventRole::Tool), + occurred_at: "2026-07-03T12:00:00Z".parse().unwrap(), + fidelity: Fidelity::Imported, + redaction_state: RedactionState::LocalPreview, + idempotency_key: None, + artifacts: Vec::new(), + payload: json!({}), + metadata: json!({}), + } + } + fn materialized_fixture(category: &str, name: &str) -> PathBuf { let source = match category { "provider" => PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -12825,6 +12927,51 @@ mod tests { assert!(!events[3].payload.to_string().contains("[REDACTED]")); } + #[test] + fn pi_session_import_rejects_malformed_event_timestamp() { + let temp = tempdir(); + let path = temp.path().join("bad-timestamp-pi.jsonl"); + fs::write( + &path, + [ + jsonl_line(json!({ + "type": "session", + "id": "pi-bad-timestamp", + "timestamp": "2026-07-03T12:00:00Z", + "version": 1 + })), + jsonl_line(json!({ + "type": "message", + "id": "pi-bad-event", + "timestamp": "not-rfc3339", + "message": { + "role": "user", + "content": "bad timestamp should not import" + } + })), + ] + .concat(), + ) + .unwrap(); + + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let summary = import_pi_session_jsonl( + &path, + &mut store, + PiSessionImportOptions { + imported_at: "2026-07-03T12:30:00Z".parse().unwrap(), + ..PiSessionImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 1, "{:?}", summary.failures); + assert!(summary.failures[0] + .error + .contains("timestamp is not a valid RFC3339 timestamp")); + assert!(store.list_sessions().unwrap().is_empty()); + } + #[test] fn pi_session_import_replays_default_session_directory_tree() { let temp = tempdir(); @@ -13493,6 +13640,80 @@ mod tests { assert!(err.to_string().contains("provider JSONL line exceeds")); } + #[test] + fn codex_session_jsonl_rejects_malformed_event_timestamp() { + let temp = tempdir(); + let path = temp.path().join("bad-timestamp-codex.jsonl"); + fs::write( + &path, + [ + jsonl_line(json!({ + "timestamp": "2026-07-03T12:00:00Z", + "type": "session_meta", + "payload": { + "id": "codex-bad-timestamp", + "timestamp": "2026-07-03T12:00:00Z", + "cwd": "/workspace", + "originator": "codex-cli" + } + })), + jsonl_line(json!({ + "timestamp": "not-rfc3339", + "type": "response_item", + "payload": { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "bad timestamp should not import"} + ] + } + })), + ] + .concat(), + ) + .unwrap(); + + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let summary = import_codex_session_jsonl( + &path, + &mut store, + CodexSessionImportOptions { + imported_at: "2026-07-03T12:30:00Z".parse().unwrap(), + fast_event_inserts: false, + ..CodexSessionImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 1, "{:?}", summary.failures); + assert!(summary.failures[0] + .error + .contains("timestamp is not a valid RFC3339 timestamp")); + assert!(store.list_sessions().unwrap().is_empty()); + } + + #[test] + fn provider_command_run_rejects_negative_duration() { + let event = test_provider_event(EventType::CommandOutput); + let err = provider_command_run_from_event(ProviderCommandRunInput { + provider: CaptureProvider::Codex, + provider_session_id: "duration-session", + session_id: new_id(), + source_id: new_id(), + run_source_id: None, + history_record_id: None, + event: &event, + payload: &json!({ + "command": "cargo test", + "duration_ms": -1 + }), + event_hash: "event-hash", + }) + .unwrap_err(); + + assert!(err.to_string().contains("duration_ms must be nonnegative")); + } + #[test] fn codex_session_tree_imports_rich_tool_outputs_and_preserves_previews() { let temp = tempdir(); From 1a09219dd02016bb51170cf2996256fc93a8b922 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:26:25 -0500 Subject: [PATCH 62/72] Bound Codex catalog line reads --- crates/ctx-history-capture/src/lib.rs | 43 ++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 50a107209..6d27ca2c4 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -3311,17 +3311,18 @@ fn catalog_codex_session_file( }) } -fn read_codex_session_meta(path: &Path) -> std::io::Result> { +fn read_codex_session_meta(path: &Path) -> Result> { let file = File::open(path)?; - let reader = BufReader::new(file); - for line in reader.lines().take(32) { - let line = line?; - if !line.as_bytes().contains(&b'{') - || !contains_bytes(line.as_bytes(), br#""session_meta""#) - { + let mut reader = BufReader::new(file); + let mut line = Vec::new(); + for _ in 0..32 { + if !read_provider_jsonl_line(&mut reader, &mut line)? { + break; + } + if !line.contains(&b'{') || !contains_bytes(&line, br#""session_meta""#) { continue; } - let Ok(value) = serde_json::from_str::(&line) else { + let Ok(value) = serde_json::from_slice::(&line) else { continue; }; if value.get("type").and_then(Value::as_str) == Some("session_meta") { @@ -13184,6 +13185,32 @@ mod tests { assert_eq!(third.failed_sessions, 0); } + #[test] + fn codex_session_catalog_rejects_oversized_metadata_line() { + let temp = tempdir(); + let root = temp.path().join("sessions/2026/07/03"); + fs::create_dir_all(&root).unwrap(); + write_oversized_jsonl_line(&root.join("oversized.jsonl")); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let err = catalog_codex_session_tree( + temp.path().join("sessions"), + &store, + CodexSessionCatalogOptions { + source_root: Some(temp.path().join("sessions")), + cataloged_at: "2026-07-03T12:00:00Z".parse().unwrap(), + allow_partial_failures: false, + ..CodexSessionCatalogOptions::default() + }, + ) + .unwrap_err(); + + assert!( + err.to_string().contains("provider JSONL line exceeds"), + "{err}" + ); + } + #[test] fn codex_session_catalog_marks_deleted_paths_stale_when_additions_outnumber_deletions() { let temp = tempdir(); From 2179b963fc1bca17caa90106f4366b03c8904869 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:29:09 -0500 Subject: [PATCH 63/72] Bound show event window queries --- crates/ctx-cli/src/main.rs | 14 +--- crates/ctx-history-store/src/lib.rs | 110 ++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 13 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 5704d01c0..1f1c3d4cb 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -3117,22 +3117,10 @@ fn event_window( after: usize, window: Option, ) -> Result> { - let Some(session_id) = event.session_id else { - return Ok(vec![event.clone()]); - }; - let events = store.events_for_session(session_id)?; - let Some(index) = events.iter().position(|candidate| candidate.id == event.id) else { - return Ok(vec![event.clone()]); - }; let (before, after) = window .map(|window| (window, window)) .unwrap_or((before, after)); - let start = index.saturating_sub(before); - let end = index - .saturating_add(after) - .saturating_add(1) - .min(events.len()); - Ok(events[start..end].to_vec()) + Ok(store.events_for_session_window(event, before, after)?) } fn write_rendered_session( diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index 6b72a1033..ef6262104 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -2707,6 +2707,58 @@ impl Store { collect_rows(rows) } + pub fn events_for_session_window( + &self, + event: &Event, + before: usize, + after: usize, + ) -> Result> { + let Some(session_id) = event.session_id else { + return Ok(vec![event.clone()]); + }; + let event_seq = i64::try_from(event.seq).unwrap_or(i64::MAX); + let mut events = if before == 0 { + Vec::new() + } else { + let mut stmt = self.conn.prepare( + event_select_sql( + "WHERE session_id = ?1 AND seq < ?2 ORDER BY seq DESC, occurred_at_ms DESC LIMIT ?3", + ) + .as_str(), + )?; + let rows = stmt.query_map( + params![ + session_id.to_string(), + event_seq, + i64::try_from(before).unwrap_or(i64::MAX) + ], + event_from_row, + )?; + let mut rows = collect_rows(rows)?; + rows.reverse(); + rows + }; + events.push(event.clone()); + if after > 0 { + let mut stmt = self.conn.prepare( + event_select_sql( + "WHERE session_id = ?1 AND seq > ?2 ORDER BY seq, occurred_at_ms LIMIT ?3", + ) + .as_str(), + )?; + let rows = stmt.query_map( + params![ + session_id.to_string(), + event_seq, + i64::try_from(after).unwrap_or(i64::MAX) + ], + event_from_row, + )?; + events.extend(collect_rows(rows)?); + } + Ok(events) + } + pub fn events_for_record(&self, record_id: Uuid) -> Result> { let mut stmt = self.conn.prepare( event_select_sql( @@ -7943,6 +7995,25 @@ mod catalog_tests { } } + fn session_event(session_id: Uuid, index: u64) -> Event { + Event { + id: new_id(), + seq: index, + history_record_id: None, + session_id: Some(session_id), + run_id: None, + event_type: EventType::Message, + role: Some(EventRole::Assistant), + occurred_at: fixed_time() + chrono::Duration::seconds(index as i64), + capture_source_id: None, + payload: serde_json::json!({"index": index}), + payload_blob_id: None, + dedupe_key: None, + redaction_state: RedactionState::LocalPreview, + sync: sync_metadata(), + } + } + fn artifact_record(id: Uuid, byte_size: u64) -> Artifact { Artifact { id, @@ -8008,6 +8079,45 @@ mod catalog_tests { assert!(after_changed > after_noop); } + #[test] + fn events_for_session_window_returns_bounded_neighbors() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + let session = imported_session("window-session"); + store.upsert_session(&session).unwrap(); + let events = (0..10) + .map(|index| { + let event = session_event(session.id, index); + store.upsert_event(&event).unwrap(); + event + }) + .collect::>(); + + let middle = store + .events_for_session_window(&events[5], 2, 3) + .unwrap() + .into_iter() + .map(|event| event.seq) + .collect::>(); + assert_eq!(middle, vec![3, 4, 5, 6, 7, 8]); + + let first = store + .events_for_session_window(&events[0], 50, 1) + .unwrap() + .into_iter() + .map(|event| event.seq) + .collect::>(); + assert_eq!(first, vec![0, 1]); + + let last = store + .events_for_session_window(&events[9], 1, 50) + .unwrap() + .into_iter() + .map(|event| event.seq) + .collect::>(); + assert_eq!(last, vec![8, 9]); + } + #[test] fn search_index_optimize_is_safe_on_initialized_store() { let temp = tempdir(); From ee1bcddfa1c01f745732fb5c64bc0a35d7d387c8 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:30:09 -0500 Subject: [PATCH 64/72] Limit provider session lookup --- crates/ctx-cli/src/main.rs | 9 +----- crates/ctx-history-store/src/lib.rs | 48 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index 1f1c3d4cb..e3bf9f99a 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -3091,14 +3091,7 @@ fn resolve_session( .ok_or_else(|| { anyhow!("session lookup requires --provider-session when no ctx session id is provided") })?; - let matches = store - .list_sessions()? - .into_iter() - .filter(|session| { - session.provider == provider - && session.external_session_id.as_deref() == Some(provider_session) - }) - .collect::>(); + let matches = store.sessions_by_external_session_limited(provider, provider_session, 2)?; match matches.as_slice() { [session] => Ok(session.clone()), [] => Err(anyhow!( diff --git a/crates/ctx-history-store/src/lib.rs b/crates/ctx-history-store/src/lib.rs index ef6262104..f31ebb283 100644 --- a/crates/ctx-history-store/src/lib.rs +++ b/crates/ctx-history-store/src/lib.rs @@ -2284,6 +2284,29 @@ impl Store { .map_err(StoreError::from) } + pub fn sessions_by_external_session_limited( + &self, + provider: CaptureProvider, + external_session_id: &str, + limit: usize, + ) -> Result> { + let mut stmt = self.conn.prepare( + session_select_sql( + "WHERE provider = ?1 AND external_session_id = ?2 ORDER BY started_at_ms DESC LIMIT ?3", + ) + .as_str(), + )?; + let rows = stmt.query_map( + params![ + provider.as_str(), + external_session_id, + i64::try_from(limit).unwrap_or(i64::MAX) + ], + session_from_row, + )?; + collect_rows(rows) + } + pub fn sessions_for_record(&self, record_id: Uuid) -> Result> { let mut stmt = self.conn.prepare( session_select_sql("WHERE history_record_id = ?1 ORDER BY started_at_ms, id").as_str(), @@ -8118,6 +8141,31 @@ mod catalog_tests { assert_eq!(last, vec![8, 9]); } + #[test] + fn sessions_by_external_session_limited_caps_ambiguity_scan() { + let temp = tempdir(); + let store = Store::open(temp.path().join("work.sqlite")).unwrap(); + for index in 0..5 { + let mut session = imported_session("shared-provider-session"); + session.started_at = fixed_time() + chrono::Duration::seconds(index); + store.upsert_session(&session).unwrap(); + } + + let matches = store + .sessions_by_external_session_limited( + CaptureProvider::Codex, + "shared-provider-session", + 2, + ) + .unwrap(); + + assert_eq!(matches.len(), 2); + assert_eq!( + matches[0].external_session_id.as_deref(), + Some("shared-provider-session") + ); + } + #[test] fn search_index_optimize_is_safe_on_initialized_store() { let temp = tempdir(); From e0077d356b58ac7dab616cc328ba2771d9734ae5 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:31:18 -0500 Subject: [PATCH 65/72] Cap history plugin manifests --- crates/ctx-cli/src/history_source_plugins.rs | 25 ++++++++++++++-- crates/ctx-cli/tests/cli.rs | 30 ++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/crates/ctx-cli/src/history_source_plugins.rs b/crates/ctx-cli/src/history_source_plugins.rs index a3c438490..24357b6e1 100644 --- a/crates/ctx-cli/src/history_source_plugins.rs +++ b/crates/ctx-cli/src/history_source_plugins.rs @@ -19,6 +19,7 @@ use serde::Deserialize; use uuid::Uuid; const PLUGIN_MANIFEST_FILE: &str = "ctx-history-plugin.json"; +const MAX_PLUGIN_MANIFEST_BYTES: usize = 1024 * 1024; const DEFAULT_PLUGIN_TIMEOUT_SECONDS: u64 = 300; const MAX_PLUGIN_STDOUT_BYTES: usize = 64 * 1024 * 1024; const MAX_PLUGIN_STDERR_BYTES: usize = 256 * 1024; @@ -548,8 +549,7 @@ fn cleanup_cursor_file(path: Option<&PathBuf>) { } fn read_plugin_manifest(path: &Path) -> Result> { - let raw = fs::read_to_string(path) - .with_context(|| format!("read history source plugin manifest {}", path.display()))?; + let raw = read_plugin_manifest_text(path)?; let manifest: HistorySourcePluginManifest = serde_json::from_str(&raw) .with_context(|| format!("parse history source plugin manifest {}", path.display()))?; validate_plugin_id("plugin name", &manifest.name)?; @@ -612,6 +612,27 @@ fn read_plugin_manifest(path: &Path) -> Result> { Ok(sources) } +fn read_plugin_manifest_text(path: &Path) -> Result { + let file = fs::File::open(path) + .with_context(|| format!("read history source plugin manifest {}", path.display()))?; + let mut bytes = Vec::new(); + file.take((MAX_PLUGIN_MANIFEST_BYTES as u64).saturating_add(1)) + .read_to_end(&mut bytes) + .with_context(|| format!("read history source plugin manifest {}", path.display()))?; + if bytes.len() > MAX_PLUGIN_MANIFEST_BYTES { + return Err(anyhow!( + "history source plugin manifest {} exceeds max bytes ({MAX_PLUGIN_MANIFEST_BYTES})", + path.display() + )); + } + String::from_utf8(bytes).with_context(|| { + format!( + "history source plugin manifest {} is not UTF-8", + path.display() + ) + }) +} + fn plugin_manifest_paths(data_root: &Path) -> Vec { let mut candidates = BTreeSet::new(); collect_manifest_path_candidates(&data_root.join("plugins"), &mut candidates); diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index f1e22002e..ad7ff6c37 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1255,6 +1255,36 @@ fn invalid_installed_history_source_plugin_is_listed_as_invalid() { .contains("parse history source plugin manifest")); } +#[test] +fn oversized_installed_history_source_plugin_is_listed_as_invalid() { + let temp = tempdir(); + let plugin_root = temp.path().join("history-plugins"); + let bad_dir = plugin_root.join("oversized"); + fs::create_dir_all(&bad_dir).unwrap(); + fs::write( + bad_dir.join("ctx-history-plugin.json"), + vec![b' '; 2 * 1024 * 1024], + ) + .unwrap(); + + let sources = json_output( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin_root) + .args(["sources", "--json"]), + ); + let invalid = sources["sources"] + .as_array() + .unwrap() + .iter() + .find(|source| source["kind"] == "history_source_plugin" && source["status"] == "invalid") + .unwrap(); + assert_eq!(invalid["importable"], false); + assert!(invalid["error"] + .as_str() + .unwrap() + .contains("exceeds max bytes")); +} + #[test] fn invalid_installed_history_source_plugin_does_not_block_valid_import() { let temp = tempdir(); From bb5ba724e2ab9be609c317ad31dc7ca0c48fcf35 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:33:33 -0500 Subject: [PATCH 66/72] Cap history plugin output lines --- crates/ctx-cli/src/main.rs | 65 ++++++++++++++++++++++++++++--------- crates/ctx-cli/tests/cli.rs | 25 ++++++++++++++ 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/crates/ctx-cli/src/main.rs b/crates/ctx-cli/src/main.rs index e3bf9f99a..8a8cc5922 100644 --- a/crates/ctx-cli/src/main.rs +++ b/crates/ctx-cli/src/main.rs @@ -73,6 +73,7 @@ const LARGE_IMPORT_SOURCE_FILES_WARNING: usize = 10_000; const LARGE_IMPORT_SOURCE_BYTES_WARNING: u64 = 1024 * 1024 * 1024; const MAX_SEARCH_LIMIT: usize = 200; pub(crate) const MAX_EVENT_WINDOW: usize = 50; +const MAX_HISTORY_SOURCE_PLUGIN_JSONL_LINE_BYTES: usize = 16 * 1024 * 1024; #[derive(Debug, Parser)] #[command(name = "ctx", version, about = "Search local agent history")] @@ -5169,15 +5170,8 @@ fn annotate_history_source_plugin_output( source: &HistorySourcePluginSource, stdout: &[u8], ) -> Result> { - let text = std::str::from_utf8(stdout).with_context(|| { - format!( - "history source plugin {} emitted non-UTF-8 ctx-history-jsonl-v1 output", - source.label() - ) - })?; let mut out = Vec::with_capacity(stdout.len()); - for (index, line) in text.lines().enumerate() { - let line_number = index + 1; + for (line_number, line) in history_source_plugin_stdout_lines(source, stdout)? { if line.trim().is_empty() { continue; } @@ -5230,16 +5224,9 @@ fn validate_history_source_plugin_output( machine_id: &str, require_after_cursor: bool, ) -> Result<()> { - let text = std::str::from_utf8(stdout).with_context(|| { - format!( - "history source plugin {} emitted non-UTF-8 ctx-history-jsonl-v1 output", - source.label() - ) - })?; let mut saw_source = false; let mut saw_after_cursor = false; - for (index, line) in text.lines().enumerate() { - let line_number = index + 1; + for (line_number, line) in history_source_plugin_stdout_lines(source, stdout)? { if line.trim().is_empty() { continue; } @@ -5300,6 +5287,52 @@ fn validate_history_source_plugin_output( Ok(()) } +fn history_source_plugin_stdout_lines<'a>( + source: &HistorySourcePluginSource, + stdout: &'a [u8], +) -> Result> { + let mut lines = Vec::new(); + let mut start = 0usize; + let mut line_number = 1usize; + for (index, byte) in stdout.iter().enumerate() { + let len = index.saturating_add(1).saturating_sub(start); + if len > MAX_HISTORY_SOURCE_PLUGIN_JSONL_LINE_BYTES { + return Err(anyhow!( + "history source plugin {} emitted ctx-history-jsonl-v1 line {line_number} exceeding max bytes ({MAX_HISTORY_SOURCE_PLUGIN_JSONL_LINE_BYTES})", + source.label() + )); + } + if *byte == b'\n' { + let line = std::str::from_utf8(&stdout[start..index]).with_context(|| { + format!( + "history source plugin {} emitted non-UTF-8 ctx-history-jsonl-v1 output at line {line_number}", + source.label() + ) + })?; + lines.push((line_number, line)); + start = index + 1; + line_number += 1; + } + } + if start < stdout.len() { + let len = stdout.len().saturating_sub(start); + if len > MAX_HISTORY_SOURCE_PLUGIN_JSONL_LINE_BYTES { + return Err(anyhow!( + "history source plugin {} emitted ctx-history-jsonl-v1 line {line_number} exceeding max bytes ({MAX_HISTORY_SOURCE_PLUGIN_JSONL_LINE_BYTES})", + source.label() + )); + } + let line = std::str::from_utf8(&stdout[start..]).with_context(|| { + format!( + "history source plugin {} emitted non-UTF-8 ctx-history-jsonl-v1 output at line {line_number}", + source.label() + ) + })?; + lines.push((line_number, line)); + } + Ok(lines) +} + fn history_source_plugin_import_failure( source: &HistorySourcePluginSource, summary: &ProviderImportSummary, diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index ad7ff6c37..3f2d5584d 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -1490,6 +1490,31 @@ for record in records: ); } +#[test] +fn history_source_plugin_rejects_oversized_stdout_line() { + let temp = tempdir(); + let script = r#"#!/usr/bin/env python3 +import sys +sys.stdout.write("x" * (17 * 1024 * 1024) + "\n") +"#; + let plugin = write_raw_history_source_plugin(&temp, "bigline", script); + + let stderr = failure_stderr( + ctx(&temp) + .env("CTX_HISTORY_PLUGIN_PATH", &plugin.manifest_dir) + .args([ + "import", + "--history-source", + "bigline/default", + "--json", + "--progress", + "none", + ]), + ); + + assert!(stderr.contains("line 1 exceeding max bytes"), "{stderr}"); +} + #[test] fn history_source_plugin_reset_requires_fresh_after_cursor() { let temp = tempdir(); From fcb1c1c7ab9a2980a1466987684e6bdfd05de307 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:42:37 -0500 Subject: [PATCH 67/72] Cap native provider SQLite values --- crates/ctx-history-capture/src/lib.rs | 38 ++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 6d27ca2c4..7c9272711 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -25,7 +25,7 @@ use ctx_history_core::{ CTX_HISTORY_JSONL_V1_SCHEMA_VERSION, PROVIDER_CAPTURE_ENVELOPE_SCHEMA_VERSION, }; use ctx_history_store::{CatalogSession, Store, StoreError}; -use rusqlite::{Connection, OpenFlags, OptionalExtension}; +use rusqlite::{limits::Limit, Connection, OpenFlags, OptionalExtension}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use thiserror::Error; @@ -41,6 +41,7 @@ pub use provider_sources::{ pub const CAPTURE_SCHEMA_VERSION: u32 = 1; const MAX_PROVIDER_JSONL_LINE_BYTES: usize = 16 * 1024 * 1024; +const MAX_PROVIDER_SQLITE_VALUE_BYTES: usize = MAX_PROVIDER_JSONL_LINE_BYTES; #[derive(Debug, Error)] pub enum CaptureError { #[error("io error: {0}")] @@ -6815,6 +6816,12 @@ fn open_provider_sqlite_readonly(path: &Path) -> Result { path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; + let value_limit = i32::try_from(MAX_PROVIDER_SQLITE_VALUE_BYTES).map_err(|_| { + CaptureError::InvalidPayload(format!( + "provider SQLite value byte limit is unrepresentable: {MAX_PROVIDER_SQLITE_VALUE_BYTES}" + )) + })?; + conn.set_limit(Limit::SQLITE_LIMIT_LENGTH, value_limit); conn.busy_timeout(std::time::Duration::from_secs(5))?; conn.pragma_update(None, "query_only", true)?; Ok(conn) @@ -14402,6 +14409,35 @@ mod tests { })); } + #[test] + fn native_opencode_rejects_oversized_sqlite_text_value() { + let temp = tempdir(); + let fixture = write_opencode_smoke_db(&temp, false); + let conn = Connection::open(&fixture).unwrap(); + let oversized_data = format!( + "{{\"time\":{{\"created\":1782259200000}},\"text\":\"{}\"}}", + "x".repeat(MAX_PROVIDER_SQLITE_VALUE_BYTES + 1) + ); + conn.execute( + "update session_message set data = ?1 where id = 'msg-user'", + [&oversized_data], + ) + .unwrap(); + drop(conn); + + let err = import_opencode_sqlite( + &fixture, + &mut Store::open(temp.path().join("work.sqlite")).unwrap(), + OpenCodeSqliteImportOptions::default(), + ) + .unwrap_err(); + + assert!( + err.to_string().contains("too big"), + "unexpected error: {err}" + ); + } + #[test] fn native_opencode_reports_malformed_and_corrupt_db() { let temp = tempdir(); From 7fe9bcbc7d67a16dbd81f08f384b6d25c422b479 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:45:21 -0500 Subject: [PATCH 68/72] Cap OpenClaw session sidecars --- crates/ctx-history-capture/src/lib.rs | 109 ++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 7c9272711..7f806e60b 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -3,7 +3,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, env, fs::{self, File}, - io::{BufRead, BufReader, BufWriter, Write}, + io::{BufRead, BufReader, BufWriter, Read, Write}, path::{Path, PathBuf}, sync::Arc, thread, @@ -42,6 +42,9 @@ pub use provider_sources::{ pub const CAPTURE_SCHEMA_VERSION: u32 = 1; const MAX_PROVIDER_JSONL_LINE_BYTES: usize = 16 * 1024 * 1024; const MAX_PROVIDER_SQLITE_VALUE_BYTES: usize = MAX_PROVIDER_JSONL_LINE_BYTES; +const MAX_OPENCLAW_SESSION_INDEX_BYTES: usize = 1024 * 1024; +const MAX_OPENCLAW_SESSION_INDEX_PATHS: usize = 256; +const MAX_OPENCLAW_SESSION_INDEX_VISITED_PATHS: usize = 4096; #[derive(Debug, Error)] pub enum CaptureError { #[error("io error: {0}")] @@ -4888,6 +4891,20 @@ fn ensure_provider_path_parents_are_not_symlinks(path: &Path) -> Result<()> { Ok(()) } +fn read_text_file_limited(path: &Path, max_bytes: usize, label: &str) -> Result { + let file = File::open(path)?; + let mut reader = file.take((max_bytes as u64).saturating_add(1)); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes)?; + if bytes.len() > max_bytes { + return Err(CaptureError::InvalidPayload(format!( + "{label} exceeds max bytes ({max_bytes})" + ))); + } + String::from_utf8(bytes) + .map_err(|err| CaptureError::InvalidPayload(format!("{label} is not valid UTF-8: {err}"))) +} + fn read_provider_jsonl_line(reader: &mut impl BufRead, buffer: &mut Vec) -> Result { buffer.clear(); let mut total = 0usize; @@ -6954,9 +6971,21 @@ fn provider_path_has_component(path: &Path, expected: &str) -> bool { fn openclaw_session_indexes(root: &Path) -> BTreeMap { let mut indexes = BTreeMap::new(); let mut paths = Vec::new(); - collect_named_paths(root, "sessions.json", &mut paths); + let mut visited = 0usize; + collect_named_paths( + root, + "sessions.json", + &mut paths, + &mut visited, + MAX_OPENCLAW_SESSION_INDEX_PATHS, + MAX_OPENCLAW_SESSION_INDEX_VISITED_PATHS, + ); for path in paths { - let Ok(text) = fs::read_to_string(&path) else { + let Ok(text) = read_text_file_limited( + &path, + MAX_OPENCLAW_SESSION_INDEX_BYTES, + "OpenClaw sessions.json", + ) else { continue; }; let Ok(value) = serde_json::from_str::(&text) else { @@ -7015,7 +7044,18 @@ fn openclaw_session_index_entries(value: Value) -> Vec<(String, Value)> { } } -fn collect_named_paths(root: &Path, name: &str, paths: &mut Vec) { +fn collect_named_paths( + root: &Path, + name: &str, + paths: &mut Vec, + visited: &mut usize, + max_paths: usize, + max_visited: usize, +) { + if paths.len() >= max_paths || *visited >= max_visited { + return; + } + *visited += 1; let Ok(metadata) = fs::symlink_metadata(root) else { return; }; @@ -7035,7 +7075,10 @@ fn collect_named_paths(root: &Path, name: &str, paths: &mut Vec) { return; }; for entry in entries.flatten() { - collect_named_paths(&entry.path(), name, paths); + if paths.len() >= max_paths || *visited >= max_visited { + break; + } + collect_named_paths(&entry.path(), name, paths, visited, max_paths, max_visited); } } @@ -14529,6 +14572,62 @@ mod tests { .contains("OpenCode SQLite message table missing required column(s): data")); } + #[test] + fn openclaw_import_ignores_oversized_session_index_sidecar() { + let temp = tempdir(); + let root = temp.path().join("openclaw"); + let sessions = root.join("agents/personal-agent/sessions"); + fs::create_dir_all(&sessions).unwrap(); + fs::write( + sessions.join("sessions.json"), + vec![b'x'; MAX_OPENCLAW_SESSION_INDEX_BYTES + 1], + ) + .unwrap(); + fs::write( + sessions.join("openclaw-oversized-index.jsonl"), + format!( + "{}\n{}\n", + json!({ + "type": "session", + "id": "openclaw-oversized-index", + "timestamp": "2026-06-24T12:00:00Z", + "cwd": "/workspace" + }), + json!({ + "type": "message", + "id": "openclaw-oversized-index-user", + "timestamp": "2026-06-24T12:00:01Z", + "message": {"role": "user", "content": "oversized sidecar should not block import"} + }) + ), + ) + .unwrap(); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_openclaw_history( + &root, + &mut store, + OpenClawImportOptions { + allow_partial_failures: true, + ..OpenClawImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 0); + assert_eq!(summary.imported_sessions, 1); + assert_eq!(summary.imported_events, 1); + let session_id = provider_session_uuid( + CaptureProvider::OpenClaw, + "personal-agent/openclaw-oversized-index", + ); + let session = store.get_session(session_id).unwrap(); + assert_eq!( + session.external_session_id.as_deref(), + Some("personal-agent/openclaw-oversized-index") + ); + } + #[test] fn native_shelley_imports_sessions_messages_metadata_and_citations() { let temp = tempdir(); From b7861185c71f2264f71be97cd6e4cc7a92666a94 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:56:06 -0500 Subject: [PATCH 69/72] Reject invalid OpenCode timestamps --- crates/ctx-history-capture/src/lib.rs | 96 +++++++++++++++++++++------ 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 7f806e60b..10ee46f8e 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -6875,6 +6875,14 @@ fn provider_timestamp_millis(value: Option, fallback: DateTime) -> Dat .unwrap_or(fallback) } +fn provider_required_timestamp_millis(value: i64, field: &'static str) -> Result> { + DateTime::::from_timestamp_millis(value).ok_or_else(|| { + CaptureError::InvalidPayload(format!( + "{field} is outside representable timestamp range: {value}" + )) + }) +} + fn provider_timestamp_value(value: Option<&Value>, fallback: DateTime) -> DateTime { match value { Some(Value::String(raw)) => parse_rfc3339_utc(raw) @@ -9232,15 +9240,16 @@ fn normalize_opencode_sqlite( let sessions = opencode_sessions(&conn)?; let messages = opencode_session_messages(&conn)?; let mut result = ProviderNormalizationResult::default(); - let session_started = sessions - .iter() - .map(|session| { - ( - session.id.clone(), - timestamp_millis_utc(session.time_created, context.imported_at), - ) - }) - .collect::>(); + let mut session_started = BTreeMap::new(); + for session in &sessions { + session_started.insert( + session.id.clone(), + provider_required_timestamp_millis( + session.time_created, + "OpenCode session time_created", + )?, + ); + } let sessions_by_id = sessions .into_iter() .map(|session| (session.id.clone(), session)) @@ -9279,9 +9288,23 @@ fn normalize_opencode_sqlite( continue; } }; - let occurred_at = opencode_event_time(&data) - .or_else(|| Some(timestamp_millis_utc(row.time_created, context.imported_at))) - .unwrap_or(context.imported_at); + let occurred_at = match opencode_event_time(&data) { + Ok(Some(time)) => time, + Ok(None) => match provider_required_timestamp_millis( + row.time_created, + "OpenCode session_message time_created", + ) { + Ok(time) => time, + Err(err) => { + push_provider_import_failure(&mut result.summary, line, err.to_string()); + continue; + } + }, + Err(err) => { + push_provider_import_failure(&mut result.summary, line, err.to_string()); + continue; + } + }; let started_at = session_started .get(&session.id) .copied() @@ -9788,14 +9811,16 @@ fn opencode_content_has_tool(data: &Value) -> bool { .unwrap_or(false) } -fn opencode_event_time(data: &Value) -> Option> { - data.pointer("/time/created") - .and_then(Value::as_i64) - .and_then(DateTime::::from_timestamp_millis) -} - -fn timestamp_millis_utc(millis: i64, fallback: DateTime) -> DateTime { - DateTime::::from_timestamp_millis(millis).unwrap_or(fallback) +fn opencode_event_time(data: &Value) -> Result>> { + let Some(value) = data.pointer("/time/created") else { + return Ok(None); + }; + let millis = value.as_i64().ok_or_else(|| { + CaptureError::InvalidPayload( + "OpenCode event time.created must be integer millis".to_owned(), + ) + })?; + provider_required_timestamp_millis(millis, "OpenCode event time.created").map(Some) } fn parse_json_object_string(value: Option<&str>) -> Value { @@ -14452,6 +14477,37 @@ mod tests { })); } + #[test] + fn native_opencode_rejects_out_of_range_message_timestamp() { + let temp = tempdir(); + let fixture = write_opencode_smoke_db(&temp, false); + let conn = Connection::open(&fixture).unwrap(); + let data_without_payload_time = json!({"text": "bad timestamp fallback"}).to_string(); + conn.execute( + "update session_message set time_created = ?1, data = ?2 where id = 'msg-user'", + rusqlite::params![i64::MAX, data_without_payload_time], + ) + .unwrap(); + drop(conn); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_opencode_sqlite( + &fixture, + &mut store, + OpenCodeSqliteImportOptions { + allow_partial_failures: true, + ..OpenCodeSqliteImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 1); + assert!(summary.failures[0] + .error + .contains("OpenCode session_message time_created")); + assert_eq!(summary.imported_events, 2); + } + #[test] fn native_opencode_rejects_oversized_sqlite_text_value() { let temp = tempdir(); From d9cd632891afd66ea1b21e8500cd8292bb5bbb10 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:07:39 -0500 Subject: [PATCH 70/72] Reject invalid Hermes timestamps --- crates/ctx-history-capture/src/lib.rs | 132 +++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 12 deletions(-) diff --git a/crates/ctx-history-capture/src/lib.rs b/crates/ctx-history-capture/src/lib.rs index 10ee46f8e..ca1c0a117 100644 --- a/crates/ctx-history-capture/src/lib.rs +++ b/crates/ctx-history-capture/src/lib.rs @@ -6854,19 +6854,33 @@ fn provider_line_from_index(index: u64) -> usize { index.min(usize::MAX as u64) as usize } -fn provider_timestamp_seconds(value: Option, fallback: DateTime) -> DateTime { - let Some(value) = value else { - return fallback; - }; +fn provider_timestamp_seconds_to_datetime(value: f64) -> Option> { if !value.is_finite() { - return fallback; + return None; } let millis = if value.abs() > 1_000_000_000_000.0 { - value.round() as i64 + value.round() } else { - (value * 1000.0).round() as i64 + (value * 1000.0).round() }; - DateTime::::from_timestamp_millis(millis).unwrap_or(fallback) + if millis < i64::MIN as f64 || millis > i64::MAX as f64 { + return None; + } + DateTime::::from_timestamp_millis(millis as i64) +} + +fn provider_timestamp_seconds(value: Option, fallback: DateTime) -> DateTime { + value + .and_then(provider_timestamp_seconds_to_datetime) + .unwrap_or(fallback) +} + +fn provider_required_timestamp_seconds(value: f64, field: &'static str) -> Result> { + provider_timestamp_seconds_to_datetime(value).ok_or_else(|| { + CaptureError::InvalidPayload(format!( + "{field} is outside representable timestamp range: {value}" + )) + }) } fn provider_timestamp_millis(value: Option, fallback: DateTime) -> DateTime { @@ -7457,11 +7471,37 @@ fn normalize_hermes_sqlite( continue; }; let provider_session_id = session.id.clone(); - let occurred_at = provider_timestamp_seconds(Some(row.timestamp), context.imported_at); - let started_at = provider_timestamp_seconds(Some(session.started_at), occurred_at); - let ended_at = session + let occurred_at = + match provider_required_timestamp_seconds(row.timestamp, "Hermes message timestamp") { + Ok(timestamp) => timestamp, + Err(err) => { + push_provider_import_failure(&mut result.summary, line, err.to_string()); + continue; + } + }; + let started_at = match provider_required_timestamp_seconds( + session.started_at, + "Hermes session started_at", + ) { + Ok(timestamp) => timestamp, + Err(err) => { + push_provider_import_failure(&mut result.summary, line, err.to_string()); + continue; + } + }; + let ended_at = match session .ended_at - .map(|timestamp| provider_timestamp_seconds(Some(timestamp), context.imported_at)); + .map(|timestamp| { + provider_required_timestamp_seconds(timestamp, "Hermes session ended_at") + }) + .transpose() + { + Ok(timestamp) => timestamp, + Err(err) => { + push_provider_import_failure(&mut result.summary, line, err.to_string()); + continue; + } + }; let content = hermes_decode_content(row.content.as_deref()); let text = provider_value_text(&content).unwrap_or_else(|| { row.tool_name @@ -14387,6 +14427,36 @@ mod tests { ); } + #[test] + fn native_hermes_rejects_out_of_range_message_timestamp() { + let temp = tempdir(); + let fixture = write_hermes_smoke_db(&temp); + let conn = Connection::open(&fixture).unwrap(); + conn.execute( + "update messages set timestamp = ?1 where content = 'bad timestamp'", + [1.0e300_f64], + ) + .unwrap(); + drop(conn); + let mut store = Store::open(temp.path().join("work.sqlite")).unwrap(); + + let summary = import_hermes_sqlite( + &fixture, + &mut store, + HermesSqliteImportOptions { + allow_partial_failures: true, + ..HermesSqliteImportOptions::default() + }, + ) + .unwrap(); + + assert_eq!(summary.failed, 1); + assert!(summary.failures[0] + .error + .contains("Hermes message timestamp")); + assert_eq!(summary.imported_events, 1); + } + #[cfg(unix)] #[test] fn native_opencode_normalizer_rejects_symlinked_sqlite() { @@ -15128,6 +15198,44 @@ mod tests { path } + fn write_hermes_smoke_db(temp: &TempDir) -> PathBuf { + let path = temp.path().join("hermes-state.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "create table sessions ( + id text primary key, + source text not null, + started_at real not null + ); + create table messages ( + id integer primary key autoincrement, + session_id text not null, + role text not null, + content text, + timestamp real not null, + active integer not null default 1, + compacted integer not null default 0 + );", + ) + .unwrap(); + conn.execute( + "insert into sessions values (?1, 'acp', 1782259200.0)", + ["hermes-root"], + ) + .unwrap(); + conn.execute( + "insert into messages (session_id, role, content, timestamp) values (?1, 'user', 'bad timestamp', 1782259201.0)", + ["hermes-root"], + ) + .unwrap(); + conn.execute( + "insert into messages (session_id, role, content, timestamp) values (?1, 'assistant', 'good timestamp', 1782259202.0)", + ["hermes-root"], + ) + .unwrap(); + path + } + fn write_opencode_session_message_without_seq_db(temp: &TempDir) -> PathBuf { let path = temp.path().join("opencode-no-seq.db"); let conn = Connection::open(&path).unwrap(); From 4bcf8d5f15463722fc89b50383975c21dbed7395 Mon Sep 17 00:00:00 2001 From: luca-ctx <216224554+luca-ctx@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:11:08 -0500 Subject: [PATCH 71/72] Avoid busy copied test binaries --- crates/ctx-cli/tests/cli.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/ctx-cli/tests/cli.rs b/crates/ctx-cli/tests/cli.rs index 3f2d5584d..9f3cf0c55 100644 --- a/crates/ctx-cli/tests/cli.rs +++ b/crates/ctx-cli/tests/cli.rs @@ -43,7 +43,9 @@ fn copied_ctx_binary(temp: &TempDir) -> PathBuf { } else { "ctx-test-copy" }); - fs::copy(&source, &target).unwrap(); + if fs::hard_link(&source, &target).is_err() { + fs::copy(&source, &target).unwrap(); + } #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; From eccfc58f9b42b0df9e9bf8c08103dffe95b1d7e4 Mon Sep 17 00:00:00 2001 From: fy2ne Date: Sat, 4 Jul 2026 05:55:13 -0700 Subject: [PATCH 72/72] docs: clarify Windows C++ compiler requirements for local builds --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index bcd3cc684..206afb2fa 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,26 @@ ctx upgrade check Source builds and package-manager installs remain unmanaged and do not self-upgrade. +### Building from source + +If you prefer to compile `ctx` locally, please note that the project relies on a bundled SQLite database. This means a C/C++ compiler is strictly required during the build process to compile the underlying C code. + +**Windows Prerequisites:** +Windows does not come with a C compiler by default. If you run `cargo build` without one, you will encounter `link.exe not found` or `gcc.exe not found` errors. + +To fix this, you must install the [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/): +1. Download and run the Visual Studio Build Tools installer. +2. Select the **"Desktop development with C++"** workload. +3. Complete the installation and restart your terminal. + +```mermaid +graph LR + A[cargo build] --> B{C++ Compiler installed?} + B -- Yes --> C[Compiles SQLite C code] + B -- No --> D[Fails: link.exe / gcc.exe missing] + C --> E[Builds ctx binary successfully] +``` + For the full pipeline, see [How ctx works](https://ctx.rs/concepts/how-it-works). For a quick first run, see [Quickstart](https://ctx.rs/first-search). ## Supported agent histories