diff --git a/apps/presentation/dashboard/src/data/state-recovery.ts b/apps/presentation/dashboard/src/data/state-recovery.ts new file mode 100644 index 0000000000..f19f05d44b --- /dev/null +++ b/apps/presentation/dashboard/src/data/state-recovery.ts @@ -0,0 +1,39 @@ +import {requestJson} from "./chat"; +import {decodeStateRecoveryAudit, type StateRecoveryAudit} from "../../../../../loopx/control_plane/state_recovery_contract"; + +export type StateRecoveryResult = StateRecoveryAudit & {checkpoint_ref: string; historical_readback?: boolean}; +export type RecoveryHistoryItem = { + checkpoint_ref: string; available: boolean; observed_at?: string; + summary?: StateRecoveryAudit["summary"]; +}; +const endpoint = "/api/chat/state-recovery"; + +export async function stateRecoveryProfile() { + const result = await requestJson<{supported: boolean; activation_available: false}>(`${endpoint}/profile`); + if (typeof result.supported !== "boolean" || result.activation_available !== false) throw new Error("Recovery profile is invalid"); + return result; +} + +export async function uploadRecoveryInput(file: File, kind: "archive" | "manifest") { + return requestJson<{input_ref: string; checkpoint_ref?: string}>(`${endpoint}/${kind}`, { + method: "POST", headers: {"Content-Type": "application/octet-stream"}, body: file, + }); +} +function result(value: unknown): StateRecoveryResult { + const audit = decodeStateRecoveryAudit(value); + if (typeof audit.checkpoint_ref !== "string" || !/^[a-f0-9]{32}$/.test(audit.checkpoint_ref)) { + throw new Error("Recovery checkpoint reference is invalid"); + } + return audit as StateRecoveryResult; +} +export async function inspectStateBackup(archiveRef: string, manifestRef: string) { + return result(await requestJson(`${endpoint}/verify`, {method: "POST", + body: JSON.stringify({archive_ref: archiveRef, manifest_ref: manifestRef, execute: true})})); +} +export async function readStateRecovery(checkpointRef: string) { + return result(await requestJson(`${endpoint}/read`, {method: "POST", + body: JSON.stringify({checkpoint_ref: checkpointRef})})); +} +export function listStateRecoveries() { + return requestJson<{items: RecoveryHistoryItem[]; has_more: boolean}>(endpoint); +} diff --git a/apps/presentation/dashboard/src/features/personal-workspace/configuration-backup-settings.tsx b/apps/presentation/dashboard/src/features/personal-workspace/configuration-backup-settings.tsx index 071078dce2..d5847f62ff 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/configuration-backup-settings.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/configuration-backup-settings.tsx @@ -1,6 +1,7 @@ import {useEffect, useState} from "react"; import {exportConfigurationBackup, restoreConfigurationCheckpoint, type ConfigurationBackupResult} from "../../data/chat"; import {useWorkspaceI18n} from "./i18n"; +import {StateRecoverySettings} from "./state-recovery-settings"; export function ConfigurationBackupSettings({goalId}: {goalId: string | null}) { const {locale} = useWorkspaceI18n(); @@ -9,7 +10,8 @@ export function ConfigurationBackupSettings({goalId}: {goalId: string | null}) { const [result, setResult] = useState(null); const [error, setError] = useState(""); const [busy, setBusy] = useState(false); - useEffect(() => {setBackup(null); setResult(null); setError("");}, [goalId]); + const [kind, setKind] = useState<"configuration" | "state">("configuration"); + useEffect(() => {setBackup(null); setResult(null); setError(""); setKind("configuration");}, [goalId]); async function perform(action: () => Promise) { setBusy(true); setError(""); try {await action();} catch (failure) {setError(String(failure));} @@ -17,6 +19,12 @@ export function ConfigurationBackupSettings({goalId}: {goalId: string | null}) { } return
{zh ? "配置备份与恢复" : "Configuration backup and recovery"} + {goalId === null ?
+ {zh ? "备份类型" : "Backup type"} + + +
: null} + {kind === "configuration" || goalId !== null ? <>

{zh ? "备份包含私有路径、Goal 设置及设备默认值,需自行保管和审查。凭据文件不包含在此组件中。恢复只建立隔离检查点;启用仍在原配置页面办理。" : "Keep and review this private backup: it includes Goal settings, paths and device defaults. Credential files are outside this component. Recovery creates an isolated checkpoint; activate settings through their existing editors."}

; } diff --git a/apps/presentation/dashboard/src/features/personal-workspace/state-recovery-settings.css b/apps/presentation/dashboard/src/features/personal-workspace/state-recovery-settings.css new file mode 100644 index 0000000000..e45cca2479 --- /dev/null +++ b/apps/presentation/dashboard/src/features/personal-workspace/state-recovery-settings.css @@ -0,0 +1,33 @@ +.personal-recovery-kind { + display: flex; + flex-wrap: wrap; + gap: 0.75rem; + margin: 0.75rem 0; + padding: 0; + border: 0; +} +.personal-recovery-kind legend { margin-bottom: 0.5rem; } +.personal-recovery-kind label { display: inline-flex; align-items: center; gap: 0.375rem; min-height: 2.75rem; } +.personal-recovery { display: grid; gap: 0.75rem; min-width: 0; } +.personal-detail-card .personal-recovery p, +.personal-detail-card .personal-recovery label, +.personal-detail-card .personal-recovery button { font-size: var(--text-body); line-height: var(--leading-body); } +.personal-detail-card .personal-recovery p { margin: 0.375rem 0; color: var(--pw-text); } +.personal-recovery-files { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1rem; } +.personal-recovery-files label { display: grid; gap: 0.5rem; min-width: 0; } +.personal-recovery-files input { width: 100%; max-width: 100%; min-width: 0; font: inherit; } +.personal-recovery-actions { display: flex; flex-wrap: wrap; gap: 0.5rem; } +.personal-recovery-actions > button { width: auto; min-height: 2.75rem; padding: 0.5rem 0.75rem; } +.personal-recovery-result { display: grid; gap: 0.5rem; } +.personal-detail-card .personal-recovery-counts { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 1rem; margin: 0.5rem 0; max-width: 28rem; } +.personal-detail-card .personal-recovery-counts > div { display: block; } +.personal-detail-card .personal-recovery-counts dt { color: var(--pw-muted); font-size: var(--text-caption); } +.personal-detail-card .personal-recovery-counts dd { margin: 0.25rem 0 0; font-size: var(--text-heading); font-weight: 600; } +.personal-recovery-components { list-style: none; margin: 0; padding: 0; } +.personal-recovery-components > li { padding: 0.75rem 0; border-top: 1px solid var(--pw-line); } +.personal-recovery code { overflow-wrap: anywhere; font-size: var(--text-caption); } +@media (max-width: 640px) { + .personal-recovery-files { grid-template-columns: minmax(0, 1fr); } + .personal-recovery-actions { display: grid; } + .personal-recovery-actions > button { width: 100%; } +} diff --git a/apps/presentation/dashboard/src/features/personal-workspace/state-recovery-settings.tsx b/apps/presentation/dashboard/src/features/personal-workspace/state-recovery-settings.tsx new file mode 100644 index 0000000000..6e3b0c6412 --- /dev/null +++ b/apps/presentation/dashboard/src/features/personal-workspace/state-recovery-settings.tsx @@ -0,0 +1,127 @@ +import {useEffect, useRef, useState} from "react"; +import {inspectStateBackup, listStateRecoveries, readStateRecovery, stateRecoveryProfile, uploadRecoveryInput, + type RecoveryHistoryItem, type StateRecoveryResult} from "../../data/state-recovery"; +import {useWorkspaceI18n} from "./i18n"; +import {ChatApiError} from "../../data/chat"; +import "./state-recovery-settings.css"; + +const ownerNames: Record = { + configuration: ["配置", "Configuration"], coordination_authority: ["Goal 工作与历史", "Goal work and history"], + sqlite_storage: ["数据库", "Database"], recovery: ["其他保存的状态", "Other saved state"], +}; +const nextActions: Record = { + retain_isolated_configuration: ["保留隔离配置,通过原设置页另行启用。", "Keep the isolated configuration; use its existing editor for any later activation."], + review_goal_identity_before_adoption: ["继续执行前,先由 Goal owner 核对身份与未完成工作。", "Before resuming, have the Goal owner review identity and unfinished work."], + inspect_captured_authority_with_owner: ["请存储 owner 检查保存的工作记录。", "Have the storage owner inspect the saved work records."], + use_provider_recovery_owner: ["通过该存储服务的恢复流程检查。", "Use this storage service's recovery procedure."], + retain_historical_database: ["保留数据库副本,暂不用于执行。", "Keep the database copy as historical data."], + inspect_database_with_owner: ["请数据库 owner 检查完整性和格式。", "Have the database owner check integrity and format."], + inspect_configuration_with_owner: ["请配置 owner 检查备份版本和内容。", "Have the configuration owner check the saved format and contents."], + classify_remaining_owner_state: ["启用前,需分别确认其他状态的归属、会话、租约和未决操作。", "Identify the remaining state owners, sessions, leases and pending operations before any activation."], +}; +const errors: Record = { + archive_digest_mismatch: ["归档与清单不匹配,请选择同一次备份的两个文件。", "The archive and manifest do not match. Choose the pair from the same backup."], + manifest_disagreement: ["内外清单不一致,请重新选择原始备份。", "The embedded and external manifests disagree. Choose the original backup pair."], + recovery_destination_exists: ["这份检查已有隔离副本,请读取历史结果。", "An isolated copy already exists for this inspection. Read its saved result."], + recovery_capacity_exceeded: ["备份超过当前检查容量,请使用较小的备份;未建立新副本。", "This backup exceeds the inspection limits. Use a smaller backup; no new copy was published."], + recovery_input_unavailable: ["上传文件已不可用,请重新选择归档与清单。", "The uploaded files are unavailable. Select the archive and manifest again."], + recovery_owner_timeout: ["离线检查超时,未发布副本;可使用较小的备份重试。", "Offline verification timed out without publishing a copy. Retry with a smaller backup."], + authority_history_capacity: ["单个 Goal 的历史超过 20,000 次提交,未发布副本。请保留完整备份,通过存储 owner 的恢复流程检查。", "One Goal exceeds the 20,000-commit inspection limit; no copy was published. Keep the complete backup and use the storage owner's recovery procedure."], + member_witness_mismatch: ["保存的文件与清单记录不符,已停止检查。", "Saved files differ from their manifest witnesses. Inspection stopped."], +}; + +export function StateRecoverySettings() { + const {locale} = useWorkspaceI18n(), zh = locale === "zh-CN", index = zh ? 0 : 1; + const [archive, setArchive] = useState(null); + const [manifest, setManifest] = useState(null); + const [audit, setAudit] = useState(null); + const [history, setHistory] = useState(null); + const [more, setMore] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [pending, setPending] = useState(null); + const [supported, setSupported] = useState(null); + const generation = useRef(0); + useEffect(() => { + let active = true; + void stateRecoveryProfile().then(profile => {if (active) setSupported(profile.supported);}) + .catch(() => {if (active) setSupported(false);}); + return () => {active = false;}; + }, []); + async function perform(operation: () => Promise) { + setBusy(true); setError(""); + try { await operation(); } catch (failure) { + const code = failure instanceof ChatApiError ? String(failure.payload.error_code ?? "") : ""; + if (code === "authority_history_capacity") setPending(null); + setError(errors[code]?.[index] ?? (zh ? "检查未完成,请核对文件或稍后重试。" : "Inspection did not finish. Check the files or retry later.")); + } + finally { setBusy(false); } + } + function choose(file: File | null, kind: "archive" | "manifest") { + generation.current++; setAudit(null); setError(""); setPending(null); + if (kind === "archive") setArchive(file); else setManifest(file); + } + return
+ {supported === false ?

{zh ? "当前服务或平台不支持完整归档检查。配置快照仍可使用。" : "This service or platform does not support full archive inspection. Configuration snapshots remain available."}

: null} +

{zh ? "选择完整归档和配套清单,建立私有隔离副本并查看可读内容。此处不会恢复 Goal 运行;旧备份缺少一致性信息时会明确保留未知。" + : "Choose an archive and its matching manifest to inspect a private, isolated copy. This does not resume a Goal; missing consistency information stays unknown."}

+
+
+
+ + +
+ {error ?

{error}

: null} + {error && pending ? : null} + {audit ?
+ {audit.historical_readback + ? (zh ? "已读取历史检查记录;尚未重新验证副本。" : "Historical inspection loaded; the copy has not been reverified.") + : (zh ? "隔离副本已核对;Goal 尚未启用。" : "Isolated copy inspected; Goal execution remains inactive.")} +

{zh ? "检查时刻" : "Observed at"}:

+
+
{zh ? "可读组件" : "Readable components"}
{audit.summary.readable}
+
{zh ? "仅历史组件" : "Historical components"}
{audit.summary.historical_only}
+
{zh ? "未知组件" : "Unknown components"}
{audit.summary.unknown}
+
+

{zh ? "旧格式没有完整的捕获版本与一致性声明,以上结果只证明记录时的读取,不证明可以安全继续执行。" + : "This legacy archive has no complete capture-revision or consistency declaration. These observations do not authorize resuming work."}

+
    {audit.components.map(row =>
  • + {ownerNames[row.owner_kind]?.[index] ?? (zh ? "待核对组件" : "Component requiring review")} + {typeof row.facts.goal_id === "string" ? · {row.facts.goal_id} : null} + {" · "}{row.state === "readable" ? (zh ? "可读" : "Readable") : row.state === "historical_only" ? (zh ? "仅历史" : "Historical only") : (zh ? "未知" : "Unknown")} + {typeof row.facts.todo_count === "number" ?

    {zh ? "保存的任务" : "Recorded tasks"}: {row.facts.todo_count} · {zh ? "历史提交" : "Historical commits"}: {String(row.facts.history_commits)}

    : null} +

    {nextActions[row.next_action]?.[index] ?? (zh ? "请对应 owner 核对。" : "Ask the owning module to review this component.")}

    +
  • )}
+
{zh ? "核对信息" : "Verification details"}{audit.recovery_id} + {audit.measurements ?

{zh ? "归档字节" : "Archive bytes"}: {audit.measurements.archive_bytes.toLocaleString(locale)} · {zh ? "普通文件字节" : "Regular file bytes"}: {audit.measurements.regular_file_bytes.toLocaleString(locale)}
+ {zh ? "归档检查与复制" : "Archive checks and copy"}: {audit.measurements.archive_processing_ms.toLocaleString(locale)} ms · {zh ? "记录读取" : "Owner reads"}: {audit.measurements.owner_read_ms.toLocaleString(locale)} ms

+ :

{zh ? "此历史报告未记录数据量和阶段耗时。" : "This historical report has no recorded size or phase durations."}

} +
    {audit.components.map(row =>
  • {row.component_id}: {row.reason_code}
  • )}
+
+
: null} + {history ?
+ {history.length === 0 ?

{zh ? "尚无检查结果。" : "No saved inspection yet."}

: null} + {history.map(row =>

)} + {more ?

{zh ? "显示最近 10 份检查。" : "Showing the 10 latest inspections."}

: null} +
: null} +
; +} diff --git a/docs/architecture/rfcs/complete-state-recovery-v0.md b/docs/architecture/rfcs/complete-state-recovery-v0.md index 2321615007..932fc0c984 100644 --- a/docs/architecture/rfcs/complete-state-recovery-v0.md +++ b/docs/architecture/rfcs/complete-state-recovery-v0.md @@ -2,7 +2,7 @@ - **RFC status:** Accepted - **Supersedes / closes:** none -- **Delivery maturity:** Proposal; backup creation and configuration-only isolated recovery exist, complete state verification and reactivation do not +- **Delivery maturity:** Partial: bounded M1 full-archive verification and inert File/SQLite inspection implemented; controlled reactivation remains open - **Authors / owners:** Control-plane state, reliability, and affected domain maintainers - **Created:** 2026-10-08 - **Last normative revision:** 2026-10-08 @@ -24,6 +24,24 @@ transfer, or a default change. ## 1. Decision summary +### Current implementation checkpoint + +The bounded M1 path now has additive `backup-state verify`, loopback HTTP upload +and immutable report readback, and a packaged Settings/Capability Center journey. +It verifies real producer archives and reads captured configuration plus selected +File/SQLite Todo/history through existing owners after original sources become +unavailable. Every v0 audit remains held for incomplete capture/owner metadata; +unknown state is not relabeled recoverable, and execution authority is always +false. Limits, rollback and reproduction are in the [backup guide](../../reference/configuration-backup.md#inspect-a-complete-backup-without-activating-it). +Reports retain measured archive/regular-file bytes and archive-processing and +owner-read durations; historical readback does not retime them. The backup guide +records individual native capacity cases, including exact member/history +boundaries and 128 File/SQLite Goals. History overflow refuses publication. +M1 operational qualification remains partial: the Section 12 maintainer freeze, +compound worst cases and broader platform/provider profiles remain open. +This checkpoint does not qualify M2/M3, live adoption, remote providers or the G1 +team journey. No original identity or receipt is rewritten. + ### First supported user outcome The first bounded user promise is recovery of one local Goal after its source @@ -196,11 +214,11 @@ Other files and directories are traversed while the source may remain live. The command therefore has component-local SQLite snapshot semantics and best-effort file capture, not a cross-owner point-in-time guarantee. -`loopx backup-state` supports plan and create. It does not verify an existing -archive, safely materialize one, classify state owners, reconcile references, -or reactivate a runtime. Current tests extract with the Python tar API, validate -SQLite integrity and content, and verify publication/permission behavior. They -do not establish a restore contract. +At the design baseline, `loopx backup-state` supported plan/create only, and +tests manually extracted archives. The current bounded M1 route in Section 1 +adds verified inert extraction and existing-owner readback through CLI and the +packaged UI. It does not adopt identity, reconcile pending effects or reactivate +a runtime; those later owner boundaries remain open. Configuration backup separately supports export, verify, and isolated restore. Its checkpoint is inert: it creates no live registry, provider selection, diff --git a/docs/architecture/rfcs/complete-state-recovery-v0.zh-CN.md b/docs/architecture/rfcs/complete-state-recovery-v0.zh-CN.md index 73c32f81ba..3668992695 100644 --- a/docs/architecture/rfcs/complete-state-recovery-v0.zh-CN.md +++ b/docs/architecture/rfcs/complete-state-recovery-v0.zh-CN.md @@ -2,7 +2,7 @@ - **RFC 状态:** 已接受 - **替代 / 关闭:** 无 -- **交付成熟度:** 提案;备份创建和仅配置的隔离恢复已存在,完整状态验证与重新激活尚不存在 +- **交付成熟度:** 部分实现:M1 完整归档验证与 File/SQLite 隔离检查已实现;受控重新启用仍开放 - **作者 / 负责人:** 控制面状态、可靠性及受影响领域的维护者 - **创建:** 2026-10-08 - **最近规范修订:** 2026-10-08 @@ -20,6 +20,19 @@ RFC 只授权有界实现切片,不授权真实恢复、provider 晋升、凭 ## 1. 决策摘要 +### 当前实现检查点 + +M1 的有界路径已包含增量 `backup-state verify`、loopback HTTP 上传及不可变报告读回, +以及打包设置/能力中心交互。真实生产归档在原来源不可达后,通过既有配置与选定的 +File/SQLite Todo/history owner 检查。v0 缺少完整捕获/owner 元数据,因此所有审计仍为 +held,未知状态不改称可恢复,执行权限始终为 false。限制与回退见[备份指南](../../reference/configuration-backup.md#inspect-a-complete-backup-without-activating-it)。 +报告保留实际归档/普通文件字节数及归档处理、owner 读取耗时;历史读回不重新计时。 +备份指南记录了独立的真实容量案例,包括精确成员/历史边界及 128 个 File/SQLite Goal; +历史超限在发布前拒绝。M1 的运行资格仍不完整:第 12 节维护者冻结、多个上限组合的 +最坏情形及更广的平台/provider 资格继续开放。 +本检查点不证明 M2/M3、live adoption、远端 provider 或 G1 团队验收,不重写原身份或回执。 + + ### 首个支持的用户结果 首个有界用户承诺是:一个本地 Goal 的来源工作区不可用或不再可信后,用户可以恢复 @@ -164,10 +177,10 @@ digest 和大小,并排除 WAL/SHM sidecar。其他文件和目录可能在来 因此,该命令只具备组件本地 SQLite snapshot 语义和 best-effort 文件捕获,不具备 跨 owner point-in-time 保证。 -`loopx backup-state` 支持 plan 和 create。它不能验证已有归档、安全物化归档、 -分类状态 owner、reconcile reference 或重新激活 runtime。当前测试用 Python tar -API 提取,验证 SQLite integrity 与内容,并检查发布/权限行为;它们没有建立 -restore 契约。 +设计基线中的 `loopx backup-state` 只有 plan/create,测试通过 Python tar API 手工解包。 +第 1 节记录的当前有界 M1 路径已通过 CLI 和打包 UI 提供验证后的隔离解包及既有 owner +读回。它不采用身份、不处置 pending effect、不重新激活 runtime;这些后续 owner 边界 +仍保持开放。 配置备份另行支持 export、verify 和 isolated restore。其 checkpoint 是惰性的: 它不会创建 live registry、provider selection、writer fence、lease、Host session、 diff --git a/docs/architecture/rfcs/loopx-overall-roadmap-v0.md b/docs/architecture/rfcs/loopx-overall-roadmap-v0.md index 4868cb4d66..5ec24c2906 100644 --- a/docs/architecture/rfcs/loopx-overall-roadmap-v0.md +++ b/docs/architecture/rfcs/loopx-overall-roadmap-v0.md @@ -1057,9 +1057,16 @@ bytes. Readback runs with the original project/runtime unreachable; missing selected-provider storage fails without replacement or File fallback. The pre-import backup separately retains original Markdown before later projection. This qualifies independent data/history recovery, not identity adoption, pending -effect disposition or live Host reactivation. Continue those boundaries and the -packaged App recovery journey through the existing owners. Do not retire the last -normal Markdown writer or claim the released SQLite default from this checkpoint. +effect disposition or live Host reactivation. The bounded complete-recovery M1 +inspection now adds `backup-state verify` and a packaged Settings journey: a new +inert copy, existing File/SQLite Todo/history readers, and explicit readable, +historical and unknown components. Corrupt inputs, occupied targets and unsafe +archive paths refuse; a bounded offline worker isolates database verification. +Source and installed-wheel journeys retain old Goal bytes and never register or +activate the copy. Legacy capture/owner gaps keep every v0 audit held. Continue +identity, pending-effect disposition and authorized adoption through the existing +owners; do not retire the last normal Markdown writer or claim the released +SQLite default from this inspection checkpoint. The qualified shadow→canonical archive→isolated File/SQLite restore path retains committed dependency archives, terminal lease facts and later canonical writes. diff --git a/docs/architecture/rfcs/loopx-overall-roadmap-v0.zh-CN.md b/docs/architecture/rfcs/loopx-overall-roadmap-v0.zh-CN.md index b37ba09788..fe422d35b9 100644 --- a/docs/architecture/rfcs/loopx-overall-roadmap-v0.zh-CN.md +++ b/docs/architecture/rfcs/loopx-overall-roadmap-v0.zh-CN.md @@ -671,7 +671,11 @@ File/SQLite 完整 head、全部 committed journal、未引用的归档要求、 capture/rollback/已释放 lease 字节。回读前让原项目与 runtime 不可访问;选定 provider 存储缺失时明确失败,不新建替代库或回退 File。导入前备份另行保留投影前的原始 Markdown。这资格化独立数据与历史回读,不代表身份采用、pending effect 处置或 -Host 重新激活。继续既有 R5/T4 的这些边界和 packaged App 恢复旅程,通过后才退役 +Host 重新激活。有界的完整恢复 M1 检查现增加 `backup-state verify` 与打包设置旅程: +建立新隔离副本,复用 File/SQLite Todo/history reader,区分可读、仅历史与未知。 +损坏输入、已有目标及危险路径被拒绝,数据库验证在有时间/内存边界的独立进程内进行。 +源码与隔离 wheel 旅程保留原 Goal 字节,不注册或启用副本;v0 的 capture/owner 缺口使 +审计保持 held。继续既有 R5/T4 的身份、pending effect 与授权采用边界,通过后才退役 受影响旧 writer;D2 与发布默认值继续开放。详见 [全量状态恢复边界](../../reference/configuration-backup.md#full-state-recovery-after-cold-import)。 diff --git a/docs/reference/configuration-backup.md b/docs/reference/configuration-backup.md index 0ef382695d..bd546a55ca 100644 --- a/docs/reference/configuration-backup.md +++ b/docs/reference/configuration-backup.md @@ -188,6 +188,155 @@ cross-owner capture consistency, recover volatile work, or reactivate a Goal. 命令不会自动删除部分输出,也不会因文件存在就宣称成功。这不证明跨 owner 一致性 或恢复后的执行权限。 +## Inspect a complete backup without activating it + +In **Settings → Capability Center → device defaults → Configuration backup and +recovery**, choose **Full archive inspection**. Select the original `.tar.gz` +and matching external `.manifest.json`, then choose **Verify and create isolated +copy**. The existing configuration-snapshot option remains separate. Full-archive +inspection is a machine-level operation; it is not shown in single-Goal scope. +Unsupported services/platforms keep the operation unavailable. + +The local server streams the two uploads into private temporary inputs. It checks +the archive digest before decoding tar data, then checks embedded/external +manifest agreement, paths, links, member hashes and extraction capacity. Only a +new inert checkpoint can be published. It reads configuration and selected +File/SQLite authority through their existing owners, including the complete Todo +read model and committed history. Original registry paths are never followed. + +The result separates **readable**, **historical only** and **unknown** components, +with an owner and one next action. SQLite bytes alone do not prove a readable Goal +work model. Remaining v0 state has no complete owner inventory, so it stays +unknown; every v0 audit carries `legacy_manifest_incomplete`, stays `held`, and +reports `execution_authority_granted=false` and `activation_eligible=false`. +No Goal is registered, no provider is selected, and no Host, lease, timer or +pending external operation is resumed. This is the M1 inspection path in the +[complete recovery RFC](../architecture/rfcs/complete-state-recovery-v0.md), +not controlled reactivation. + +The following public synthetic scene uses a built wheel and the real File +backend. Before this change, the entry only restored configuration snapshots. +The new path reads the saved Goal work after its original source was removed; +it does not register that Goal. The phone view is explicitly a historical report. + +| Before: configuration only | Inert full-archive inspection | Phone historical readback | +| --- | --- | --- | +| ![Configuration-only entry](images/state-recovery/before-configuration.png) | ![Inert backup audit](images/state-recovery/inert-audit-desktop.png) | ![Historical audit on phone](images/state-recovery/historical-audit-phone.png) | + +Use **Previous inspections** after reloading to read the saved, digest-checked +report. A historical report describes its recorded observation time; reading it +again does not reverify copied data. A damaged new input clears the prior success +view and leaves existing checkpoints intact. If a response was lost, **Check for +an existing result** reads the original checkpoint; a repeated write to that +occupied target is refused. + +**Verification details** retain the archive byte size, copied regular-file +bytes (including the embedded manifest), and two measured phases. Archive +processing covers input checks, both scans and copying; owner reads cover the +isolated configuration/storage audit. Durations use monotonic clocks and exclude +HTTP upload, verifier process startup, the final source rehash/report publication, +and historical readback. They describe processing cost, not end-to-end recovery +time or a deadline guarantee. Earlier reports without measurements show that +absence instead of zero. Reading history preserves the original measurements. + +The additive CLI route preserves existing backup creation syntax: + +```bash +uv run --extra test loopx --format json backup-state verify \ + --archive BACKUP.tar.gz --manifest BACKUP.manifest.json \ + --destination NEW_PHYSICAL_DIRECTORY + +# Explicitly create and inspect that new inert copy. +uv run --extra test loopx --format json backup-state verify \ + --archive BACKUP.tar.gz --manifest BACKUP.manifest.json \ + --destination NEW_PHYSICAL_DIRECTORY --execute +``` + +Preview checks archive/manifest bytes and capacity; owner records are not yet +read and no zero-readability result is invented. Execution writes `data/` and +`recovery-audit.json` below the new destination. Source archive and manifest must +be private regular files; the destination must not exist and its ancestors must +be physical directories. Existing targets are never replaced. Browser callers +use opaque upload/checkpoint refs and cannot supply filesystem destinations. +Uploads expire after an hour and are cleaned by a later upload; completed audits +are retained. Removing these inert copies cannot undo or authorize live state. + +The first profile is POSIX with local File/SQLite state. Bounds are 128 MiB +compressed archive, 512 MiB expanded tar stream, 16 MiB external/embedded +manifest, 20,000 entries, expansion ratio 2,000, 128 KiB per extended tar header +and 16 MiB total extended metadata. Paths have at most 64 components, 1,024 +characters and 255 UTF-8 bytes per component; JSON nesting is bounded at 64. +Owner inspection permits 128 Goals, 128 SQLite files and 20,000 committed +transactions per Goal. Overflow refuses publication rather than truncating a +required inventory. Internal symlinks remain inert; escaping/cyclic links, +hard links, sparse or special files and normalized name collisions reject. + +Archive parsing never loads restored code. Database/domain inspection runs in a +disposable verifier with a 90-second deadline and a 512 MiB V8 heap ceiling. +SQLite gets a 64 MiB process-local [hard heap limit](https://www.sqlite.org/pragma.html#pragma_hard_heap_limit); +this is why that verifier is not dispatched in the shared Effect daemon. Views +and virtual tables cannot masquerade as authority tables. WAL-mode copies and +unsupported/service-owned providers remain unknown and need their existing +owner; PostgreSQL is never contacted. The normal 2 MiB RPC wire is unchanged; +only the two complete-manifest checks opt into the existing 64 MiB private-file +transport. These candidate bounds define this inspection profile. The RFC's +Section 12 maintainer freeze, compound worst-case qualification and broader +platform/provider qualification remain open. No production SLO is claimed. + +### Measured local capacity cases + +At source `884eae425c77ae6db9d8042bd98ee19663e7bdd4`, disposable native backups +were inspected through the real `backup-state verify --execute` CLI after hiding +their original project/runtime/registry. Measurements used macOS arm64, Python +3.11.7 and Node 22.22.3. Times below include the verifier command, not backup +creation; peak RSS is the macOS `time -l` command metric, not a simultaneous +process-tree total. These are individual synthetic observations under local +development load, not latency targets or proof that all maxima combine. + +| Native input / boundary | Result | Wall seconds | Peak RSS MiB | +| --- | --- | ---: | ---: | +| 511 MiB zero-filled regular file | Held audit; captured configuration readable | 4.941 | 97.8 | +| 512 MiB regular file plus archive metadata | Refused: expanded capacity | 1.931 | 88.2 | +| 127 MiB seeded incompressible file; 133,212,049 archive bytes | Held audit | 1.284 | 100.6 | +| Archive input 134,217,729 bytes | Refused before decoding: archive capacity | 0.521 | 81.5 | +| External manifest padded with JSON whitespace to 16 MiB | Held audit; bounds input bytes, not maximum structured metadata | 0.945 | 113.8 | +| External manifest 16 MiB + 1 byte | Refused: manifest capacity | 0.520 | 81.4 | +| Exactly 20,000 archive members | Complete inventory and held audit | 8.698 | 225.8 | +| Exactly 20,001 archive members | Refused: member capacity | 1.888 | 143.8 | +| 128 native File Goals, one retained task/commit each | All 128 Goal records readable | 0.843 | 103.5 | +| 128 native SQLite Goals/databases, one retained task/commit each | All 128 Goal records readable; database bytes historical-only | 1.041 | 107.1 | +| 129 File Goals | Refused: Goal inventory capacity | 0.738 | 96.8 | +| 129 SQLite databases | Refused: SQLite inventory capacity | 0.534 | 83.7 | + +All refusal cases left no destination or staging directory. The real SQLite +history regression additionally proves that **20,000 committed transactions +are fully read**, while **20,001 refuse publication**. An earlier implementation +incorrectly turned that overflow into an unknown component and published the +copy. The owner now checks the canonical cursor before scanning and propagates +the capacity error instead of treating it as unreadable source state: + +```bash +uv run --extra test python -m pytest tests/test_state_recovery.py \ + -k real_sqlite_history_limit -q +``` + +Keep an over-limit backup complete and use the owning storage recovery process; +do not trim historical transactions to force inspection through this profile. +The optional browser capacity case accepts a native over-limit archive pair via +`LOOPX_RECOVERY_HISTORY_OVERFLOW_ARCHIVE` and +`LOOPX_RECOVERY_HISTORY_OVERFLOW_MANIFEST`, alongside the existing +`configuration-backup` scenario. It must show refusal and retain the earlier +historical report. Installed-wheel checks on Python 3.12.13 and 3.13.12 also +passed normal source-loss inspection and early GNU/PAX sparse/metadata refusal; +they do not expand the capacity observations or qualify other platforms. + +中文:在设置的能力中心选择设备范围,打开既有备份入口并切到“完整归档检查”, +选择同一份备份的归档与清单,一次明确操作建立隔离副本。可读、仅历史和未知分别 +显示;旧格式始终保留一致性/owner 缺口,不能据此启动 Goal。重载后可读历史报告, +但历史读回不等于重新验证副本。CLI 不带 `--execute` 只做计划,不虚构 owner 读取结果。 +当前支持本地 POSIX File/SQLite 检查;Windows、服务端 PostgreSQL、身份接纳和实际 +重新启用仍未取得此路径的资格。备份内可能包含私有数据和凭据,副本及上传始终私有。 + ## Privacy, consistency and limits Backups are **private by default**. Full Goal rows can contain private paths, diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index bd036163ab..81698a4d10 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -185,6 +185,14 @@ Effect-program settlement step executed for one Turn. - typescript: [`SETTLEMENT_STEP_KINDS`](../../loopx/control_plane/effect_program.ts). - Values / 值: `validation`, `durable_writeback`, `quota_spend`, `terminal_closeout`. +## state_recovery_component_state + +Offline owner-local readability of a component in an inert recovery audit; never execution or adoption authority. + +- Tier / 层级: `cross_runtime`; status / 状态: `canonical`. +- typescript: [`RECOVERY_COMPONENT_STATES`](../../loopx/control_plane/state_recovery_contract.ts). +- Values / 值: `readable`, `historical_only`, `unknown`. + ## task_lease_lifecycle_operation Operation accepted by the shipped native lease lifecycle request decoder; acquisition has its own transaction. diff --git a/docs/reference/images/state-recovery/before-configuration.png b/docs/reference/images/state-recovery/before-configuration.png new file mode 100644 index 0000000000..3e958ae301 Binary files /dev/null and b/docs/reference/images/state-recovery/before-configuration.png differ diff --git a/docs/reference/images/state-recovery/historical-audit-phone.png b/docs/reference/images/state-recovery/historical-audit-phone.png new file mode 100644 index 0000000000..632b5186a6 Binary files /dev/null and b/docs/reference/images/state-recovery/historical-audit-phone.png differ diff --git a/docs/reference/images/state-recovery/inert-audit-desktop.png b/docs/reference/images/state-recovery/inert-audit-desktop.png new file mode 100644 index 0000000000..fcfe6b6ffa Binary files /dev/null and b/docs/reference/images/state-recovery/inert-audit-desktop.png differ diff --git a/examples/personal-workspace-browser/configuration-backup.mjs b/examples/personal-workspace-browser/configuration-backup.mjs index 0c9bd8d366..cd6753ff0e 100644 --- a/examples/personal-workspace-browser/configuration-backup.mjs +++ b/examples/personal-workspace-browser/configuration-backup.mjs @@ -10,14 +10,28 @@ import {resolveTestPython} from "../../scripts/test-python.mjs"; // Only the backup routes use a real disposable server. Other workspace state // uses the shared browser fixture; no Host, model or live runtime is contacted. const serverCode = ` -import json,pathlib,sys,loopx +import json,pathlib,sys,loopx,subprocess,os from loopx.chat_server import ChatHTTPServer,ChatRequestHandler +from loopx.state_backup import build_state_backup_plan,execute_state_backup_plan +from loopx.control_plane.coordination.runtime_shadow import build_todo_runtime_shadow_projection r=pathlib.Path(sys.argv[1]); runtime=r/'runtime'; registry=r/'registry.json' registry.write_text(json.dumps({'common_runtime_root':str(runtime),'goals':[{'id':'fixture','repo':str(r),'control_plane':{'optional':{'context':'complete '*10000}}}]})) p=runtime/'machine/configuration.json';p.parent.mkdir(parents=True) p.write_text(json.dumps({'schema_version':'loopx_machine_configuration_v0','namespaces':{'goal_storage':{'schema_version':'loopx_goal_storage_defaults_v0','new_goal_provider':'sqlite'}}})) +source_project=r/'source-project';source_project.mkdir() +source_runtime=r/'source-runtime';source_registry=r/'source-registry.json' +(source_project/'ACTIVE.md').write_text('# Original Goal\\n\\nPreserve every unfinished commitment.\\n') +source_registry.write_text(json.dumps({'common_runtime_root':str(source_runtime),'goals':[{'id':'recovery-fixture','repo':str(source_project),'state_file':'ACTIVE.md','coordination':{'registered_agents':['worker']}}]})) +projection=build_todo_runtime_shadow_projection(goal_id='recovery-fixture',todos=[{'schema_version':'todo_item_v0','todo_id':'todo_recovery_work','role':'agent','status':'open','done':False,'archive_state':'active','source_section':'Agent Todo','text':'Preserve the original commitment','task_class':'advancement_task'}],handoff_mode='soft_claim') +provider=(pathlib.Path(loopx.__file__).parent/'control_plane/coordination/local_authority_provider.ts').as_uri() +source_provider=os.environ.get('LOOPX_RECOVERY_TEST_PROVIDER','sqlite');assert source_provider in ('file','sqlite') +script='import {selectLocalSqliteAuthority,openLocalAuthorityStore} from '+json.dumps(provider)+';let input="";for await(const c of process.stdin)input+=c;const r=JSON.parse(input);if(r.provider==="sqlite")await selectLocalSqliteAuthority(r.root,r.goal,true);const s=await openLocalAuthorityStore(r.root,r.goal);const x=await s.commitAuthority({expected_provider_revision:null,operation_id:"original-work",events:[],receipts:[],next_projection:r.projection});if(x.status!=="applied")throw new Error(JSON.stringify(x));' +subprocess.run(['node','--no-warnings','--experimental-sqlite','--experimental-strip-types','--input-type=module','-e',script],input=json.dumps({'root':str(source_runtime),'goal':'recovery-fixture','provider':source_provider,'projection':projection}),text=True,check=True,capture_output=True) +saved=execute_state_backup_plan(build_state_backup_plan(project=source_project,runtime_root=source_runtime,registry_path=source_registry,output_dir=r/'source-backup',backup_id='recovery-fixture',include_registry_projects=False,include_automations=False,include_skills=False)) +source_project.rename(r/'unavailable-project');source_runtime.rename(r/'unavailable-runtime');source_registry.rename(r/'unavailable-registry.json') +assert not source_project.exists() and not source_runtime.exists() and not source_registry.exists() s=ChatHTTPServer(('127.0.0.1',0),ChatRequestHandler);s.registry_path=registry;s.runtime_root=runtime;s.verbose=False -print(json.dumps({'port':s.server_address[1],'python':sys.executable,'module_path':loopx.__file__}),flush=True);s.serve_forever() +print(json.dumps({'port':s.server_address[1],'python':sys.executable,'module_path':loopx.__file__,'archive':saved['archive_path'],'manifest':saved['manifest_path'],'source_provider':source_provider,'original_sources_absent':True}),flush=True);s.serve_forever() `; export const configurationBackupScenario = { @@ -70,6 +84,11 @@ export const configurationBackupScenario = { const response = await route.fetch({url: `http://127.0.0.1:${port}${path}`}); await route.fulfill({response}); }); + await page.route("**/api/chat/state-recovery**", async route => { + const path = new URL(route.request().url()).pathname; + const response = await route.fetch({url: `http://127.0.0.1:${port}${path}`}); + await route.fulfill({response}); + }); }}); const {page} = context; await page.getByRole("button", {name: "设置", exact: true}).click(); @@ -90,9 +109,86 @@ export const configurationBackupScenario = { await input.setInputFiles({name: "bad.json", mimeType: "application/json", buffer: Buffer.from(JSON.stringify({...backup, sha256: "changed"}))}); await panel.getByRole("alert").waitFor(); assert.equal(await panel.getByRole("button", {name: "恢复为隔离检查点"}).count(), 0); + await panel.getByRole("radio", {name: "完整归档检查", exact: true}).check(); + const recovery = panel.getByRole("region", {name: "完整备份检查"}); + await recovery.scrollIntoViewIfNeeded(); + await page.screenshot({path: resolve(outputDir, "state-recovery-ready.png"), animations: "disabled"}); + await recovery.getByLabel("完整归档", {exact: true}).setInputFiles(backend.archive); + await recovery.getByLabel("配套清单", {exact: true}).setInputFiles(backend.manifest); + await recovery.getByLabel("配套清单", {exact: true}).focus(); + await page.keyboard.press("Tab"); + assert.equal(await page.evaluate(() => document.activeElement.textContent), "验证并建立隔离副本"); + await page.keyboard.press("Enter"); + await recovery.getByText("隔离副本已核对;Goal 尚未启用。", {exact: true}).waitFor(); + await recovery.getByText("recovery-fixture", {exact: false}).first().waitFor(); + await recovery.getByText("保存的任务: 1", {exact: false}).waitFor(); + const verificationDetails = recovery.locator("details"); + await verificationDetails.locator("summary").click(); + await verificationDetails.getByText("归档字节:", {exact: false}).waitFor(); + const recordedMeasurements = await verificationDetails.locator("p").innerText(); + await verificationDetails.scrollIntoViewIfNeeded(); + await page.screenshot({path: resolve(outputDir, "state-recovery-measurements.png"), animations: "disabled"}); + await verificationDetails.locator("summary").click(); + assert.equal(backend.original_sources_absent, true); + const registryAfter = JSON.parse(await readFile(join(root, "registry.json"), "utf8")); + assert.deepEqual(registryAfter.goals.map(goal => goal.id), ["fixture"], "inspection must not register the restored Goal"); + await recovery.getByRole("status").scrollIntoViewIfNeeded(); + await page.screenshot({path: resolve(outputDir, "state-recovery-audit.png"), animations: "disabled"}); + await recovery.getByLabel("完整归档", {exact: true}).setInputFiles({name: "damaged.tar.gz", mimeType: "application/gzip", buffer: Buffer.from("damaged")}); + await recovery.getByRole("button", {name: "验证并建立隔离副本", exact: true}).click(); + await recovery.getByRole("alert").waitFor(); + assert.equal(await recovery.getByText("保存的任务: 1", {exact: false}).count(), 0, "a changed input must clear the old success view"); + // Optional real capacity fixture, produced through the normal backup + // command. The ordinary browser smoke does not seed 20,001 transactions. + if (process.env.LOOPX_RECOVERY_HISTORY_OVERFLOW_ARCHIVE) { + assert.ok(process.env.LOOPX_RECOVERY_HISTORY_OVERFLOW_MANIFEST); + await recovery.getByLabel("完整归档", {exact: true}).setInputFiles(process.env.LOOPX_RECOVERY_HISTORY_OVERFLOW_ARCHIVE); + await recovery.getByLabel("配套清单", {exact: true}).setInputFiles(process.env.LOOPX_RECOVERY_HISTORY_OVERFLOW_MANIFEST); + await recovery.getByRole("button", {name: "验证并建立隔离副本", exact: true}).click(); + await recovery.getByRole("alert").filter({hasText: "20,000"}).waitFor(); + assert.equal(await recovery.getByRole("button", {name: "检查是否已有结果", exact: true}).count(), 0, + "a definitive pre-publication refusal must not suggest a lost response"); + assert.equal(await recovery.getByText("保存的任务: 1", {exact: false}).count(), 0); + await page.screenshot({path: resolve(outputDir, "state-recovery-history-capacity.png"), animations: "disabled"}); + } + await recovery.getByRole("button", {name: "历史检查结果", exact: true}).click(); + const history = recovery.getByLabel("历史备份检查", {exact: true}); + await history.getByRole("button").click(); + await recovery.getByText("已读取历史检查记录;尚未重新验证副本。", {exact: true}).waitFor(); await page.setViewportSize({width: 390, height: 844}); assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth), false); - return {coverageEntries: context.coverageEntries, errors: context.errors, backend, verified: "Packaged download, full real-backend checkpoint recovery, damaged-file rejection and narrow viewport"}; + await recovery.getByRole("status").scrollIntoViewIfNeeded(); + await page.screenshot({path: resolve(outputDir, "state-recovery-mobile.png"), animations: "disabled"}); + await recovery.getByRole("status").evaluate(element => element.scrollIntoView({block: "start"})); + await page.screenshot({path: resolve(outputDir, "state-recovery-mobile-result.png"), animations: "disabled"}); + await recovery.getByRole("button", {name: "历史检查结果", exact: true}).click(); + await recovery.getByLabel("历史备份检查", {exact: true}).getByRole("button").click(); + await page.setViewportSize({width: 1512, height: 982}); + await page.reload(); + await page.getByRole("button", {name: "设置", exact: true}).or(page.locator(".personal-settings-tabs")).first().waitFor(); + if (await page.getByRole("button", {name: "设置", exact: true}).count()) await page.getByRole("button", {name: "设置", exact: true}).click(); + await page.locator(".personal-settings-tabs").getByRole("button", {name: "能力中心", exact: true}).click(); + const reopened = page.locator("details").filter({has: page.getByText("配置备份与恢复", {exact: true})}); + await reopened.locator("summary").click(); + await reopened.getByRole("radio", {name: "完整归档检查", exact: true}).check(); + await reopened.getByRole("button", {name: "历史检查结果", exact: true}).click(); + await reopened.getByLabel("历史备份检查", {exact: true}).getByRole("button").click(); + await reopened.getByText("已读取历史检查记录;尚未重新验证副本。", {exact: true}).waitFor(); + await reopened.locator(".personal-recovery-result details summary").click(); + assert.equal(await reopened.locator(".personal-recovery-result details p").innerText(), recordedMeasurements, + "historical readback must retain original measured values, not time the reload"); + await page.getByRole("button", {name: "语言", exact: true}).click(); + await page.getByRole("radio", {name: "English", exact: true}).check(); + await page.locator(".personal-settings-tabs").getByRole("button", {name: /Capabilit/}).click(); + const english = page.locator("details").filter({has: page.getByText("Configuration backup and recovery", {exact: true})}); + await english.locator("summary").click(); + await english.getByRole("radio", {name: "Full archive inspection", exact: true}).check(); + await english.getByRole("button", {name: "Previous inspections", exact: true}).click(); + await english.getByLabel("Previous backup inspections", {exact: true}).getByRole("button").click(); + await english.getByText("Historical inspection loaded; the copy has not been reverified.", {exact: true}).waitFor(); + await english.getByRole("status").scrollIntoViewIfNeeded(); + await page.screenshot({path: resolve(outputDir, "state-recovery-english.png"), animations: "disabled"}); + return {coverageEntries: context.coverageEntries, errors: context.errors, backend, verified: `Packaged configuration recovery and full archive inspection; real ${backend.source_provider} head/history after original-source loss, no Goal registration, keyboard, corrupt-input recovery, immutable history reload and narrow viewport`}; } finally { try {await context?.close();} finally { if (!closed) { diff --git a/loopx/cli_commands/support_control_backup.py b/loopx/cli_commands/support_control_backup.py index 6ed35962dd..73c12fa7ed 100644 --- a/loopx/cli_commands/support_control_backup.py +++ b/loopx/cli_commands/support_control_backup.py @@ -30,6 +30,11 @@ def register_backup_state_command( help="Preview or create a private local archive of LoopX state.", ) add_subcommand_format(parser) + parser.add_argument("backup_action", nargs="?", choices=["verify"], + help="Verify a full archive and inspect an inert copy; never reactivate it.") + parser.add_argument("--archive", help="For verify, the private full-state tar.gz archive.") + parser.add_argument("--manifest", help="For verify, the matching private external manifest.") + parser.add_argument("--destination", help="For verify, a new inert directory below a physical parent.") parser.add_argument( "--project", default=".", @@ -82,6 +87,27 @@ def handle_backup_state_command( print_payload: PrintPayload, output_format: FormatSelector, ) -> int: + if getattr(args, "backup_action", None) == "verify": + from ..state_recovery import StateRecoveryError, render_state_recovery_markdown, verify_state_backup + + try: + if not all(getattr(args, key, None) for key in ("archive", "manifest", "destination")): + raise StateRecoveryError("recovery_inputs_required") + if args.output_dir or args.backup_id or args.no_automations or args.no_skills or args.current_project_only or args.project != ".": + raise StateRecoveryError("recovery_creation_options_invalid") + payload = verify_state_backup(archive=Path(args.archive).expanduser(), + manifest=Path(args.manifest).expanduser(), destination=Path(args.destination).expanduser().absolute(), + execute=bool(args.execute)) + except StateRecoveryError as exc: + payload = {"ok": False, "error_code": exc.code, "execution_authority_granted": False} + except OSError: + payload = {"ok": False, "error_code": "recovery_io_unavailable", "execution_authority_granted": False} + print_payload(payload, output_format(args), render_state_recovery_markdown) + return 0 if payload.get("ok") else 1 + if any(getattr(args, key, None) for key in ("archive", "manifest", "destination")): + payload = {"ok": False, "error": "archive, manifest and destination require backup-state verify"} + print_payload(payload, output_format(args), render_state_backup_markdown) + return 1 try: runtime_root = ( Path(args.runtime_root).expanduser() if args.runtime_root else None diff --git a/loopx/control_plane/effect_program.ts b/loopx/control_plane/effect_program.ts index 9f3d806a97..822f5164f8 100644 --- a/loopx/control_plane/effect_program.ts +++ b/loopx/control_plane/effect_program.ts @@ -1,6 +1,6 @@ import {assertActionSelectionConsistency} from "./work_items/action_selection_admission.ts"; import { EffectRuntimeRequestError } from "./effect_runtime_errors.ts"; -import { isStringLiteral } from "./runtime_decode.ts"; +import { isStringLiteral, type JsonObject } from "./runtime_decode.ts"; import { decodeInteractionContract } from "./work_items/interaction_contract.ts"; import { EffectiveAction } from "./quota/effective_action.generated.ts"; @@ -27,7 +27,7 @@ export const SETTLEMENT_PLAN_SCHEMA_VERSION = "quota_settlement_plan_v1"; export const SETTLEMENT_RECEIPT_SCHEMA_VERSION = "quota_settlement_receipt_v1"; -export type JsonObject = Record; +export type {JsonObject} from "./runtime_decode.ts"; export interface EffectRequest { kind: string; diff --git a/loopx/control_plane/effect_runtime.py b/loopx/control_plane/effect_runtime.py index c90e742898..5d57b90d7c 100644 --- a/loopx/control_plane/effect_runtime.py +++ b/loopx/control_plane/effect_runtime.py @@ -49,6 +49,8 @@ "goal.checkpoint_read_context.commit", "goal.checkpoint_read_context.inspect_replay", "performance_diagnosis.inspect", + "state.recovery.archive.verify", + "state.recovery.manifest.verify", }) MAX_STARTUP_DIAGNOSTIC_BYTES = 8 * 1024 STARTUP_LOCK_TIMEOUT_SECONDS = 15.0 diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 0f752d681c..8f083422c1 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -410,6 +410,9 @@ export function createEffectRuntimeHandlers( ["coordination.local_authority.promote", lazyHandler(() => import("./coordination/local_authority_runtime.ts"), ({promoteLocalCoordinationAuthority}) => promoteLocalCoordinationAuthority)], ["coordination.authority_archive.manage", lazyHandler(() => import("./coordination/local_authority_archive.ts"), ({manageLocalAuthorityArchive}) => manageLocalAuthorityArchive)], ["configuration.backup", lazyHandler(() => import("./configuration_backup.ts"), ({configurationBackupOperation}) => configurationBackupOperation)], + ["state.recovery.archive.verify", lazyHandler(() => import("./state_recovery.ts"), ({verifyStateRecoveryArchive}) => verifyStateRecoveryArchive)], + ["state.recovery.manifest.verify", lazyHandler(() => import("./state_recovery.ts"), ({verifyStateRecoveryManifest}) => verifyStateRecoveryManifest)], + ["state.recovery.audit.readback", lazyHandler(() => import("./state_recovery.ts"), ({verifySavedStateRecoveryAudit}) => verifySavedStateRecoveryAudit)], ["coordination.sqlite_backup.snapshot", lazyHandler(() => import("./coordination/sqlite_backup.ts"), ({snapshotSqliteBackup}) => snapshotSqliteBackup)], ["coordination.local_authority.new_goal_storage", lazyHandler(() => import("./coordination/local_authority_defaults.ts"), ({manageNewGoalStorage}) => manageNewGoalStorage)], ["coordination.local_authority.promotion_review", lazyHandler(() => Promise.all([import("./coordination/local_authority_runtime.ts"), import("./coordination/source_transfer.ts")]), ([{reviewLocalCoordinationAuthorityPromotion}, {withCoordinationSourceTransfer}]) => withCoordinationSourceTransfer("coordination.local_authority.promotion_review", reviewLocalCoordinationAuthorityPromotion))], diff --git a/loopx/control_plane/effect_runtime_server.ts b/loopx/control_plane/effect_runtime_server.ts index 39139eb382..8110bc3917 100644 --- a/loopx/control_plane/effect_runtime_server.ts +++ b/loopx/control_plane/effect_runtime_server.ts @@ -35,6 +35,8 @@ const LOCAL_SNAPSHOT_METHODS = new Set([ "goal.checkpoint_read_context.commit", "goal.checkpoint_read_context.inspect_replay", "performance_diagnosis.inspect", + "state.recovery.archive.verify", + "state.recovery.manifest.verify", ]); const DEFAULT_IDLE_MS = 5 * 60 * 1_000; // Bounds for LOOPX_EFFECT_RUNTIME_IDLE_MS. The upper bound is the largest diff --git a/loopx/control_plane/runtime_decode.ts b/loopx/control_plane/runtime_decode.ts index 8245e18c62..975b7766aa 100644 --- a/loopx/control_plane/runtime_decode.ts +++ b/loopx/control_plane/runtime_decode.ts @@ -1,6 +1,7 @@ -import type { JsonObject } from "./effect_program.ts"; import { EffectRuntimeRequestError } from "./effect_runtime_errors.ts"; +export type JsonObject = Record; + export function jsonObject(value: unknown): JsonObject | null { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as JsonObject) diff --git a/loopx/control_plane/state_recovery.ts b/loopx/control_plane/state_recovery.ts new file mode 100644 index 0000000000..89f24d401e --- /dev/null +++ b/loopx/control_plane/state_recovery.ts @@ -0,0 +1,294 @@ +/** Offline backup inspection. Existing owners decide readability, never activation. */ +import {createHash} from "node:crypto"; +import {lstat, readFile} from "node:fs/promises"; +import {join, relative} from "node:path"; +import type {JsonObject} from "./effect_program.ts"; +import {requireJsonObject} from "./runtime_decode.ts"; +import {canonicalAuthoritySha256, AuthorityStoreProtocolError, parseAuthorityCursor} from "./coordination/authority_store_codec.ts"; +import {EffectRuntimeRequestError} from "./effect_runtime_errors.ts"; +import {verifyConfigurationBackup} from "./configuration_backup.ts"; +import {decodeLocalAuthoritySelection, localAuthorityProviderPaths, + openLocalAuthorityStoreHandle} from "./coordination/local_authority_provider.ts"; +import {sqliteAuthorityRuntime} from "./coordination/sqlite_runtime.ts"; +import {sqliteAuthorityPath} from "./coordination/sqlite_authority_store.ts"; +import {canonicalTodoCollection} from "./coordination/local_authority_read.ts"; +import {STATE_RECOVERY_AUDIT_SCHEMA, decodeRecoveryMeasurements, decodeStateRecoveryAudit, isRecoveryDigest, recoveryGoalLabel, + type RecoveryComponent, type RecoveryComponentState} from "./state_recovery_contract.ts"; + +const MAX_GOALS = 128; +const MAX_SQLITE_FILES = 128; +const MAX_COMMITS = 20_000; +const hash = (value: string) => createHash("sha256").update(value).digest("hex"); +const digest = isRecoveryDigest; +class RecoveryInputError extends Error { + readonly code: string; + constructor(code: string) { super(code); this.code = code; } +} +function check(value: unknown, code: string): asserts value { + if (!value) throw new RecoveryInputError(code); +} +function reject(code: string): JsonObject { + return {ok: false, error_code: code, execution_authority_granted: false}; +} +function manifestCore(value: JsonObject): JsonObject { + const execution = {...requireJsonObject(value.execution, "backup execution")}; + execution.archive_sha256 = null; + execution.archive_size_bytes = null; + delete execution.archive_to_logical_ratio; + return {...value, execution}; +} + +function externalEvidence(request: JsonObject): {external: JsonObject; execution: JsonObject} { + const external = requireJsonObject(request.external_manifest, "external backup manifest"); + check(external.schema_version === "loopx_state_backup_v0", "backup_schema_unsupported"); + check(external.dry_run === false && external.execute_requested === true && external.ok === true, + "backup_not_committed"); + check(typeof external.backup_id === "string" && external.backup_id.length > 0, "backup_identity_missing"); + const execution = requireJsonObject(external.execution, "backup execution"); + check(digest(request.archive_sha256) && execution.archive_sha256 === request.archive_sha256, + "archive_digest_mismatch"); + check(Number.isSafeInteger(request.archive_size_bytes) && execution.archive_size_bytes === request.archive_size_bytes, + "archive_size_mismatch"); + return {external, execution}; +} +function inputFailure(error: unknown): JsonObject { + if (error instanceof RecoveryInputError) return reject(error.code); + if (error instanceof EffectRuntimeRequestError || error instanceof AuthorityStoreProtocolError) { + return reject("backup_manifest_invalid"); + } + throw error; +} + +/** Reject a mismatched archive before its untrusted tar structure is decoded. */ +export function verifyStateRecoveryArchive(value: unknown): JsonObject { + try { + externalEvidence(requireJsonObject(value, "recovery archive verification")); + return {ok: true, archive_digest_verified: true, execution_authority_granted: false}; + } catch (error) { return inputFailure(error); } +} + +/** Host IO supplies observed bytes; input manifests never supply observed facts. */ +export function verifyStateRecoveryManifest(value: unknown): JsonObject { + try { + const request = requireJsonObject(value, "recovery manifest verification"); + const {external, execution} = externalEvidence(request); + const embedded = requireJsonObject(request.embedded_manifest, "embedded backup manifest"); + check(embedded.schema_version === external.schema_version, + "backup_schema_unsupported"); + check(canonicalAuthoritySha256(manifestCore(external)) === canonicalAuthoritySha256(manifestCore(embedded)), + "manifest_disagreement"); + check(Array.isArray(request.files), "member_inventory_invalid"); + check(Number.isSafeInteger(request.sqlite_member_count) && Number(request.sqlite_member_count) >= 0 + && Number(request.sqlite_member_count) <= MAX_SQLITE_FILES, "sqlite_inventory_capacity"); + const observed = new Map(); + for (const item of request.files) { + const file = requireJsonObject(item, "observed member"); + check(typeof file.archive_path === "string" && !observed.has(file.archive_path) + && digest(file.sha256) && Number.isSafeInteger(file.size_bytes) && Number(file.size_bytes) >= 0, + "member_inventory_invalid"); + observed.set(file.archive_path, file); + } + const witnesses = execution.file_members; + if (witnesses !== undefined) { + check(Array.isArray(witnesses) && witnesses.length === observed.size, "member_witness_mismatch"); + const seen = new Set(); + for (const item of witnesses) { + const witness = requireJsonObject(item, "member witness"); + const path = String(witness.archive_path); + check(!seen.has(path), "member_witness_mismatch"); + seen.add(path); + const file = observed.get(path); + check(file && file.sha256 === witness.sha256 && file.size_bytes === witness.size_bytes, + "member_witness_mismatch"); + } + } + check(digest(request.external_manifest_sha256), "manifest_digest_invalid"); + return {ok: true, schema_version: STATE_RECOVERY_AUDIT_SCHEMA, + recovery_id: hash(`${request.archive_sha256}:${request.external_manifest_sha256}`), + backup: {schema_version: external.schema_version, archive_sha256: request.archive_sha256, + external_manifest_sha256: request.external_manifest_sha256}, + file_count: observed.size, file_witnesses_verified: witnesses !== undefined, + manifest_verified: true, capture_profile: "legacy_best_effort", execution_authority_granted: false}; + } catch (error) { + return inputFailure(error); + } +} + +function component(id: string, owner: string, state: RecoveryComponentState, reason: string, + next: string, facts: JsonObject = {}, dependencies: string[] = []): RecoveryComponent { + return {component_id: id, owner_kind: owner, state, reason_code: reason, + next_action: next, facts, dependency_ids: dependencies}; +} +async function regular(path: string): Promise { + const stat = await lstat(path); + check(stat.isFile() && !stat.isSymbolicLink(), "owner_source_not_regular"); +} + +async function inspectGoal(root: string, goalId: string, standaloneSqlite: Set): Promise { + const id = `goal:${hash(goalId).slice(0, 20)}`; + const paths = localAuthorityProviderPaths(root, goalId); + const goalFacts = recoveryGoalLabel(goalId) === null ? {} : {goal_id: goalId}; + try { + let selection: ReturnType | null = null; + try { + await regular(paths.marker); + selection = decodeLocalAuthoritySelection(JSON.parse(await readFile(paths.marker, "utf8")), goalId); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (selection?.provider === "postgresql") return component(id, "coordination_authority", "unknown", + "offline_provider_unsupported", "use_provider_recovery_owner", goalFacts, ["configuration"]); + if (selection?.provider === "sqlite") check(standaloneSqlite.has(sqliteAuthorityPath(paths.sqlite, goalId)), + "sqlite_requires_owner_recovery"); + // The existing owner retains legacy File reads and rejects missing selected + // storage or a lost SQLite selector. existingOnly forbids initializing either. + const handle = await openLocalAuthorityStoreHandle(root, goalId, {}, {existingOnly: true}); + const loaded = await handle.store.loadAuthority(); + check(loaded.status === "loaded", "authority_head_unavailable"); + check(parseAuthorityCursor(loaded.cursor) <= BigInt(MAX_COMMITS), "authority_history_capacity"); + // Byte-valid storage alone is not a readable Goal work model. Reuse the + // same full-record, lease and acceptance decoder as canonical Todo reads. + const collection = canonicalTodoCollection(loaded.head, goalId, true); + let cursor: string | null = null; + let count = 0; + let final: JsonObject | null = null; + let finalRevision: string | null = null; + while (true) { + const page = await handle.store.scanCommitted(cursor, 200); + check(page.status === "page", "authority_history_unavailable"); + count += page.transactions.length; + check(count <= MAX_COMMITS, "authority_history_capacity"); + for (const row of page.transactions) { final = row.projection; finalRevision = row.provider_revision; } + if (!page.has_more) break; + check(page.next_cursor !== null && page.next_cursor !== cursor && page.transactions.length > 0, + "authority_history_cursor_invalid"); + cursor = page.next_cursor; + } + check(String(count) === loaded.cursor && final !== null && finalRevision === loaded.provider_revision + && canonicalAuthoritySha256(final) === canonicalAuthoritySha256(loaded.head), "authority_history_head_mismatch"); + const readback = await handle.store.loadAuthority(); + check(readback.status === "loaded" && readback.provider_revision === loaded.provider_revision, + "authority_changed_during_audit"); + return component(id, "coordination_authority", "readable", "historical_head_and_history_readable", + "review_goal_identity_before_adoption", {...goalFacts, provider: handle.provider, + head_digest: canonicalAuthoritySha256(loaded.head), history_commits: count, + todo_count: collection.projection.todo_ids.length, + historical_lease_count: collection.leaseIndex?.lease_todo_ids.length ?? 0}, ["configuration"]); + } catch (error) { + // Inspection profile overflow aborts publication; it is not evidence that + // this owner's captured state is merely unknown or unreadable. + if (error instanceof RecoveryInputError && error.code === "authority_history_capacity") throw error; + return component(id, "coordination_authority", "unknown", + error instanceof RecoveryInputError ? error.code : "authority_source_unavailable", + "inspect_captured_authority_with_owner", goalFacts, ["configuration"]); + } +} + +/** Reads only a host-validated inert directory; never registers or repairs a store. */ +async function auditOwners(value: unknown): Promise { + const started = performance.now(); + const request = requireJsonObject(value, "recovery owner audit"); + const measurements = decodeRecoveryMeasurements({ + ...requireJsonObject(request.measurements, "host recovery measurements"), owner_read_ms: 0}); + const basis = requireJsonObject(request.verified, "verified recovery basis"); + check(basis.ok === true && basis.manifest_verified === true && basis.schema_version === STATE_RECOVERY_AUDIT_SCHEMA, + "recovery_basis_invalid"); + check(typeof request.data_root === "string" && Array.isArray(request.sqlite_members), "recovery_owner_request_invalid"); + const dataRoot = request.data_root; + check(request.sqlite_members.length <= MAX_SQLITE_FILES, "sqlite_inventory_capacity"); + const components: RecoveryComponent[] = []; + let configuration: JsonObject | null = null; + try { + const path = join(dataRoot, "configuration-backup.json"); + await regular(path); + const candidate = requireJsonObject(JSON.parse(await readFile(path, "utf8")), "captured configuration"); + const verified = verifyConfigurationBackup(candidate); + configuration = candidate; + components.push(component("configuration", "configuration", "readable", "configuration_digest_verified", + "retain_isolated_configuration", {goal_count: verified.goal_count})); + } catch { + components.push(component("configuration", "configuration", "unknown", "configuration_unavailable", + "inspect_configuration_with_owner")); + } + check(Array.isArray(request.standalone_sqlite_members), "recovery_owner_request_invalid"); + const standaloneSqlite = new Set(request.standalone_sqlite_members.map(raw => join(dataRoot, String(raw)))); + const ownerReadableSqlite = new Set(); + for (const raw of request.sqlite_members) { + const path = String(raw); + // The IO adapter has already verified all names and refused symlink ancestors. + check(!path.startsWith("/") && !path.split("/").includes(".."), "sqlite_member_invalid"); + const id = `sqlite:${hash(path).slice(0, 20)}`; + try { + const fullPath = join(dataRoot, path); + check(relative(dataRoot, fullPath) === path && standaloneSqlite.has(fullPath), "sqlite_requires_owner_recovery"); + await regular(fullPath); + const {driver} = sqliteAuthorityRuntime(); + const limits = new driver.DatabaseSync(":memory:"); + try { + // SQLite's heap limit is process-wide; this module is invoked only in + // the disposable verifier, never in the shared Effect daemon. + limits.exec("PRAGMA hard_heap_limit=67108864"); + const actual = Number(limits.prepare("PRAGMA hard_heap_limit").get()?.hard_heap_limit); + check(actual > 0 && actual <= 64 * 1024 * 1024, "sqlite_memory_limit_unavailable"); + } finally { limits.close(); } + const database = new driver.DatabaseSync(fullPath, {readOnly: true}); + try { + database.exec("PRAGMA query_only=ON; PRAGMA trusted_schema=OFF"); + const tables = database.prepare("PRAGMA table_list").all(); + const physicalTablesOnly = tables.every(row => row.type === "table"); + check(physicalTablesOnly, "sqlite_schema_requires_owner_recovery"); + const rows = database.prepare("PRAGMA integrity_check").all(); + check(rows.length === 1 && rows[0]?.integrity_check === "ok", "sqlite_integrity_failed"); + // Live store readers assume their own physical tables. Never evaluate + // a restored view or virtual table as an authority table. + ownerReadableSqlite.add(fullPath); + } finally { database.close(); } + components.push(component(id, "sqlite_storage", "historical_only", "sqlite_bytes_verified", + "retain_historical_database")); + } catch (error) { + components.push(component(id, "sqlite_storage", "unknown", + error instanceof RecoveryInputError ? error.code : "sqlite_verification_failed", + "inspect_database_with_owner")); + } + } + if (configuration !== null) { + const goals = (configuration.data as JsonObject).goals as JsonObject[]; + check(goals.length <= MAX_GOALS, "goal_inventory_capacity"); + for (const goal of goals) components.push(await inspectGoal(join(dataRoot, "runtime-root"), String(goal.goal_id), ownerReadableSqlite)); + } + // A v0 file inventory does not identify every session/effect/extension owner. + // Retained bytes must not masquerade as a successful owner-local read. + components.push(component("remaining_state", "recovery", "unknown", "legacy_owner_inventory_incomplete", + "classify_remaining_owner_state")); + const audit = {...basis, status: "held", written: true, observed_at: new Date().toISOString(), + measurements: {...measurements, owner_read_ms: Math.floor(performance.now() - started)}, + workspace: {isolation: "inert", registered: false}, components, + findings: [{reason_code: "legacy_manifest_incomplete", owner_kind: "backup_capture", + next_action: "review_capture_consistency"}], + summary: {readable: components.filter(row => row.state === "readable").length, + historical_only: components.filter(row => row.state === "historical_only").length, + unknown: components.filter(row => row.state === "unknown").length}, + activation_eligible: false, execution_authority_granted: false}; + return decodeStateRecoveryAudit({...audit, audit_sha256: canonicalAuthoritySha256(audit)}); +} + +export async function auditStateRecoveryOwners(value: unknown): Promise { + try { return await auditOwners(value); } + catch (error) { return inputFailure(error); } +} + +export function verifySavedStateRecoveryAudit(value: unknown): JsonObject { + try { + const audit = decodeStateRecoveryAudit(value); + const {audit_sha256, ...body} = audit; + check(canonicalAuthoritySha256(body) === audit_sha256, "recovery_audit_digest_mismatch"); + const backup = requireJsonObject(audit.backup, "recovery backup basis"); + check(audit.recovery_id === hash(`${backup.archive_sha256}:${backup.external_manifest_sha256}`), + "recovery_audit_identity_mismatch"); + return audit; + } catch (error) { + if (error instanceof RecoveryInputError) return reject(error.code); + if (error instanceof EffectRuntimeRequestError || error instanceof AuthorityStoreProtocolError) return reject("recovery_audit_invalid"); + throw error; + } +} diff --git a/loopx/control_plane/state_recovery_contract.ts b/loopx/control_plane/state_recovery_contract.ts new file mode 100644 index 0000000000..89394c084d --- /dev/null +++ b/loopx/control_plane/state_recovery_contract.ts @@ -0,0 +1,99 @@ +/** Shared readback contract for the offline owner and packaged client. */ +import {BARE_SHA256_PATTERN} from "./content_digest.ts"; +import {EffectRuntimeRequestError} from "./effect_runtime_errors.ts"; +import {requireJsonObject, type JsonObject} from "./runtime_decode.ts"; + +export const STATE_RECOVERY_AUDIT_SCHEMA = "loopx_state_recovery_audit_v0"; +export const RECOVERY_COMPONENT_STATES = ["readable", "historical_only", "unknown"] as const; +export type RecoveryComponentState = typeof RECOVERY_COMPONENT_STATES[number]; +export interface RecoveryMeasurements extends JsonObject { + archive_bytes: number; + regular_file_bytes: number; + archive_processing_ms: number; + owner_read_ms: number; +} +export interface RecoveryComponent extends JsonObject { + component_id: string; + owner_kind: string; + state: RecoveryComponentState; + reason_code: string; + next_action: string; + facts: JsonObject; + dependency_ids: string[]; +} +export interface StateRecoveryAudit extends JsonObject { + schema_version: typeof STATE_RECOVERY_AUDIT_SCHEMA; + recovery_id: string; + audit_sha256: string; + observed_at: string; + status: "held"; + execution_authority_granted: false; + activation_eligible: false; + components: RecoveryComponent[]; + findings: Array<{reason_code: string; owner_kind: string; next_action: string}>; + summary: Record; + measurements?: RecoveryMeasurements; +} +function requireValue(condition: unknown): asserts condition { + if (!condition) throw new EffectRuntimeRequestError("State recovery audit contract is invalid", "recovery_audit_invalid"); +} +function token(value: unknown): boolean { + return typeof value === "string" && /^[a-z0-9_:.~-]{1,128}$/.test(value); +} +export function isRecoveryDigest(value: unknown): value is string { + return typeof value === "string" && BARE_SHA256_PATTERN.test(value); +} +/** Display only; unsafe labels retain their opaque component identity. */ +export function recoveryGoalLabel(value: string): string | null { + return /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(value) ? value : null; +} +export function decodeRecoveryMeasurements(value: unknown): RecoveryMeasurements { + const measurements = requireJsonObject(value, "recovery measurements"); + for (const field of ["archive_bytes", "regular_file_bytes", "archive_processing_ms", "owner_read_ms"]) { + requireValue(Number.isSafeInteger(measurements[field]) && Number(measurements[field]) >= 0); + } + return measurements as RecoveryMeasurements; +} +export function decodeStateRecoveryAudit(value: unknown): StateRecoveryAudit { + const audit = requireJsonObject(value, "state recovery audit"); + requireValue(audit.ok === true && audit.schema_version === STATE_RECOVERY_AUDIT_SCHEMA + && audit.status === "held" && audit.written === true && audit.manifest_verified === true + && audit.execution_authority_granted === false && audit.activation_eligible === false + && isRecoveryDigest(audit.recovery_id) && isRecoveryDigest(audit.audit_sha256)); + requireValue(typeof audit.observed_at === "string" && audit.observed_at.length <= 40 + && Number.isFinite(Date.parse(audit.observed_at))); + const workspace = requireJsonObject(audit.workspace, "inert recovery workspace"); + requireValue(workspace.isolation === "inert" && workspace.registered === false); + // Earlier reports remain historical observations; absent measurements are + // unknown, never synthesized from their timestamp or today's runtime. + if (audit.measurements !== undefined) decodeRecoveryMeasurements(audit.measurements); + const backup = requireJsonObject(audit.backup, "recovery backup basis"); + requireValue(backup.schema_version === "loopx_state_backup_v0" + && isRecoveryDigest(backup.archive_sha256) && isRecoveryDigest(backup.external_manifest_sha256)); + requireValue(Array.isArray(audit.components) && audit.components.length <= 512); + const ids = new Set(); + const counts = {readable: 0, historical_only: 0, unknown: 0}; + for (const value of audit.components) { + const row = requireJsonObject(value, "recovery component"); + requireValue(typeof row.component_id === "string" && token(row.component_id) && !ids.has(row.component_id) + && token(row.owner_kind) && token(row.reason_code) && token(row.next_action) + && RECOVERY_COMPONENT_STATES.includes(row.state as RecoveryComponentState) + && Array.isArray(row.dependency_ids) && row.dependency_ids.every(token)); + const facts = requireJsonObject(row.facts, "recovery component facts"); + requireValue(facts.goal_id === undefined || (typeof facts.goal_id === "string" && recoveryGoalLabel(facts.goal_id) === facts.goal_id)); + ids.add(row.component_id); + counts[row.state as RecoveryComponentState]++; + } + for (const value of audit.components) { + const row = value as RecoveryComponent; + requireValue(row.dependency_ids.every(id => ids.has(id) && id !== row.component_id)); + } + const summary = requireJsonObject(audit.summary, "recovery summary"); + requireValue(RECOVERY_COMPONENT_STATES.every(state => summary[state] === counts[state])); + requireValue(Array.isArray(audit.findings)); + for (const value of audit.findings) { + const finding = requireJsonObject(value, "recovery finding"); + requireValue(token(finding.reason_code) && token(finding.owner_kind) && token(finding.next_action)); + } + return audit as StateRecoveryAudit; +} diff --git a/loopx/control_plane/state_recovery_worker.ts b/loopx/control_plane/state_recovery_worker.ts new file mode 100644 index 0000000000..be0aaf590d --- /dev/null +++ b/loopx/control_plane/state_recovery_worker.ts @@ -0,0 +1,20 @@ +/** Disposable offline verifier: untrusted database work cannot stall the daemon. */ +import {auditStateRecoveryOwners} from "./state_recovery.ts"; + +try { + let size = 0; + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += data.length; + if (size > 2 * 1024 * 1024) throw new Error("input capacity"); + chunks.push(data); + } + const result = await auditStateRecoveryOwners(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + process.stdout.write(JSON.stringify(result)); + process.exitCode = result.ok === true ? 0 : 1; +} catch { + process.stdout.write(JSON.stringify({ok: false, error_code: "recovery_owner_unavailable", + execution_authority_granted: false})); + process.exitCode = 1; +} diff --git a/loopx/presentation/configuration_api.py b/loopx/presentation/configuration_api.py index bfc3110351..be03234550 100644 --- a/loopx/presentation/configuration_api.py +++ b/loopx/presentation/configuration_api.py @@ -3,6 +3,7 @@ from collections.abc import Callable from . import configuration_backup_api as backup_api +from . import state_recovery_api as recovery_api from . import goal_ownership_api as ownership_api from . import goal_storage_api as storage_api from .. import chat_usage_statistics_api as usage_api @@ -21,11 +22,14 @@ class ChatConfigurationRequestMixin( machine_api.MachineConfigurationRequestMixin, operator_api.OperatorProviderRequestMixin, backup_api.ConfigurationBackupRequestMixin, + recovery_api.StateRecoveryRequestMixin, ): """Expose machine and Goal configuration through one route registry.""" def _configuration_get_routes(self) -> dict[str, Callable[[], None]]: return { + recovery_api.STATE_RECOVERY_PATH: self._state_recovery_list, + f"{recovery_api.STATE_RECOVERY_PATH}/profile": self._state_recovery_profile, storage_api.CHAT_GOAL_STORAGE_PATH: self._storage_inspect, ownership_api.CHAT_GOAL_OWNERSHIP_PATH: self._ownership_inspect, cadence_api.CHAT_AUTOMATION_CADENCE_PATH: self._cadence_read, @@ -37,6 +41,10 @@ def _configuration_get_routes(self) -> dict[str, Callable[[], None]]: def _configuration_post_routes(self) -> dict[str, Callable[[], None]]: return { + f"{recovery_api.STATE_RECOVERY_PATH}/archive": lambda: self._state_recovery_upload("archive"), + f"{recovery_api.STATE_RECOVERY_PATH}/manifest": lambda: self._state_recovery_upload("manifest"), + f"{recovery_api.STATE_RECOVERY_PATH}/verify": self._state_recovery_verify, + f"{recovery_api.STATE_RECOVERY_PATH}/read": self._state_recovery_read, f"{storage_api.CHAT_GOAL_STORAGE_PATH}/preview": lambda: self._storage_update(action="plan-migration"), f"{storage_api.CHAT_GOAL_STORAGE_PATH}/apply": lambda: self._storage_update(action="migrate"), f"{storage_api.CHAT_GOAL_STORAGE_PATH}/recover": lambda: self._storage_update(action="migration-readback"), diff --git a/loopx/presentation/state_recovery_api.py b/loopx/presentation/state_recovery_api.py new file mode 100644 index 0000000000..9f596399d9 --- /dev/null +++ b/loopx/presentation/state_recovery_api.py @@ -0,0 +1,152 @@ +"""Loopback-only backup upload and immutable inert-audit readback.""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import re +import shutil +import tempfile +import time +from uuid import uuid4 + +from ..state_recovery import ( + MAX_ARCHIVE_BYTES, MAX_MANIFEST_BYTES, StateRecoveryError, + read_state_recovery_audit, verify_state_backup, +) +from ..status_server import is_loopback_host + +STATE_RECOVERY_PATH = "/api/chat/state-recovery" +_UPLOAD_TTL_SECONDS = 3600 +_REF = re.compile(r"^(archive|manifest):([a-f0-9]{32})$") + + +class StateRecoveryRequestMixin: + def _recovery_root(self, *, create: bool = False) -> Path: + if os.name != "posix": + raise StateRecoveryError("recovery_platform_unqualified") + if not is_loopback_host(str(self.server.server_address[0])): + raise StateRecoveryError("recovery_requires_loopback_server") + root = self.server.runtime_root / "backups" / "state-recovery" + if root.absolute() != root.resolve(): + raise StateRecoveryError("recovery_storage_not_physical") + if create: + root.mkdir(parents=True, exist_ok=True, mode=0o700) + (root / "inputs").mkdir(exist_ok=True, mode=0o700) + (root / "checkpoints").mkdir(exist_ok=True, mode=0o700) + for directory in (root, root / "inputs", root / "checkpoints"): + if directory.exists() or directory.is_symlink(): + info = directory.lstat() + if directory.is_symlink() or not directory.is_dir() or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise StateRecoveryError("recovery_storage_not_private") + return root + + def _state_recovery_profile(self) -> None: + supported = os.name == "posix" and is_loopback_host(str(self.server.server_address[0])) + self._send_json({"ok": True, "supported": supported, "activation_available": False, + "maximum_archive_bytes": MAX_ARCHIVE_BYTES, "maximum_manifest_bytes": MAX_MANIFEST_BYTES}) + + def _recovery_failure(self, error: Exception) -> None: + code = error.code if isinstance(error, StateRecoveryError) else "recovery_unavailable" + self._send_error("Backup inspection did not finish. Check the archive, manifest and available disk space.", + status=400 if isinstance(error, ValueError) else 503, error_code=code) + + def _state_recovery_upload(self, kind: str) -> None: + temporary: Path | None = None + previous_timeout = self.connection.gettimeout() + try: + if os.name != "posix": + raise StateRecoveryError("recovery_platform_unqualified") + maximum = MAX_ARCHIVE_BYTES if kind == "archive" else MAX_MANIFEST_BYTES + lengths = self.headers.get_all("Content-Length") or [] + if len(lengths) != 1 or not lengths[0].isdigit() or self.headers.get("Transfer-Encoding"): + raise StateRecoveryError("recovery_upload_length_invalid") + length = int(lengths[0]) + if not 0 < length <= maximum: + raise StateRecoveryError("recovery_capacity_exceeded") + root = self._recovery_root(create=True) + if shutil.disk_usage(root).free < length + MAX_MANIFEST_BYTES: + raise StateRecoveryError("recovery_disk_capacity") + inputs = root / "inputs" + # These are disposable upload inputs, not backups or audit receipts. + for old in inputs.iterdir(): + if re.fullmatch(r"(?:archive|manifest)-[a-f0-9]{32}", old.name) and not old.is_symlink(): + if old.stat().st_mtime < time.time() - _UPLOAD_TTL_SECONDS: + old.unlink() + fd, name = tempfile.mkstemp(prefix=".upload-", dir=inputs) + temporary = Path(name) + digest, remaining = hashlib.sha256(), length + self.connection.settimeout(30) + with os.fdopen(fd, "wb") as output: + while remaining: + chunk = self.rfile.read(min(256 * 1024, remaining)) + if not chunk: + raise StateRecoveryError("recovery_upload_incomplete") + output.write(chunk) + digest.update(chunk) + remaining -= len(chunk) + output.flush() + os.fsync(output.fileno()) + identifier = uuid4().hex + destination = inputs / f"{kind}-{identifier}" + os.link(temporary, destination) + self._send_json({"ok": True, "input_ref": f"{kind}:{identifier}", + "sha256": digest.hexdigest(), "size_bytes": length, + **({"checkpoint_ref": identifier} if kind == "archive" else {})}) + except (ValueError, OSError, RuntimeError) as error: + self.close_connection = True + self._recovery_failure(error) + finally: + self.connection.settimeout(previous_timeout) + if temporary is not None: + temporary.unlink(missing_ok=True) + + def _recovery_input(self, ref: object, kind: str) -> tuple[Path, str]: + match = _REF.fullmatch(ref) if isinstance(ref, str) else None + if match is None or match.group(1) != kind: + raise StateRecoveryError("recovery_input_ref_invalid") + root = self._recovery_root() + return root / "inputs" / f"{kind}-{match.group(2)}", match.group(2) + + def _state_recovery_verify(self) -> None: + try: + body = self._read_json() + if set(body) != {"archive_ref", "manifest_ref", "execute"} or type(body["execute"]) is not bool: + raise StateRecoveryError("recovery_request_invalid") + archive, identifier = self._recovery_input(body["archive_ref"], "archive") + manifest, _ = self._recovery_input(body["manifest_ref"], "manifest") + destination = self._recovery_root() / "checkpoints" / identifier + audit = verify_state_backup(archive=archive, manifest=manifest, destination=destination, + execute=body["execute"]) + self._send_json({**audit, "checkpoint_ref": identifier}) + except (ValueError, OSError, RuntimeError) as error: + self._recovery_failure(error) + + def _state_recovery_read(self) -> None: + try: + body = self._read_json() + ref = body.get("checkpoint_ref") + if set(body) != {"checkpoint_ref"} or not isinstance(ref, str) or not re.fullmatch(r"[a-f0-9]{32}", ref): + raise StateRecoveryError("recovery_checkpoint_ref_invalid") + audit = read_state_recovery_audit(self._recovery_root() / "checkpoints" / ref) + self._send_json({**audit, "checkpoint_ref": ref, "historical_readback": True}) + except (ValueError, OSError, RuntimeError) as error: + self._recovery_failure(error) + + def _state_recovery_list(self) -> None: + try: + parent = self._recovery_root() / "checkpoints" + candidates = [] if not parent.exists() else [path for path in parent.iterdir() + if re.fullmatch(r"[a-f0-9]{32}", path.name) and path.is_dir() and not path.is_symlink()] + candidates.sort(key=lambda path: path.stat().st_mtime_ns, reverse=True) + items = [] + for path in candidates[:10]: + try: + audit = read_state_recovery_audit(path) + items.append({"checkpoint_ref": path.name, "available": True, + "observed_at": audit["observed_at"], "summary": audit["summary"]}) + except (ValueError, OSError, RuntimeError): + items.append({"checkpoint_ref": path.name, "available": False}) + self._send_json({"ok": True, "items": items, "has_more": len(candidates) > 10}) + except (ValueError, OSError, RuntimeError) as error: + self._recovery_failure(error) diff --git a/loopx/semantics/project_registry_io_manifest_v1.json b/loopx/semantics/project_registry_io_manifest_v1.json index 96b0a61889..cbbbfd746e 100644 --- a/loopx/semantics/project_registry_io_manifest_v1.json +++ b/loopx/semantics/project_registry_io_manifest_v1.json @@ -919,7 +919,7 @@ }, { "site": "loopx/cli_commands/support_control_backup.py::.handle_backup_state_command::codec_read:load_registry#1", - "line": 90, + "line": 116, "column": 49, "kind": "codec_read", "api": "load_registry", diff --git a/loopx/semantics/vocabulary_v0.json b/loopx/semantics/vocabulary_v0.json index b66aaa51fe..dcb099dff1 100644 --- a/loopx/semantics/vocabulary_v0.json +++ b/loopx/semantics/vocabulary_v0.json @@ -13,7 +13,7 @@ "formal_model": { "schema_version": "loopx_semantic_formal_model_v0", "universes": { - "vocabularies": "V: registered vocabulary identifiers; Producers(V) ⊆ V is exactly the subset declaring a producers key (including compatibility-only entries with an empty list). Kernel(V) ⊆ Producers(V). Producers(V) contains 6 vocabularies: 5 kernel and 1 outside the kernel tier. The 20 cross_runtime vocabularies outside Producers(V) are unverified.", + "vocabularies": "V: registered vocabulary identifiers; Producers(V) ⊆ V is exactly the subset declaring a producers key (including compatibility-only entries with an empty list). Kernel(V) ⊆ Producers(V). Producers(V) contains 6 vocabularies: 5 kernel and 1 outside the kernel tier. The 21 cross_runtime vocabularies outside Producers(V) are unverified.", "values": "U(v): ambient runtime values; S(v): registered admitted values", "sites": "L: source locations that define, produce, consume, interpret, pass through, project, or persist values", "scopes": "Scope: global or bounded_context(context_id); ScopeDeclarations are the forked names the registry declares as bounded contexts", @@ -73,25 +73,25 @@ "invariants": [ { "id": "F1_producer_closedness", - "statement": "∀v ∈ Producers(V): Produced_scan(v) ⊆ S(v) ⊆ U(v), where Produced_scan(v) is the production the fixed forms observe inside the code-owned scan reach. Producers(V) contains 6 vocabularies: 5 kernel and 1 outside the kernel tier. The 20 cross_runtime vocabularies outside Producers(V) are unverified. Production outside the scan reach is unverified rather than proven closed.", + "statement": "∀v ∈ Producers(V): Produced_scan(v) ⊆ S(v) ⊆ U(v), where Produced_scan(v) is the production the fixed forms observe inside the code-owned scan reach. Producers(V) contains 6 vocabularies: 5 kernel and 1 outside the kernel tier. The 21 cross_runtime vocabularies outside Producers(V) are unverified. Production outside the scan reach is unverified rather than proven closed.", "enforcement": "m0_5", "evidence": "bounded production-form AST scan over the PRODUCER_ROOTS/PRODUCER_FILES reach in loopx/semantics/production.py; check_producers visits only vocabularies that declare producers, and validate_production claims no whole-program closedness; unresolved dynamic sites are reported, not treated as proven safe", "domain": { "quantifies_over": "vocabularies[producers].producers", "verified": 6, - "registered": 26, + "registered": 27, "evidence_bound": "producer_scan_reach" } }, { "id": "F2_canonical_value_liveness", - "statement": "∀v ∈ Producers(V): Canonical(v) ⊆ Produced_scan(v) ∪ CompatibilityOnly(v). Producers(V) contains 6 vocabularies: 5 kernel and 1 outside the kernel tier. The 20 cross_runtime vocabularies outside Producers(V) are unverified. Liveness outside the walked set is not proven.", + "statement": "∀v ∈ Producers(V): Canonical(v) ⊆ Produced_scan(v) ∪ CompatibilityOnly(v). Producers(V) contains 6 vocabularies: 5 kernel and 1 outside the kernel tier. The 21 cross_runtime vocabularies outside Producers(V) are unverified. Liveness outside the walked set is not proven.", "enforcement": "m0_5", "evidence": "every value of a vocabulary in Producers(V) has a producer recognised inside the scan reach, an executable input witness, or an explicit compatibility-only reason; a value whose only producer lies outside the reach would be reported as dead, not silently accepted", "domain": { "quantifies_over": "vocabularies[producers].producers", "verified": 6, - "registered": 26, + "registered": 27, "evidence_bound": "producer_scan_reach" } }, @@ -103,7 +103,7 @@ "domain": { "quantifies_over": "vocabularies[*]", "verified": 0, - "registered": 26, + "registered": 27, "evidence_bound": "inventory_only" } }, @@ -947,6 +947,25 @@ "control_repair": "Appended when the turn is routed to a control-plane repair instead of delivery: stall self-repair, a capability gate asking to repair the bridge, a workspace guard being present, or a required automation-prompt upgrade. The repair is control-plane work, so it displaces the delivery it would otherwise run alongside.", "delivery_not_allowed": "The catch-all appended last when normal delivery is off for a reason none of the four specific causes names, such as a monitor-only work mode, a settled receipt-bound monitor, a monitor-quiet skip or an external-evidence wait. Because it is appended last it only surfaces as the chosen reason when no more specific cause applies." } + }, + "state_recovery_component_state": { + "meaning": "Offline owner-local readability of a component in an inert recovery audit; never execution or adoption authority.", + "tier": "cross_runtime", + "status": "canonical", + "owners": { + "python": null, + "typescript": "loopx/control_plane/state_recovery_contract.ts::RECOVERY_COMPONENT_STATES" + }, + "values": [ + "readable", + "historical_only", + "unknown" + ], + "value_notes": { + "readable": "An existing configuration verifier or the selected File/SQLite authority and full Todo/history readers accepted the captured component. This is historical readability, not a current Goal identity, permission or execution grant.", + "historical_only": "A standalone SQLite copy passed its bounded offline structural integrity check, but its bytes have not been adopted by a current domain owner. Restored provider identities and leases remain historical data.", + "unknown": "No supported owner inventory exists for remaining v0 state, or a component failed format, availability, qualified-runtime or owner-read validation. Unknown never becomes readable merely because archive or file hashes matched." + } } }, "relations": { diff --git a/loopx/state_recovery.py b/loopx/state_recovery.py new file mode 100644 index 0000000000..8afa6fc959 --- /dev/null +++ b/loopx/state_recovery.py @@ -0,0 +1,442 @@ +"""Bounded tar/filesystem adapter for the typed, inert recovery audit owner. + +No restored Python, plugin, shell, credential or live registry is loaded. +Semantic manifest agreement and owner readback remain in TypeScript. +""" +from __future__ import annotations + +from contextlib import contextmanager +from collections.abc import Callable +import gzip +import hashlib +import io +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import subprocess +import tarfile +import tempfile +import time +import unicodedata +import zlib +from typing import Any, BinaryIO, Iterator, NoReturn + +from .control_plane.effect_runtime import _node_executable, effect_runtime_result, effect_runtime_request_scope + +MAX_ARCHIVE_BYTES = 128 * 1024 * 1024 +MAX_EXPANDED_BYTES = 512 * 1024 * 1024 +MAX_MANIFEST_BYTES = 16 * 1024 * 1024 +MAX_MEMBERS = 20_000 +MAX_EXPANSION_RATIO = 2000 +_CHUNK = 256 * 1024 +_OWNER_TIMEOUT_SECONDS = 90 + + +class StateRecoveryError(ValueError): + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +def _require(condition: bool, code: str) -> None: + if not condition: + raise StateRecoveryError(code) + + +def _json(data: bytes) -> dict[str, Any]: + def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + _require(key not in result, "manifest_duplicate_key") + result[key] = value + return result + + def constant(_value: str) -> None: + raise StateRecoveryError("manifest_invalid_json") + + try: + value = json.loads(data, object_pairs_hook=pairs, parse_constant=constant) + except (UnicodeError, json.JSONDecodeError, RecursionError) as exc: + raise StateRecoveryError("manifest_invalid_json") from exc + if not isinstance(value, dict): + raise StateRecoveryError("manifest_invalid_json") + pending = [(value, 0)] + while pending: + item, depth = pending.pop() + _require(depth <= 64, "recovery_capacity_exceeded") + if isinstance(item, dict): + pending.extend((child, depth + 1) for child in item.values()) + elif isinstance(item, list): + pending.extend((child, depth + 1) for child in item) + return value + + +def _runtime(method: str, request: dict[str, Any]) -> dict[str, Any]: + value = effect_runtime_result(method, request, timeout=90, + large_local_snapshot=method in {"state.recovery.archive.verify", "state.recovery.manifest.verify"}) + if not isinstance(value, dict): + raise StateRecoveryError("recovery_owner_result_invalid") + if value.get("ok") is not True: + code = value.get("error_code") + raise StateRecoveryError(code if isinstance(code, str) else "recovery_owner_result_invalid") + return value + + +def _audit_owners(request: dict[str, Any]) -> dict[str, Any]: + command = [_node_executable(), "--no-warnings", "--max-old-space-size=512", + "--experimental-sqlite", "--experimental-strip-types", + str(Path(__file__).parent / "control_plane/state_recovery_worker.ts")] + try: + completed = subprocess.run(command, input=json.dumps(request), text=True, encoding="utf-8", + capture_output=True, timeout=_OWNER_TIMEOUT_SECONDS, check=False) + except subprocess.TimeoutExpired as exc: + # subprocess.run kills and waits for this child only, never the daemon. + raise StateRecoveryError("recovery_owner_timeout") from exc + _require(bool(completed.stdout.strip()), "recovery_owner_unavailable") + _require(len(completed.stdout) <= 2 * 1024 * 1024, "recovery_owner_result_invalid") + result = _json(completed.stdout.encode()) + if result.get("ok") is not True: + raise StateRecoveryError(str(result.get("error_code") or "recovery_owner_unavailable")) + _require(completed.returncode == 0, "recovery_owner_unavailable") + return result + + +@contextmanager +def _regular_input(path: Path, limit: int) -> Iterator[BinaryIO]: + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except OSError as exc: + raise StateRecoveryError("recovery_input_unavailable") from exc + with os.fdopen(fd, "rb") as handle: + info = os.fstat(handle.fileno()) + _require(stat.S_ISREG(info.st_mode), "recovery_input_not_regular") + _require(info.st_uid == os.getuid() and not info.st_mode & 0o077, "recovery_input_not_private") + _require(0 < info.st_size <= limit, "recovery_capacity_exceeded") + yield handle + + +def _sha(handle: BinaryIO) -> tuple[str, int]: + handle.seek(0) + digest, count = hashlib.sha256(), 0 + while chunk := handle.read(_CHUNK): + count += len(chunk) + _require(count <= MAX_ARCHIVE_BYTES, "recovery_capacity_exceeded") + digest.update(chunk) + handle.seek(0) + return digest.hexdigest(), count + + +class _BoundedGzip(io.RawIOBase): + def __init__(self, handle: BinaryIO, archive_size: int): + super().__init__() + self.source = gzip.GzipFile(fileobj=handle, mode="rb") + self.limit = min(MAX_EXPANDED_BYTES, archive_size * MAX_EXPANSION_RATIO) + self.total = 0 + + def read(self, size: int = -1) -> bytes: + _require(size >= 0, "archive_unbounded_read") + data = self.source.read(min(size, self.limit - self.total + 1)) + self.total += len(data) + _require(self.total <= self.limit, "recovery_capacity_exceeded") + return data + + +class _RecoveryTarInfo(tarfile.TarInfo): + def _reject_sparse(self, *_args: object) -> NoReturn: + raise StateRecoveryError("archive_sparse_unsupported") + + # The stdlib invokes these hooks before yielding the final member. Refuse + # unsupported extents before their metadata/body is decoded or expanded. + _proc_gnusparse_00 = _reject_sparse + _proc_gnusparse_01 = _reject_sparse + _proc_gnusparse_10 = _reject_sparse + + def _proc_member(self, archive: tarfile.TarFile) -> tarfile.TarInfo | None: + _require(self.type != tarfile.GNUTYPE_SPARSE, "archive_sparse_unsupported") + # Bound extended metadata before tarfile allocates/decodes its body. + if self.type in (tarfile.XHDTYPE, tarfile.XGLTYPE, tarfile.SOLARIS_XHDTYPE, + tarfile.GNUTYPE_LONGNAME, tarfile.GNUTYPE_LONGLINK): + _require(self.size <= 128 * 1024, "archive_metadata_too_large") + if isinstance(archive, _RecoveryTarFile): + archive.recovery_metadata_bytes += self.size + _require(archive.recovery_metadata_bytes <= MAX_MANIFEST_BYTES, "archive_metadata_too_large") + # The stdlib hook is absent from typeshed; keep its real bound signature + # here rather than weakening the public stream/return types. + process: Callable[[tarfile.TarFile], tarfile.TarInfo | None] = getattr(super(), "_proc_member") + return process(archive) + + +class _RecoveryTarFile(tarfile.TarFile): + recovery_metadata_bytes = 0 + + +def _member_name(raw: str) -> str: + _require(bool(raw) and len(raw) <= 1024 and "\\" not in raw and "\x00" not in raw, + "archive_path_invalid") + path = PurePosixPath(raw) + _require(bool(path.parts) and len(path.parts) <= 64 and not path.is_absolute() + and ".." not in path.parts and ":" not in path.parts[0] + and all(len(part.encode("utf-8")) <= 255 for part in path.parts), + "archive_path_invalid") + normalized = str(path) + _require(normalized not in ("", "."), "archive_path_invalid") + return normalized + + +def _physical_ancestors(destination: Path) -> None: + _require(destination.is_absolute() and str(destination) == os.path.normpath(destination), + "recovery_destination_invalid") + for parent in destination.parents: + try: + info = parent.lstat() + except OSError as exc: + raise StateRecoveryError("recovery_destination_parent_unavailable") from exc + _require(stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode), + "recovery_destination_parent_not_physical") + + +def _physical_parent(destination: Path) -> None: + _physical_ancestors(destination) + _require(not destination.exists() and not destination.is_symlink(), "recovery_destination_exists") + + +def _directories(root: Path, parts: tuple[str, ...]) -> Path: + current = root + for part in parts: + current /= part + current.mkdir(mode=0o700, exist_ok=True) + _require(current.is_dir() and not current.is_symlink(), "archive_parent_not_directory") + return current + + +def _check_links(links: list[tuple[str, str]]) -> None: + by_name = {unicodedata.normalize("NFC", name).casefold(): target for name, target in links} + for name, target in links: + pending = list(PurePosixPath(name).parent.parts) + list(PurePosixPath(target).parts) + resolved: list[str] = [] + hops = 0 + while pending: + part = pending.pop(0) + if part == "..": + _require(bool(resolved), "archive_link_escapes") + resolved.pop() + elif part != ".": + resolved.append(part) + key = unicodedata.normalize("NFC", "/".join(resolved)).casefold() + if key in by_name: + hops += 1 + _require(hops <= 40, "archive_link_cycle") + resolved.pop() + pending = list(PurePosixPath(by_name[key]).parts) + pending + + +def _scan(handle: BinaryIO, archive_size: int, output: Path | None = None) -> dict[str, Any]: + handle.seek(0) + stream = _BoundedGzip(handle, archive_size) + names: dict[str, str] = {} + folded: dict[str, str] = {} + ancestors: set[str] = set() + links: list[tuple[str, str]] = [] + files, sqlite_members, standalone = [], [], [] + embedded: dict[str, Any] | None = None + logical = 0 + try: + with _RecoveryTarFile.open(fileobj=stream, mode="r|", tarinfo=_RecoveryTarInfo) as archive: + for member in archive: + name = _member_name(member.name) + key = unicodedata.normalize("NFC", name).casefold() + _require(name not in names and key not in folded, "archive_name_collision") + _require(len(names) < MAX_MEMBERS, "recovery_capacity_exceeded") + _require(member.sparse is None, "archive_sparse_unsupported") + kind = "file" if member.isfile() else "directory" if member.isdir() else "symlink" if member.issym() else None + _require(kind is not None, "archive_member_type_unsupported") + names[name], folded[key] = str(kind), name + for parent in PurePosixPath(name).parents: + parent_key = unicodedata.normalize("NFC", str(parent)).casefold() + ancestors.add(parent_key) + _require(names.get(folded.get(parent_key, ""), "directory") == "directory", + "archive_parent_not_directory") + if kind != "directory": + _require(key not in ancestors, "archive_parent_not_directory") + if kind == "symlink": + target = member.linkname + _require(bool(target) and len(target) <= 1024 and "\x00" not in target + and not PurePosixPath(target).is_absolute() and "\\" not in target, + "archive_link_escapes") + resolved = os.path.normpath(str(PurePosixPath(name).parent / target)) + _require(resolved != ".." and not resolved.startswith("../"), "archive_link_escapes") + links.append((name, target)) + continue + if output is not None: + _directories(output, PurePosixPath(name).parts if kind == "directory" else PurePosixPath(name).parts[:-1]) + if kind == "directory": + continue + logical += member.size + _require(0 <= member.size <= MAX_EXPANDED_BYTES and logical <= MAX_EXPANDED_BYTES, + "recovery_capacity_exceeded") + if name == "manifest.json": + _require(member.size <= MAX_MANIFEST_BYTES, "archive_metadata_too_large") + source = archive.extractfile(member) + if source is None: + raise StateRecoveryError("archive_member_unreadable") + data, prefix = bytearray(), bytearray() + digest, size = hashlib.sha256(), 0 + destination = None + try: + if output is not None: + destination = open(output / name, "xb") + os.chmod(output / name, 0o600) + while chunk := source.read(_CHUNK): + size += len(chunk) + _require(size <= member.size, "archive_member_size_mismatch") + digest.update(chunk) + prefix.extend(chunk[:max(0, 100 - len(prefix))]) + if name == "manifest.json": + data.extend(chunk) + if destination is not None: + destination.write(chunk) + _require(size == member.size, "archive_member_size_mismatch") + if destination is not None: + destination.flush() + os.fsync(destination.fileno()) + finally: + source.close() + if destination is not None: + destination.close() + if name == "manifest.json": + embedded = _json(bytes(data)) + else: + files.append({"archive_path": name, "sha256": digest.hexdigest(), "size_bytes": size}) + if prefix[:16] == b"SQLite format 3\0": + sqlite_members.append(name) + if prefix[18:20] == b"\x01\x01": + standalone.append(name) + # tar EOF must not conceal an appended archive or unparsed payload. + while trailing := archive.fileobj.read(_CHUNK): + _require(not trailing.strip(b"\0"), "archive_trailing_data") + # Consume gzip EOF too: truncated CRC/trailing expansion is not a valid tar. + while stream.read(_CHUNK): + pass + except StateRecoveryError: + raise + except (tarfile.TarError, EOFError, gzip.BadGzipFile, OSError, zlib.error, + ValueError, OverflowError, KeyError, RecursionError) as exc: + raise StateRecoveryError("archive_unreadable") from exc + finally: + stream.source.close() + _require(embedded is not None, "embedded_manifest_missing") + _check_links(links) + # Links are materialized last and no member may live beneath a link. + if output is not None: + for name, target in links: + _directories(output, PurePosixPath(name).parts[:-1]) + os.symlink(target, output / name) + return {"embedded_manifest": embedded, "files": files, "logical_bytes": logical, + "sqlite_members": sqlite_members, "standalone_sqlite_members": standalone, + "entry_count": len(names)} + + +@effect_runtime_request_scope() +def verify_state_backup( + *, archive: Path, manifest: Path, destination: Path, execute: bool = False, +) -> dict[str, Any]: + """Preview or publish one inert audit. This API has no live-adoption operation.""" + started = time.monotonic_ns() + _require(os.name == "posix", "recovery_platform_unqualified") + _physical_parent(destination) + with _regular_input(manifest, MAX_MANIFEST_BYTES) as manifest_handle: + manifest_bytes = manifest_handle.read(MAX_MANIFEST_BYTES + 1) + _require(len(manifest_bytes) <= MAX_MANIFEST_BYTES, "recovery_capacity_exceeded") + external = _json(manifest_bytes) + with _regular_input(archive, MAX_ARCHIVE_BYTES) as archive_handle: + archive_sha, archive_size = _sha(archive_handle) + _runtime("state.recovery.archive.verify", {"external_manifest": external, + "archive_sha256": archive_sha, "archive_size_bytes": archive_size}) + observation = _scan(archive_handle, archive_size) + verified = _runtime("state.recovery.manifest.verify", { + "external_manifest": external, "embedded_manifest": observation["embedded_manifest"], + "archive_sha256": archive_sha, "archive_size_bytes": archive_size, + "external_manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + "files": observation["files"], + "sqlite_member_count": len(observation["sqlite_members"]), + }) + _require(shutil.disk_usage(destination.parent).free >= observation["logical_bytes"] + MAX_MANIFEST_BYTES, + "recovery_disk_capacity") + if not execute: + return {**verified, "status": "planned", "written": False, + "logical_bytes": observation["logical_bytes"], "entry_count": observation["entry_count"], + "archive_processing_ms": (time.monotonic_ns() - started) // 1_000_000, + "activation_eligible": False} + staging = Path(tempfile.mkdtemp(prefix=".loopx-recovery-", dir=destination.parent)) + reserved = False + try: + data = staging / "data" + data.mkdir(mode=0o700) + copied = _scan(archive_handle, archive_size, data) + _require(copied == observation and _sha(archive_handle) == (archive_sha, archive_size), + "recovery_input_changed") + audit = _audit_owners({"data_root": str(data), "verified": verified, + "measurements": {"archive_bytes": archive_size, + "regular_file_bytes": copied["logical_bytes"], + "archive_processing_ms": (time.monotonic_ns() - started) // 1_000_000}, + "sqlite_members": observation["sqlite_members"], + "standalone_sqlite_members": observation["standalone_sqlite_members"]}) + audit = _runtime("state.recovery.audit.readback", audit) + for member in observation["files"]: + with (data / member["archive_path"]).open("rb") as handle: + digest = hashlib.file_digest(handle, "sha256").hexdigest() + _require(digest == member["sha256"], "recovery_owner_mutated_source") + audit_path = staging / "recovery-audit.json" + with audit_path.open("x", encoding="utf-8") as handle: + os.chmod(audit_path, 0o600) + json.dump(audit, handle, ensure_ascii=False, sort_keys=True, indent=2) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + _require(_json(audit_path.read_bytes()) == audit, "recovery_audit_readback_mismatch") + _physical_parent(destination) + destination.mkdir(mode=0o700) + reserved = True + os.rename(staging, destination) + reserved = False + return read_state_recovery_audit(destination) + finally: + try: + if reserved: + # Never recursively remove a destination occupied meanwhile. + destination.rmdir() + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def read_state_recovery_audit(destination: Path) -> dict[str, Any]: + path = destination / "recovery-audit.json" + _physical_ancestors(path) + with _regular_input(path, MAX_MANIFEST_BYTES) as handle: + audit = _json(handle.read(MAX_MANIFEST_BYTES + 1)) + return _runtime("state.recovery.audit.readback", audit) + + +def render_state_recovery_markdown(payload: dict[str, Any]) -> str: + if payload.get("ok") is not True: + return f"Backup inspection failed: {payload.get('error_code', 'recovery_unavailable')}. No execution authority was granted." + if payload.get("status") == "planned": + return "\n".join([ + "Archive and manifest checks passed. No copy has been created and owner records have not been inspected.", + f"Saved files: {payload['file_count']}; logical bytes: {payload['logical_bytes']}", + "Repeat with --execute to create and inspect a new inert copy; this never activates a Goal.", + ]) + summary = payload.get("summary", {}) + lines = [ + f"Backup inspection: {payload['status']}", + f"Recovery: {payload['recovery_id']}", + f"Readable: {summary.get('readable', 0)}; historical only: {summary.get('historical_only', 0)}; unknown: {summary.get('unknown', 0)}", + "The copy is inert. Goal execution and live settings were not activated.", + ] + measurements = payload.get("measurements") + if isinstance(measurements, dict): + lines.append(f"Archive: {measurements['archive_bytes']} bytes; regular files: {measurements['regular_file_bytes']} bytes; " + f"archive checks/copy: {measurements['archive_processing_ms']} ms; owner reads: {measurements['owner_read_ms']} ms.") + return "\n".join(lines) diff --git a/tests/control_plane_ts/content_digest_single_owner.test.ts b/tests/control_plane_ts/content_digest_single_owner.test.ts index a538ad830b..24dba6e077 100644 --- a/tests/control_plane_ts/content_digest_single_owner.test.ts +++ b/tests/control_plane_ts/content_digest_single_owner.test.ts @@ -133,6 +133,7 @@ const CANONICAL_CONSUMERS = [ "control_plane/runtime/usage_statistics_cycles.ts", "control_plane/runtime/usage_statistics_goal_contract.ts", "control_plane/runtime/usage_statistics_goals.ts", + "control_plane/state_recovery_contract.ts", "control_plane/todos/completion_transaction.ts", "control_plane/todos/completion_validation_revision.ts", "control_plane/turn_driver/chat_turn_acceptance.ts", diff --git a/tests/control_plane_ts/state_recovery.test.ts b/tests/control_plane_ts/state_recovery.test.ts new file mode 100644 index 0000000000..f8ecc23265 --- /dev/null +++ b/tests/control_plane_ts/state_recovery.test.ts @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import {canonicalAuthoritySha256} from "../../loopx/control_plane/coordination/authority_store_codec.ts"; +import {createEffectRuntimeHandlers, dispatchEffectRuntimeMethod} from "../../loopx/control_plane/effect_runtime_handlers.ts"; +import {verifyStateRecoveryArchive, verifyStateRecoveryManifest} from "../../loopx/control_plane/state_recovery.ts"; +import {decodeStateRecoveryAudit, STATE_RECOVERY_AUDIT_SCHEMA} from "../../loopx/control_plane/state_recovery_contract.ts"; + +const member = {archive_path: "runtime-root/retained", size_bytes: 1, sha256: "c".repeat(64)}; +function request() { + const external = {schema_version: "loopx_state_backup_v0", ok: true, dry_run: false, + execute_requested: true, backup_id: "fixture", private_hint: "private-marker", + execution: {archive_sha256: "a".repeat(64), archive_size_bytes: 500, + file_members: [member], archive_to_logical_ratio: 0.5}}; + const embedded = structuredClone(external) as Record; + embedded.execution.archive_sha256 = embedded.execution.archive_size_bytes = null; + delete embedded.execution.archive_to_logical_ratio; + return {external_manifest: external, embedded_manifest: embedded, files: [member], + archive_sha256: "a".repeat(64), archive_size_bytes: 500, external_manifest_sha256: "b".repeat(64), + sqlite_member_count: 0}; +} + +test("archive and manifest checks bind observed bytes without projecting private capture metadata", () => { + const input = request(), before = structuredClone(input); + const first = verifyStateRecoveryArchive(input); + assert.equal(first.ok, true); + const verified = verifyStateRecoveryManifest(input); + assert.equal(verified.ok, true); + assert.equal(verified.file_witnesses_verified, true); + assert.equal(verified.manifest_verified, true); + assert.equal(verified.execution_authority_granted, false); + assert.equal(JSON.stringify(verified).includes("private-marker"), false); + assert.deepEqual(input, before); + assert.equal(verifyStateRecoveryArchive({...input, archive_sha256: "d".repeat(64)}).error_code, "archive_digest_mismatch"); + assert.equal(verifyStateRecoveryManifest({...input, files: [{...member, size_bytes: 2}]}).error_code, "member_witness_mismatch"); + assert.equal(verifyStateRecoveryManifest({...input, embedded_manifest: {...input.embedded_manifest, backup_id: "other"}}).error_code, "manifest_disagreement"); + assert.equal(verifyStateRecoveryManifest({...input, sqlite_member_count: 129}).error_code, "sqlite_inventory_capacity"); +}); + +test("legacy absent member witnesses stay unqualified rather than becoming verified by an empty list", () => { + const input = request(); + delete (input.external_manifest.execution as Record).file_members; + delete input.embedded_manifest.execution.file_members; + const result = verifyStateRecoveryManifest(input); + assert.equal(result.ok, true); + assert.equal(result.file_witnesses_verified, false); + assert.equal(result.capture_profile, "legacy_best_effort"); +}); + +function audit() { + const body = {ok: true, schema_version: STATE_RECOVERY_AUDIT_SCHEMA, + recovery_id: "e".repeat(64), status: "held", written: true, manifest_verified: true, + observed_at: "2026-10-12T00:00:00.000Z", execution_authority_granted: false, activation_eligible: false, + backup: {schema_version: "loopx_state_backup_v0", archive_sha256: "a".repeat(64), external_manifest_sha256: "b".repeat(64)}, + workspace: {isolation: "inert", registered: false}, + components: [{component_id: "configuration", owner_kind: "configuration", state: "readable", + reason_code: "configuration_digest_verified", next_action: "retain_isolated_configuration", facts: {}, dependency_ids: []}], + summary: {readable: 1, historical_only: 0, unknown: 0}, findings: []}; + return {...body, audit_sha256: canonicalAuthoritySha256(body)}; +} + +test("the packaged client and owner reject permission, unknown-state and summary contradictions", () => { + const input = audit(); + assert.doesNotThrow(() => decodeStateRecoveryAudit(input)); + for (const delta of [{execution_authority_granted: true}, {activation_eligible: true}, + {workspace: {isolation: "inert", registered: true}}, {summary: {...input.summary, readable: 2}}]) { + assert.throws(() => decodeStateRecoveryAudit({...input, ...delta})); + } + assert.throws(() => decodeStateRecoveryAudit({...input, components: [{...input.components[0], state: "active"}]})); + assert.throws(() => decodeStateRecoveryAudit({...input, components: [{...input.components[0], dependency_ids: ["missing"]}]})); +}); + +test("untrusted owner inspection has no shared-daemon dispatch", async () => { + const handlers = createEffectRuntimeHandlers({fingerprint: "recovery-isolation", requestShutdown() {}}); + await assert.rejects(() => dispatchEffectRuntimeMethod(handlers, "state.recovery.owners.audit", {}), /unsupported Effect runtime method/); +}); + +test("recorded measurements reject invalid values and absent historical measurements remain absent", () => { + const input = audit(); + const measurements = {archive_bytes: 500, regular_file_bytes: 256, archive_processing_ms: 3, owner_read_ms: 2}; + assert.equal(decodeStateRecoveryAudit(input).measurements, undefined); + assert.deepEqual(decodeStateRecoveryAudit({...input, measurements}).measurements, measurements); + for (const field of Object.keys(measurements)) { + for (const value of [-1, 1.5, "3", null, Number.NaN, Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => decodeStateRecoveryAudit({...input, measurements: {...measurements, [field]: value}})); + } + } +}); diff --git a/tests/test_state_recovery.py b/tests/test_state_recovery.py new file mode 100644 index 0000000000..ea9fceeb54 --- /dev/null +++ b/tests/test_state_recovery.py @@ -0,0 +1,481 @@ +from __future__ import annotations + +import hashlib +import gzip +import io +import http.client +import json +import os +from pathlib import Path +import subprocess +import sqlite3 +import sys +import tarfile +import threading + +import pytest + +from loopx.state_backup import build_state_backup_plan, execute_state_backup_plan +from loopx.state_recovery import StateRecoveryError, read_state_recovery_audit, render_state_recovery_markdown, verify_state_backup +from tests.control_plane.canonical_authority_fixture import promoted_create_fixture + + +def _cli(*args: object) -> tuple[int, dict]: + result = subprocess.run([sys.executable, "-m", "loopx.cli", "--format", "json", *map(str, args)], + capture_output=True, text=True, env={**os.environ, "LOOPX_USAGE_PING": "0"}, timeout=90) + return result.returncode, json.loads(result.stdout) + + +def _backup(root: Path, provider: str | None = None) -> tuple[Path, Path]: + if provider: + registry, runtime, state = promoted_create_fixture(root, provider=provider) + project = state.parents[3] + code, result = _cli("--registry", registry, "--runtime-root", runtime, "todo", "add", + "--goal-id", "goal-a", "--role", "agent", "--claimed-by", "agent-a", + "--operation-id", "retained-work", "--text", "Keep this unfinished commitment") + assert code == 0, result + state.write_text(state.read_text() + "\n## Original commitments\nPreserve the full Goal, including unfinished acceptance.\n") + else: + project, runtime = root / "project", root / "runtime" + project.mkdir() + runtime.mkdir() + registry = root / "registry.json" + registry.write_text(json.dumps({"goals": []})) + (runtime / "retained.txt").write_text("saved data") + result = execute_state_backup_plan(build_state_backup_plan(project=project, runtime_root=runtime, + registry_path=registry, output_dir=root / "backups", backup_id="source", include_registry_projects=False, + include_automations=False, include_skills=False)) + return Path(result["archive_path"]), Path(result["manifest_path"]) + + +def _rewrite(archive: Path, manifest: Path, extra: tarfile.TarInfo | None = None, + body: bytes = b"", replacements: dict[str, bytes] | None = None) -> None: + with tarfile.open(archive, "r:gz") as source: + records = [(row, source.extractfile(row).read() if row.isfile() else None) for row in source] + with tarfile.open(archive, "w:gz") as target: + for row, data in records: + if replacements and row.name in replacements: + data = replacements[row.name] + row.size = len(data) + target.addfile(row, io.BytesIO(data) if data is not None else None) + if extra is not None: + target.addfile(extra, io.BytesIO(body) if extra.isfile() else None) + data = json.loads(manifest.read_text()) + data["execution"]["archive_sha256"] = hashlib.sha256(archive.read_bytes()).hexdigest() + data["execution"]["archive_size_bytes"] = archive.stat().st_size + manifest.write_text(json.dumps(data)) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_real_backup_recovers_readable_authority_after_original_sources_disappear(tmp_path, provider): + archive, manifest = _backup(tmp_path, provider) + (tmp_path / "project").rename(tmp_path / "unavailable-project") + (tmp_path / "runtime").rename(tmp_path / "unavailable-runtime") + (tmp_path / "registry.json").rename(tmp_path / "unavailable-registry.json") + original = archive.read_bytes(), manifest.read_bytes() + destination = tmp_path / "inert" + preview = verify_state_backup(archive=archive, manifest=manifest, destination=destination) + assert preview["status"] == "planned" and preview["written"] is False + preview_text = render_state_recovery_markdown(preview) + assert "owner records have not been inspected" in preview_text and "Readable: 0" not in preview_text + assert not destination.exists() + code, audit = _cli("backup-state", "verify", "--archive", archive, "--manifest", manifest, + "--destination", destination, "--execute") + assert code == 0, audit + assert audit["execution_authority_granted"] is False and audit["activation_eligible"] is False + assert audit["workspace"] == {"isolation": "inert", "registered": False} + assert audit["status"] == "held" + assert audit["measurements"]["archive_bytes"] == archive.stat().st_size + assert audit["measurements"]["regular_file_bytes"] == preview["logical_bytes"] + assert all(type(value) is int and value >= 0 for value in audit["measurements"].values()) + assert read_state_recovery_audit(destination)["measurements"] == audit["measurements"] + assert "archive checks/copy:" in render_state_recovery_markdown(audit) + [goal] = [row for row in audit["components"] if row["owner_kind"] == "coordination_authority"] + assert goal["state"] == "readable", goal + assert goal["facts"]["todo_count"] == 1 + assert goal["facts"]["history_commits"] == 2 + assert goal["facts"]["provider"] == provider + captures = json.loads(manifest.read_text())["included"] + state_path = next(row["archive_path"] for row in captures if row["key"] == "registry_active_state:goal-a") + assert "Preserve the full Goal, including unfinished acceptance." in (destination / "data" / state_path).read_text() + assert (archive.read_bytes(), manifest.read_bytes()) == original + assert json.loads((destination / "recovery-audit.json").read_text()) == audit + assert str(tmp_path) not in json.dumps(audit) + assert not (destination / "registry.global.json").exists() + for path in [destination, destination / "data", destination / "recovery-audit.json"]: + assert path.stat().st_mode & 0o077 == 0 + before = (destination / "recovery-audit.json").read_bytes() + with pytest.raises(StateRecoveryError, match="recovery_destination_exists"): + verify_state_backup(archive=archive, manifest=manifest, destination=destination, execute=True) + assert (destination / "recovery-audit.json").read_bytes() == before + + +def test_valid_internal_symlink_stays_inert_and_chained_escape_is_refused(tmp_path): + archive, manifest = _backup(tmp_path) + link = tarfile.TarInfo("readable-link") + link.type, link.linkname = tarfile.SYMTYPE, "runtime-root/retained.txt" + _rewrite(archive, manifest, link) + result = verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "valid", execute=True) + assert result["execution_authority_granted"] is False + assert (tmp_path / "valid/data/readable-link").is_symlink() + alias = tarfile.TarInfo("alias") + alias.type, alias.linkname = tarfile.SYMTYPE, "." + _rewrite(archive, manifest, alias) + escape = tarfile.TarInfo("escape-link") + escape.type, escape.linkname = tarfile.SYMTYPE, "alias/../outside" + _rewrite(archive, manifest, escape) + with pytest.raises(StateRecoveryError, match="archive_link_escapes"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "invalid", execute=True) + assert not (tmp_path / "invalid").exists() + + +def test_capacity_and_private_input_fail_before_publication(tmp_path, monkeypatch): + import loopx.state_recovery as recovery + + archive, manifest = _backup(tmp_path) + original = archive.stat().st_mode + archive.chmod(0o644) + with pytest.raises(StateRecoveryError, match="input_not_private"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "invalid", execute=True) + archive.chmod(original) + monkeypatch.setattr(recovery, "MAX_MEMBERS", 1) + with pytest.raises(StateRecoveryError, match="capacity_exceeded"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "invalid", execute=True) + assert not (tmp_path / "invalid").exists() + + +def test_large_manifest_uses_existing_private_snapshot_transport(tmp_path): + archive, manifest = _backup(tmp_path) + external = json.loads(manifest.read_text()) + with tarfile.open(archive) as source: + embedded = json.load(source.extractfile("manifest.json")) + # Complete metadata exceeds the ordinary 2 MiB RPC; no wire ceiling is raised. + external["warnings"] = embedded["warnings"] = ["synthetic warning " * 150000] + manifest.write_text(json.dumps(external)) + _rewrite(archive, manifest, replacements={"manifest.json": json.dumps(embedded).encode()}) + result = verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "inert") + assert result["manifest_verified"] is True and not result["written"] + + +def test_changed_audit_and_true_execution_flag_are_not_trusted_on_readback(tmp_path): + archive, manifest = _backup(tmp_path) + destination = tmp_path / "inert" + result = verify_state_backup(archive=archive, manifest=manifest, destination=destination, execute=True) + path = destination / "recovery-audit.json" + altered_measurement = {**result, "measurements": {**result["measurements"], + "owner_read_ms": result["measurements"]["owner_read_ms"] + 1}} + path.write_text(json.dumps(altered_measurement)) + with pytest.raises(StateRecoveryError, match="audit_digest_mismatch"): + read_state_recovery_audit(destination) + changed = {**result, "observed_at": "2020-01-01T00:00:00.000Z"} + path.write_text(json.dumps(changed)) + with pytest.raises(StateRecoveryError, match="audit_digest_mismatch"): + read_state_recovery_audit(destination) + changed["execution_authority_granted"] = True + path.write_text(json.dumps(changed)) + with pytest.raises(StateRecoveryError, match="audit_invalid"): + read_state_recovery_audit(destination) + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_missing_selected_storage_is_unknown_without_creation_or_fallback(tmp_path, provider): + archive, manifest = _backup(tmp_path, provider) + destination = tmp_path / "inert" + original = verify_state_backup(archive=archive, manifest=manifest, destination=destination, execute=True) + from loopx.state_recovery import _audit_owners + + selected = destination / "data/runtime-root/authority" / f"{provider}-v0" + sqlite_files = [path.relative_to(destination / "data").as_posix() for path in selected.glob("*.sqlite")] + selected.rename(tmp_path / "unavailable-store") + observed = _audit_owners({"data_root": str(destination / "data"), + "measurements": original["measurements"], + "verified": {key: original[key] for key in ("ok", "schema_version", "recovery_id", "backup", "file_count", + "file_witnesses_verified", "manifest_verified", "capture_profile", "execution_authority_granted")}, + "sqlite_members": sqlite_files, "standalone_sqlite_members": sqlite_files}) + [goal] = [row for row in observed["components"] if row["owner_kind"] == "coordination_authority"] + assert goal["state"] == "unknown" + assert observed["execution_authority_granted"] is False + assert not selected.exists() + if provider == "sqlite": + assert not (destination / "data/runtime-root/authority/file-v0").exists() + + +def test_restored_code_remains_unclassified_data_and_is_never_loaded(tmp_path): + archive, manifest = _backup(tmp_path) + # Re-capture through the real producer so the hostile file has valid witnesses. + marker = tmp_path / "executed" + (tmp_path / "runtime/untrusted_plugin.py").write_text(f"from pathlib import Path\nPath({str(marker)!r}).touch()\n") + result = execute_state_backup_plan(build_state_backup_plan(project=tmp_path / "project", runtime_root=tmp_path / "runtime", + registry_path=tmp_path / "registry.json", output_dir=tmp_path / "with-code", backup_id="code", + include_registry_projects=False, include_automations=False, include_skills=False)) + audit = verify_state_backup(archive=Path(result["archive_path"]), manifest=Path(result["manifest_path"]), + destination=tmp_path / "inert-code", execute=True) + assert not marker.exists() + [remaining] = [row for row in audit["components"] if row["component_id"] == "remaining_state"] + assert remaining["state"] == "unknown" and audit["execution_authority_granted"] is False + + +def test_sqlite_views_are_never_evaluated_as_authority_tables(tmp_path): + archive, manifest = _backup(tmp_path, "sqlite") + with tarfile.open(archive) as source: + database_member = next(row.name for row in source if row.name.endswith(".sqlite")) + database_bytes = source.extractfile(database_member).read() + embedded = json.load(source.extractfile("manifest.json")) + database_path = tmp_path / "view.sqlite" + database_path.write_bytes(database_bytes) + with sqlite3.connect(database_path) as database: + database.executescript("ALTER TABLE metadata RENAME TO saved_metadata; " + "CREATE VIEW metadata AS WITH RECURSIVE spin(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM spin) " + "SELECT *, (SELECT count(*) FROM spin) AS expensive FROM saved_metadata;") + changed = database_path.read_bytes() + external = json.loads(manifest.read_text()) + for envelope in (external, embedded): + witness = next(row for row in envelope["execution"]["file_members"] if row["archive_path"] == database_member) + witness.update(sha256=hashlib.sha256(changed).hexdigest(), size_bytes=len(changed)) + manifest.write_text(json.dumps(external)) + _rewrite(archive, manifest, replacements={database_member: changed, "manifest.json": json.dumps(embedded).encode()}) + result = verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "view-inert", execute=True) + [goal] = [row for row in result["components"] if row["owner_kind"] == "coordination_authority"] + assert goal["state"] == "unknown" + assert goal["reason_code"] == "sqlite_requires_owner_recovery" + + +def test_offline_worker_timeout_does_not_leave_a_copy_or_stop_shared_runtime(tmp_path, monkeypatch): + import loopx.state_recovery as recovery + from loopx.control_plane.effect_runtime import effect_runtime_result + + archive, manifest = _backup(tmp_path) + monkeypatch.setattr(recovery, "_OWNER_TIMEOUT_SECONDS", 0.001) + with pytest.raises(StateRecoveryError, match="recovery_owner_timeout"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "timed-out", execute=True) + assert not (tmp_path / "timed-out").exists() + assert not list(tmp_path.glob(".loopx-recovery-*")) + # This is still the real shared helper, not a fabricated availability result. + normal = effect_runtime_result("configuration.backup", {"action": "capture", "data": {"machine_configuration": None, "goals": []}}) + assert normal["activation_performed"] is False + + +def test_real_sqlite_history_limit_is_complete_and_overflow_refuses_publication(tmp_path): + registry, runtime, state = promoted_create_fixture(tmp_path, provider="sqlite") + project = state.parents[3] + module = Path(__file__).parents[1] / "loopx/control_plane/coordination/local_authority_provider.ts" + script = f"import {{openLocalAuthorityStore}} from {json.dumps(module.as_uri())};" + """ +let input='';for await(const chunk of process.stdin)input+=chunk;const r=JSON.parse(input); +const store=await openLocalAuthorityStore(r.root,'goal-a');const head=await store.loadAuthority(); +if(head.status!=='loaded')throw new Error(JSON.stringify(head));let revision=head.provider_revision; +for(let i=r.first;i<=r.last;i++){ + const result=await store.commitAuthority({expected_provider_revision:revision,operation_id:`history-capacity-${i}`, + events:[],receipts:[],next_projection:head.head}); + if(result.status!=='applied')throw new Error(JSON.stringify(result));revision=result.provider_revision; +} +""" + for first, last, expected in [(2, 20000, "readable"), (20001, 20001, "refused")]: + subprocess.run(["node", "--no-warnings", "--experimental-sqlite", "--experimental-strip-types", + "--input-type=module", "-e", script], input=json.dumps({"root": str(runtime), "first": first, "last": last}), + text=True, capture_output=True, check=True, timeout=120) + backup = execute_state_backup_plan(build_state_backup_plan(project=project, runtime_root=runtime, + registry_path=registry, output_dir=tmp_path / "backups", backup_id=f"history-{last}", + include_registry_projects=False, include_automations=False, include_skills=False)) + destination = tmp_path / f"inert-{last}" + code, result = _cli("backup-state", "verify", "--archive", backup["archive_path"], + "--manifest", backup["manifest_path"], "--destination", destination, "--execute") + if expected == "readable": + assert code == 0, result + [goal] = [row for row in result["components"] if row["owner_kind"] == "coordination_authority"] + assert goal["state"] == "readable" and goal["facts"]["history_commits"] == 20000 + else: + assert code == 1 and result["error_code"] == "authority_history_capacity", result + assert not destination.exists() + assert not list(tmp_path.glob(".loopx-recovery-*")) + + +def test_digest_refusal_precedes_tar_decoding_and_writes_nothing(tmp_path): + archive, manifest = _backup(tmp_path) + archive.write_bytes(b"not a tar archive") + with pytest.raises(StateRecoveryError, match="archive_digest_mismatch"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "inert", execute=True) + assert not (tmp_path / "inert").exists() + assert not list(tmp_path.glob(".loopx-recovery-*")) + + +def test_external_manifest_cannot_relabel_the_embedded_capture(tmp_path): + archive, manifest = _backup(tmp_path) + data = json.loads(manifest.read_text()) + data["backup_id"] = "different-backup" + manifest.write_text(json.dumps(data)) + with pytest.raises(StateRecoveryError, match="manifest_disagreement"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "inert", execute=True) + assert not (tmp_path / "inert").exists() + + +def test_recomputed_archive_digest_does_not_hide_member_corruption(tmp_path): + archive, manifest = _backup(tmp_path) + _rewrite(archive, manifest, replacements={"runtime-root/retained.txt": b"altered data"}) + with pytest.raises(StateRecoveryError, match="member_witness_mismatch"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "inert", execute=True) + + +@pytest.mark.parametrize("name,kind,target,reason", [ + ("../escape", tarfile.REGTYPE, "", "archive_path_invalid"), + ("/absolute", tarfile.REGTYPE, "", "archive_path_invalid"), + ("runtime-root/RETAINED.txt", tarfile.REGTYPE, "", "archive_name_collision"), + ("runtime-root/retained.txt", tarfile.REGTYPE, "", "archive_name_collision"), + ("fifo", tarfile.FIFOTYPE, "", "archive_member_type_unsupported"), + ("hardlink", tarfile.LNKTYPE, "runtime-root/retained.txt", "archive_member_type_unsupported"), + ("outside", tarfile.SYMTYPE, "../escape", "archive_link_escapes"), + ("runtime-root", tarfile.SYMTYPE, "project", "archive_name_collision"), +]) +def test_archive_paths_and_member_types_fail_before_destination_creation(tmp_path, name, kind, target, reason): + archive, manifest = _backup(tmp_path) + member = tarfile.TarInfo(name) + member.type, member.linkname = kind, target + _rewrite(archive, manifest, member) + with pytest.raises(StateRecoveryError, match=reason): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "inert", execute=True) + assert not (tmp_path / "inert").exists() + assert not (tmp_path.parent / "escape").exists() + + +def test_large_pax_metadata_is_refused_before_tarfile_decodes_its_body(tmp_path): + archive, manifest = _backup(tmp_path) + member = tarfile.TarInfo("extended-metadata") + member.pax_headers = {"comment": "x" * (128 * 1024 + 1)} + _rewrite(archive, manifest, member) + with pytest.raises(StateRecoveryError, match="archive_metadata_too_large"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "invalid", execute=True) + assert not (tmp_path / "invalid").exists() + + +@pytest.mark.parametrize("kind,headers,decoder", [ + (tarfile.GNUTYPE_SPARSE, {}, "_proc_sparse"), + (tarfile.REGTYPE, {"GNU.sparse.size": "invalid"}, "_proc_gnusparse_00"), + (tarfile.REGTYPE, {"GNU.sparse.map": "invalid"}, "_proc_gnusparse_01"), + (tarfile.REGTYPE, {"GNU.sparse.major": "1", "GNU.sparse.minor": "0"}, "_proc_gnusparse_10"), +]) +def test_sparse_records_are_refused_before_stdlib_extent_decoding(tmp_path, monkeypatch, kind, headers, decoder): + archive, manifest = _backup(tmp_path) + member = tarfile.TarInfo("unsupported-sparse") + member.type, member.pax_headers = kind, headers + _rewrite(archive, manifest, member) + + # Malformed sparse metadata must produce the same bounded refusal as a + # valid unsupported record, rather than leaking a decoder exception. + with pytest.raises(StateRecoveryError, match="^archive_sparse_unsupported$"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "inert", execute=True) + + def unexpected_decode(*_args): + pytest.fail("unsupported sparse extents reached the stdlib decoder") + + monkeypatch.setattr(tarfile.TarInfo, decoder, unexpected_decode) + with pytest.raises(StateRecoveryError, match="^archive_sparse_unsupported$"): + verify_state_backup(archive=archive, manifest=manifest, destination=tmp_path / "inert", execute=True) + assert not (tmp_path / "inert").exists() + assert not list(tmp_path.glob(".loopx-recovery-*")) + + +@pytest.mark.parametrize("kind", [tarfile.XHDTYPE, tarfile.XGLTYPE, tarfile.SOLARIS_XHDTYPE]) +def test_oversized_pax_variants_are_refused_without_reading_the_body(kind): + from loopx.state_recovery import _scan + + member = tarfile.TarInfo("oversized-pax") + member.type, member.size = kind, 128 * 1024 + 1 + compressed = gzip.compress(member.tobuf()) + with pytest.raises(StateRecoveryError, match="^archive_metadata_too_large$"): + _scan(io.BytesIO(compressed), len(compressed)) + + +def test_malformed_pax_record_length_is_a_bounded_archive_error(): + from loopx.state_recovery import _scan + + payload = b"9" * 5000 + b" comment=invalid\n" + member = tarfile.TarInfo("malformed-pax") + member.type, member.size = tarfile.XHDTYPE, len(payload) + raw = member.tobuf() + payload + b"\0" * (-len(payload) % 512 + 1024) + compressed = gzip.compress(raw) + with pytest.raises(StateRecoveryError, match="^archive_unreadable$"): + _scan(io.BytesIO(compressed), len(compressed)) + + +def test_failed_publication_cleans_staging_even_when_destination_becomes_occupied(tmp_path, monkeypatch): + import loopx.state_recovery as recovery + + archive, manifest = _backup(tmp_path) + destination = tmp_path / "inert" + + def failed_rename(source, target): + assert target == destination and (source / "recovery-audit.json").is_file() + (destination / "foreign-file").write_text("preserve this") + raise OSError("publication interrupted") + + monkeypatch.setattr(recovery.os, "rename", failed_rename) + with pytest.raises(OSError): + verify_state_backup(archive=archive, manifest=manifest, destination=destination, execute=True) + assert (destination / "foreign-file").read_text() == "preserve this" + assert not (destination / "recovery-audit.json").exists() + assert not list(tmp_path.glob(".loopx-recovery-*")) + + +def test_symlink_parent_and_occupied_directory_are_not_recovery_targets(tmp_path): + archive, manifest = _backup(tmp_path) + link = tmp_path / "alias" + link.symlink_to(tmp_path / "project", target_is_directory=True) + with pytest.raises(StateRecoveryError, match="parent_not_physical"): + verify_state_backup(archive=archive, manifest=manifest, destination=link / "new", execute=True) + occupied = tmp_path / "occupied" + occupied.mkdir() + (occupied / "keep").write_text("unchanged") + with pytest.raises(StateRecoveryError, match="destination_exists"): + verify_state_backup(archive=archive, manifest=manifest, destination=occupied, execute=True) + assert (occupied / "keep").read_text() == "unchanged" + + +def test_real_http_upload_verify_reload_and_corrupt_input_keep_live_state_unchanged(tmp_path): + from loopx.chat_server import ChatHTTPServer, ChatRequestHandler + + archive, manifest = _backup(tmp_path) + runtime = tmp_path / "replacement-runtime" + runtime.mkdir() + server = ChatHTTPServer(("127.0.0.1", 0), ChatRequestHandler) + server.runtime_root, server.registry_path, server.verbose = runtime, tmp_path / "absent-registry.json", False + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + + def request(path, body=None, *, binary=False): + connection = http.client.HTTPConnection("127.0.0.1", server.server_address[1], timeout=90) + encoded = body if binary else json.dumps(body).encode() if body is not None else None + connection.request("POST" if body is not None else "GET", "/api/chat/state-recovery" + path, + body=encoded, headers={"Content-Type": "application/octet-stream" if binary else "application/json"}) + response = connection.getresponse() + payload = json.loads(response.read()) + status = response.status + connection.close() + return status, payload + + try: + assert request("")[1]["items"] == [] + assert not (runtime / "backups").exists() + _, first = request("/archive", archive.read_bytes(), binary=True) + _, second = request("/manifest", manifest.read_bytes(), binary=True) + inputs = {"archive_ref": first["input_ref"], "manifest_ref": second["input_ref"], "execute": True} + status, result = request("/verify", inputs) + assert status == 200, result + assert result["execution_authority_granted"] is False + assert not server.registry_path.exists() + status, rows = request("") + assert status == 200 and rows["items"][0]["checkpoint_ref"] == result["checkpoint_ref"] + read = request("/read", {"checkpoint_ref": result["checkpoint_ref"]})[1] + assert read["historical_readback"] is True and read["audit_sha256"] == result["audit_sha256"] + duplicate_status, duplicate = request("/verify", inputs) + assert duplicate_status == 400 and duplicate["error_code"] == "recovery_destination_exists" + checkpoint = runtime / "backups/state-recovery/checkpoints" / result["checkpoint_ref"] + before = (checkpoint / "recovery-audit.json").read_bytes() + _, damaged = request("/archive", b"damaged archive", binary=True) + status, rejected = request("/verify", {**inputs, "archive_ref": damaged["input_ref"]}) + assert status == 400 and rejected["error_code"] == "archive_digest_mismatch" + assert (checkpoint / "recovery-audit.json").read_bytes() == before + assert len(list((runtime / "backups/state-recovery/checkpoints").iterdir())) == 1 + assert not server.registry_path.exists() + finally: + server.shutdown() + worker.join(timeout=5) + server.server_close()