From 8581abde06ae48fe46be4c2c918c080aeb273d95 Mon Sep 17 00:00:00 2001 From: pmwl Date: Fri, 28 Aug 2026 18:32:36 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(claude):=20claude.ai=20=E5=8D=95?= =?UTF-8?q?=E5=AF=B9=E8=AF=9D=E5=AF=BC=E5=87=BA=20+=20=E7=AB=99=E7=82=B9?= =?UTF-8?q?=E9=80=82=E9=85=8D=E5=99=A8=E6=9E=B6=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 把「隐式假设 ChatGPT 数据模型」的代码重构成「站点适配器 + 中间表示」， 再在其上实装 claude.ai 适配器。 架构 - core/ir.ts 站点无关的中间表示（IRConversation / IRTurn / IRBlock） - core/render.ts IR → Markdown，两站点 × 油猴端/离线 CLI 四条路径共用 - core/fetcher.ts 限速 / 退避 / 并发池 / 取消 / 限流观测，每站点独立实例 - sites/types.ts SiteAdapter 契约：取数 + 转换 + 界面锚点 + 批量能力 - main.ts / ui.ts 不再认识任何一家的端点、字段与 DOM 重构以「现有 102 个测试断言一字不改地通过」为验收标准，行为逐字节等价； convert/markdown.ts 与 api.ts 退化为兼容壳，离线 CLI 无需改动。 Claude 侧 - 主线沿 current_leaf_message_uuid 的 parent 链回溯（与 mapping 树同构） - 块级分发：一条消息里 text / thinking / tool_use / tool_result 按序交错 - artifact 折叠 create/update/rewrite 还原终稿；update 用函数替换器， 避免 new_str 里的 $& 等被当作替换模式静默损坏内容 - 附件两处来源：files[] 下载，attachments[].extracted_content 内联进 展开的 callout（导出的笔记因此对全文检索自包含，ChatGPT 侧做不到） - 未识别块与重放失败一律原始 JSON 折叠兜底，不静默丢内容 - 过期的搜索结果不写进笔记：死链比没有链接更糟 ⚠️ 首版刻意只开放「导出当前对话」 批量的地基（分页器、水位线、并发池、保护性中止）已就位并单测，但 supportsBatch: false 关着——Claude 的限流画像没有任何实测数据，调研过的 三个开源 claude.ai 导出器也无一实现 429 退避（最激进的是 3 并发 + 固定 200ms 且不看 429），没有可借鉴的安全参数。起步节奏取 ChatGPT 的两倍慢 （间距 1500ms、每 40 请求歇 30s），吃到 429 时完成文案会报出次数、被推大的 间距与服务端要求的最长等待。 docs/claude-adapter-feasibility.md 是决策依据，docs/claude-probe.js 可贴进 控制台做结构与限流探测（≤8 请求、遇 429 立即停止、不打印对话内容）。 测试 132 通过（新增 30），typecheck 干净，构建产物 121 kB。 --- PLAN.md | 80 ++++- README.md | 40 ++- README.zh-CN.md | 36 ++- docs/claude-adapter-feasibility.md | 275 +++++++++++++++++ docs/claude-probe.js | 172 +++++++++++ src/api.ts | 473 +++++++---------------------- src/convert/citations.ts | 8 +- src/convert/markdown.ts | 417 ++----------------------- src/core/fetcher.ts | 224 ++++++++++++++ src/core/ir.ts | 81 +++++ src/core/render.ts | 215 +++++++++++++ src/main.ts | 234 +++++++------- src/sites/chatgpt/convert.ts | 256 ++++++++++++++++ src/sites/chatgpt/index.ts | 98 ++++++ src/sites/claude/api.ts | 189 ++++++++++++ src/sites/claude/artifacts.ts | 131 ++++++++ src/sites/claude/convert.ts | 355 ++++++++++++++++++++++ src/sites/claude/index.ts | 83 +++++ src/sites/claude/types.ts | 106 +++++++ src/sites/index.ts | 23 ++ src/sites/types.ts | 88 ++++++ src/ui.ts | 115 ++----- test/claude-pager.test.ts | 100 ++++++ test/claude.test.ts | 275 +++++++++++++++++ test/fixtures/claude-basic.json | 119 ++++++++ vite.config.ts | 24 +- 26 files changed, 3225 insertions(+), 992 deletions(-) create mode 100644 docs/claude-adapter-feasibility.md create mode 100644 docs/claude-probe.js create mode 100644 src/core/fetcher.ts create mode 100644 src/core/ir.ts create mode 100644 src/core/render.ts create mode 100644 src/sites/chatgpt/convert.ts create mode 100644 src/sites/chatgpt/index.ts create mode 100644 src/sites/claude/api.ts create mode 100644 src/sites/claude/artifacts.ts create mode 100644 src/sites/claude/convert.ts create mode 100644 src/sites/claude/index.ts create mode 100644 src/sites/claude/types.ts create mode 100644 src/sites/index.ts create mode 100644 src/sites/types.ts create mode 100644 test/claude-pager.test.ts create mode 100644 test/claude.test.ts create mode 100644 test/fixtures/claude-basic.json diff --git a/PLAN.md b/PLAN.md index 1c52c4a..71de94d 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,4 +1,4 @@ -# Inkstone（砚）— ChatGPT 对话导出 +# Inkstone（砚）— ChatGPT / Claude 对话导出 > 名取「砚」：把 GPT 的原始输出研磨成能写进笔记的墨；石对石（砚 ↔ Obsidian）。 @@ -6,7 +6,8 @@ ## 目标 -- 在 chatgpt.com 页内一键**批量导出全部对话**为 Obsidian 等笔记软件友好的 Markdown +- 在 chatgpt.com / claude.ai 页内一键导出对话为 Obsidian 等笔记软件友好的 Markdown + （ChatGPT 支持批量与增量；Claude 首版只做当前对话，理由见 P5） - 高保真：公式、引用链接、代码、图片/附件、思维链、Canvas 不丢不乱 - 增量同步：重跑只导出有变化的对话 - 全程本地处理，不经任何第三方服务 @@ -57,30 +58,52 @@ ## 项目结构 +依赖方向单向收敛：`sites/* → core/*`。core 不认识任何站点，sites 不认识编排。 + ``` inkstone/ package.json # bun（scripts: dev/build/test/typecheck/offline） - vite.config.ts # vite-plugin-monkey + vite.config.ts # vite-plugin-monkey（match: chatgpt.com / claude.ai） cli/ export.ts # 官方导出 zip → vault 的离线 CLI（bun 直跑，P4.2） + docs/ + claude-adapter-feasibility.md # Claude 移植可行性评估（P5 的依据） + claude-probe.js # 贴进 claude.ai 控制台的结构 / 限流探针 src/ - main.ts # UI 注入 + 流程编排 + 输出 Sink（zip/直写） - api.ts # backend-api 客户端（token/列表/全文/附件） - convert/ + main.ts # 流程编排 + 输出 Sink（zip/直写）——站点无关 + ui.ts # 浮动面板 + 进度——站点无关，锚点与配色问 adapter + state.ts # 增量水位线（按站点分表）+ 设置持久化 + core/ # ← 站点无关内核 + ir.ts # 中间表示：IRConversation / IRTurn / IRBlock + render.ts # IR → Markdown（轮次标题、callout、围栏、frontmatter） + fetcher.ts # 限速 / 退避 / 并发池 / 取消 / 限流观测（每站点一个实例） + sites/ + types.ts # SiteAdapter 契约（取数 + 转换 + 界面锚点 + 批量能力） + index.ts # 按 location.host 分派 + chatgpt/ + index.ts # adapter 实装 + convert.ts # backend-api JSON → IR（content_type 分发、canmore 语义） + claude/ + index.ts # adapter 实装（supportsBatch: false） + api.ts # 内部 API 客户端 + 保守限流参数 + 分页器（未接界面） + types.ts # 从宽的字段类型，[待测] 处已标注 + convert.ts # 内部 API JSON → IR（块级分发、主线回溯、附件两处来源） + artifacts.ts # artifact create/update/rewrite 折叠成终稿 + convert/ # ChatGPT 专属转换 + 通用文本工具（历史路径，测试直接引用） + markdown.ts # 兼容壳：conversationToIR + renderConversation linearize.ts # mapping 树 → 线性消息 - markdown.ts # 消息 → md（content_type 分发；assetLink 链接风格） - math.ts # 公式定界符转换（代码块感知） - citations.ts # 引用标记还原（matched_text 通道 + 官方导出 token/顺序配对通道） - headings.ts # 标题降级 / 剥离为加粗 canvas.ts # Canvas textdoc patch 重放 + citations.ts # 私有区引用标记还原 + math.ts # 公式定界符转换（代码块感知）—— 两站点共用 + headings.ts # 标题降级 / 剥离为加粗 —— 两站点共用 + codeaware.ts # 代码块感知的文本变换基础设施 —— 两站点共用 + api.ts # ChatGPT backend-api 客户端（端点与字段，节奏交给 core） output/ zip.ts # fflate 打包 fsaccess.ts # File System Access 直写 vault（句柄存 IndexedDB） - state.ts # 增量水位线 + 设置持久化 - ui.ts # 浮动面板 + 进度 test/ - fixtures/*.json # 真实对话 JSON（脱敏） - *.test.ts # bun test + fixtures/*.json # 对话 JSON（ChatGPT 真实脱敏 / Claude 合成） + *.test.ts # bun test（132 个） ``` ## 阶段 @@ -94,7 +117,34 @@ inkstone/ - **P4 脱离油猴（用户明确期望）**：转换层（convert/）零浏览器依赖、api.ts 只依赖 fetch，天然可复用到： 1. **MV3 浏览器扩展**——同一套 src，加 manifest + content script 打包目标（vite 多入口）；不再依赖 Tampermonkey，可上架商店（用户拍板：等功能完善后再做/上架） 2. **官方导出 zip 的离线 CLI** ✅（2026-07-10，`bun run offline [-o 输出]`）——完全不碰 backend-api，零限流风险；432 对话 + 248 附件 ~8s 转完；支持 `--link-style/--heading-mode/--no-thoughts/--no-assets` - 3. Claude/Gemini adapter + 3. Claude adapter ✅ 骨架（2026-08-28，见下）／ Gemini 待做 + +- **P5 多站点架构 + Claude adapter**（2026-08-28）：见 `docs/claude-adapter-feasibility.md`（可行性评估） + - **架构**：引入站点无关的中间表示（IR）与适配器契约，依赖方向变成 + `sites/{chatgpt,claude}/ → core/{ir,render,fetcher}`。编排（main.ts）与界面（ui.ts） + 不再认识任何一家的端点、字段或 DOM；新增站点 = 新增一个 adapter，不改编排。 + 重构以「现有 102 个测试断言一字不改地通过」为验收标准，行为逐字节等价。 + - **两侧数据模型的结构性差异**：ChatGPT 一条消息一种 content_type（消息级分发）， + Claude 一条消息多个 typed block 按序交错（块级分发）。主线定位则同构—— + 两边都是「叶子 + parent 链回溯后反转」。 + - **Claude 侧的脏活更少**：Canvas 的正则 patch 重放（150 行）与私有区 Unicode 引用 + 还原（178 行）在 Claude 都不需要——artifact 的 update 是字面量 `old_str`→`new_str`， + 引用是结构化数组。artifact 折叠约 40 行。 + - **⚠️ 首版刻意只做「导出当前对话」**：批量的地基（分页器、水位线、并发池、 + 保护性中止）全部就位且已单测，但 `supportsBatch: false` 关着。理由是限流画像 + 未知——ChatGPT 侧的参数是 344 + 432 对话实测调出来的，Claude 侧一条实测数据 + 都没有。调研过的三个开源 claude.ai 导出器**没有一个实现了 429 退避** + （最激进的是 3 并发 + 固定 200ms 间隔且不看 429），所以没有可借鉴的安全参数。 + - **Claude 限流起步参数**（保守，待实测调整）：间距 1500ms（ChatGPT 侧的两倍慢）、 + 上限 8000ms、每 40 请求歇 30s、最多重试 6 次。每个站点持有独立的 fetcher 实例， + 一边的限流不拖累另一边。吃到 429 时导出完成文案会报出次数、被推大的间距与 + 服务端要求的最长等待——未知站点的节奏只能靠实测看清，先让它可见再谈调参。 + - **待实测**：`docs/claude-probe.js` 可直接粘进 claude.ai 控制台，打印字段骨架 + （不打印对话内容）并做一次 ≤8 请求的保守限流试探。清单见可行性文档第五节： + 分页是否生效、thinking 字段名、citations 挂载形态、附件地址能否直取字节、 + 公式定界符、Projects 字段名、FAB 锚点选择器。 + - **未做**：Claude 的行内引用锚定（citations 的字符级定位字段未实测，首版只把 + 来源汇总进文末 Sources，正文一字不动）；Claude 官方导出 zip 的离线 CLI 通道。 ## 实战经验（2026-07-08 E2E，344 对话实测） diff --git a/README.md b/README.md index 3764e59..ff023b5 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ # Inkstone · 「砚」 -**Batch-export your entire chatgpt.com history to Obsidian-friendly Markdown — one click, in the page, fully local.** +**Export your chatgpt.com and claude.ai conversations to Obsidian-friendly Markdown — one click, in the page, fully local.** -*An inkstone grinds raw pigment into ink for writing. Inkstone helps you grind GPT's raw output into ink for your notes.* +*An inkstone grinds raw pigment into ink for writing. Inkstone helps you grind raw model output into ink for your notes.* [![release](https://img.shields.io/github/v/release/ZhenHuangLab/inkstone)](https://github.com/ZhenHuangLab/inkstone/releases/latest) [![downloads](https://img.shields.io/github/downloads/ZhenHuangLab/inkstone/total)](https://github.com/ZhenHuangLab/inkstone/releases) @@ -20,7 +20,7 @@  →  ② Install Inkstone  →  - ③ Open chatgpt.com, hit ⤓ + ③ Open chatgpt.com or claude.ai, hit ⤓

--- @@ -29,7 +29,25 @@ Your ChatGPT history holds real work, but getting it into a vault is painful. The official export is a raw JSON dump: tool and system messages are stripped (Canvas and code-interpreter output simply aren't there), some attachments have already expired server-side, math arrives in `\( \)` delimiters Obsidian won't render, and web-search citations turn into private-use Unicode garbage. Copy-pasting by hand doesn't scale past ten conversations, let alone a thousand. -Inkstone runs inside chatgpt.com and fetches conversations through the same backend API the app itself uses, then converts everything locally in your browser — nothing ever leaves the page. The result is Markdown that reads natively in Obsidian: real headings per turn, `$` / `$$` math, resolved citations, downloaded images, clean frontmatter. +Inkstone runs inside the page and fetches conversations through the same backend API the app itself uses, then converts everything locally in your browser — nothing ever leaves the page. The result is Markdown that reads natively in Obsidian: real headings per turn, `$` / `$$` math, resolved citations, downloaded images, clean frontmatter. + +## Supported sites + +| | ChatGPT | Claude | +| --- | --- | --- | +| Export current conversation | ✅ | ✅ | +| Batch / export-all | ✅ | ⏳ not yet enabled | +| Incremental sync | ✅ | ⏳ not yet enabled | +| Rich documents | Canvas patch replay | Artifact fold-up to final version | +| Thoughts / tool traces | ✅ opt-in | ✅ opt-in | +| Attachments | images and files downloaded | images downloaded, documents linked, text extractions inlined | + +**Why no batch export on Claude yet?** It isn't missing, it's switched off. The pager, +watermark, concurrency pool and protective abort are all in place and unit-tested — but +there is no measured rate-limit profile for Claude yet. The ChatGPT numbers only became +trustworthy after 344 + 432 real conversations. Until comparable evidence exists, the cost +of a wrong guess lands on your account, and that isn't a call a default-on switch should +make. See [`docs/claude-adapter-feasibility.md`](./docs/claude-adapter-feasibility.md). ## Screenshots @@ -97,10 +115,11 @@ bun run build # → dist/inkstone.user.js, drag it into Tampermonkey ## Usage -Open chatgpt.com (logged in) → click the **⤓ button left of the Share button** in the top bar → pick **Markdown zip** or **raw JSON zip** → unzip into your Obsidian vault. +Open chatgpt.com or claude.ai (logged in) → click the **⤓ button** in the top bar → pick **Markdown zip** or **raw JSON zip** → unzip into your Obsidian vault. +- On Claude only **current conversation** is offered; the batch options are hidden, not disabled - The button position is switchable (panel → advanced settings): next to Share in the top bar, or a glass button beside the input box -- The UI follows ChatGPT's appearance settings automatically (light/dark + accent color) +- The UI follows the host page's appearance automatically (light/dark + accent color) - Exports are cancelable; a single failed conversation never aborts the run — failures are summarized in `_failures.json` ## Offline CLI @@ -126,9 +145,16 @@ bun run typecheck bun run build ``` +Note for claude.ai: its CSP may block the dev-server script, so verify Claude-side changes +against a real `bun run build` artifact loaded into Tampermonkey rather than `bun run dev`. + +Architecture: `src/core/` is site-agnostic (IR, renderer, throttled fetcher) and +`src/sites//` holds everything that knows one provider's endpoints, fields and DOM. +Adding a site means adding an adapter, not touching the orchestration. See `PLAN.md` § P5. + ## Roadmap -MV3 browser extension (no Tampermonkey, store release) and Claude / Gemini support. Already done: incremental sync, direct-write to an Obsidian vault, settings panel, Canvas patch replay, and the offline CLI. Details in [PLAN.md](./PLAN.md) (Chinese). +Batch export on Claude once its rate-limit profile has actually been measured, an MV3 browser extension (no Tampermonkey, store release), and Gemini support. Already done: the multi-site adapter architecture, Claude single-conversation export, incremental sync, direct-write to an Obsidian vault, settings panel, Canvas patch replay, Artifact fold-up, and the offline CLI. Details in [PLAN.md](./PLAN.md) (Chinese). ## License diff --git a/README.zh-CN.md b/README.zh-CN.md index 45a91a6..ff5bda5 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -2,9 +2,9 @@ # Inkstone · 「砚」 -**在 chatgpt.com 页内一键批量导出全部对话为 Obsidian 友好的 Markdown——全程本地处理。** +**在 chatgpt.com / claude.ai 页内一键导出对话为 Obsidian 友好的 Markdown——全程本地处理。** -*砚是把原料研磨成墨、供你书写的石头——Inkstone 帮助你把 GPT 的原始输出研磨成能写进笔记的墨。* +*砚是把原料研磨成墨、供你书写的石头——Inkstone 帮助你把模型的原始输出研磨成能写进笔记的墨。* [![release](https://img.shields.io/github/v/release/ZhenHuangLab/inkstone)](https://github.com/ZhenHuangLab/inkstone/releases/latest) [![downloads](https://img.shields.io/github/downloads/ZhenHuangLab/inkstone/total)](https://github.com/ZhenHuangLab/inkstone/releases) @@ -20,7 +20,7 @@  →  ② 一键安装 Inkstone  →  - ③ 打开 chatgpt.com，点 ⤓ + ③ 打开 chatgpt.com 或 claude.ai，点 ⤓

--- @@ -29,7 +29,24 @@ ChatGPT 的对话历史里沉淀着真正的工作成果，但想把它们搬进 vault 很痛苦：官方导出是一坨原始 JSON——tool/system 消息被剥掉（Canvas、代码解释器的产出根本不在里面），部分附件在服务端已经过期，公式是 Obsidian 不认的 `\( \)` 定界符，联网搜索引用变成私有区 Unicode 乱码。手动复制粘贴撑不过十条对话，更别说上千条。 -Inkstone 直接运行在 chatgpt.com 页内，通过应用自己使用的 backend API 抓取对话，然后全部在浏览器本地转换——数据不出页面。产出的 Markdown 在 Obsidian 里原生可读：每轮对话有真实标题、`$` / `$$` 公式、还原后的引用、下载好的图片、干净的 frontmatter。 +Inkstone 直接运行在页内，通过应用自己使用的 backend API 抓取对话，然后全部在浏览器本地转换——数据不出页面。产出的 Markdown 在 Obsidian 里原生可读：每轮对话有真实标题、`$` / `$$` 公式、还原后的引用、下载好的图片、干净的 frontmatter。 + +## 支持的站点 + +| | ChatGPT | Claude | +| --- | --- | --- | +| 导出当前对话 | ✅ | ✅ | +| 批量 / 全部导出 | ✅ | ⏳ 暂不开放 | +| 增量同步 | ✅ | ⏳ 暂不开放 | +| 富文档还原 | Canvas patch 重放 | Artifact 折叠还原终稿 | +| 思维链 / 工具痕迹 | ✅ 可开关 | ✅ 可开关 | +| 附件 | 图片与文件下载 | 图片下载、文档链接、文本抽取件内联 | + +**Claude 端为什么先不做批量？** 不是没写，是没开。批量所需的分页器、水位线、并发池、 +保护性中止都已就位并通过单测，但 Claude 侧的限流画像还没有任何实测数据—— +ChatGPT 端那套参数是 344 + 432 条对话跑出来才敢用的。在拿到同等的实测证据之前， +批量抓取整个历史的风险由用户账号承担，这个代价不该由一个默认开启的开关来决定。 +细节与实测计划见 [`docs/claude-adapter-feasibility.md`](./docs/claude-adapter-feasibility.md)。 ## 截图 @@ -96,7 +113,7 @@ bun run build # 产物 dist/inkstone.user.js，拖进 Tampermonkey 即可 ## 使用 -打开 chatgpt.com（已登录）→ 点**顶栏 Share 左侧的 ⤓ 按钮** → 选 **Markdown zip** 或**原始 JSON zip** → 解压到 Obsidian vault。 +打开 chatgpt.com 或 claude.ai（已登录）→ 点**顶栏 Share 左侧的 ⤓ 按钮** → 选 **Markdown zip** 或**原始 JSON zip** → 解压到 Obsidian vault。 - 按钮位置可换（面板 → 高级设置）：顶栏 Share 旁，或输入框旁的玻璃圆钮 - UI 主题色自动跟随 ChatGPT 的外观设置（明暗 + accent color） @@ -125,9 +142,16 @@ bun run typecheck bun run build ``` +claude.ai 的 CSP 可能拦掉 dev server 的脚本，Claude 端的改动请用 `bun run build` 的产物 +装进 Tampermonkey 验证，别只依赖 `bun run dev`。 + +架构：`src/core/` 站点无关（中间表示、渲染器、带限速的取数内核）， +`src/sites/<站点>/` 收纳一家的端点、字段与 DOM 知识。新增站点 = 新增一个适配器， +不改编排与界面。详见 [PLAN.md](./PLAN.md) 的 P5 一节。 + ## 路线图 -MV3 浏览器扩展（脱离 Tampermonkey、上架商店）、Claude / Gemini 适配。已完成：增量同步、直写 vault、设置面板、Canvas patch 重放、离线 CLI。详见 [PLAN.md](./PLAN.md)。 +Claude 端的批量导出（等限流画像实测清楚再开）、MV3 浏览器扩展（脱离 Tampermonkey、上架商店）、Gemini 适配。已完成：多站点适配器架构、Claude 单对话导出、增量同步、直写 vault、设置面板、Canvas patch 重放、Artifact 折叠还原、离线 CLI。详见 [PLAN.md](./PLAN.md)。 ## 许可证 diff --git a/docs/claude-adapter-feasibility.md b/docs/claude-adapter-feasibility.md new file mode 100644 index 0000000..50fff44 --- /dev/null +++ b/docs/claude-adapter-feasibility.md @@ -0,0 +1,275 @@ +# Inkstone → Claude 对话导出：可行性分析 + +> **实施状态（2026-08-28）**：本文第四节的架构改造与第六节的 P1–P3 已完成， +> Claude 单对话导出可用；批量（P4）按第五节的风险判断**刻意未开放**。 +> 落地记录见 `PLAN.md` § P5，待实测清单见本文第五节，探针脚本见 `docs/claude-probe.js`。 +> 本文其余部分保持评估当时的原貌，不随实施回填——它是决策依据的快照。 + +> 评估日期：2026-08-28 · 基准代码：`2121e9f`（v0.2.3，与上游 ZhenHuangLab/inkstone 同步） +> +> 证据分级：**[码]** = 读本仓库源码得出；**[开源]** = 依据 claude.ai 现有开源导出器的实现与其维护文档（二手，标注记录时间）；**[待测]** = 需要在真实 claude.ai 会话里实测确认。本文不含任何未标注来源的推断。 + +## 结论 + +**可行，且比预期便宜。** 移植的主要成本不在写 Claude 适配器，而在把现有代码从「隐式假设 ChatGPT 数据模型」重构成「站点适配器 + 中间表示」。 + +- 转换层（`convert/`）里真正**站点无关**的部分——公式、标题降级、代码块感知、排版收尾、附件链接、frontmatter、文件名——可以原样复用，约占转换层代码的 40%。 +- ChatGPT 最脏的两块逻辑（PUA Unicode 引用还原、Canvas 正则 patch 重放）在 Claude 侧**都不需要**：Claude 的 artifact 修订是显式 `old_str`→`new_str` 结构 [开源]，引用是结构化数组 [开源/待测]，反而更简单。 +- 取数层的**工程价值全部可复用**：全局限速、突发桶退避、条目级 429 与全局限流的区分、空页重试、失败重试与保护性中止——这些是 344/432 对话实测踩出来的经验 [码/PLAN.md]，换个站点端点即可继续用。 +- 唯一真正的新增风险是**风控未知**：Claude 侧没有对应的实测数据，ChatGPT 的限流画像不能照搬。 + +工作量估算：**8–12 人天**（熟悉本代码库的前提下），其中重构 3、Claude 适配器 3、UI 适配 2、实测调试与 fixture 2。 + +--- + +## 一、现状：这个项目是什么 + +Tampermonkey userscript，TypeScript + Bun + Vite + vite-plugin-monkey，产物单文件 `inkstone.user.js`，注入 chatgpt.com，把全部对话批量转成 Obsidian 友好的 Markdown。3553 行源码，纯本地处理。[码] + +### 分层（作者刻意的两层解耦） + +| 层 | 文件 | 行数 | 职责 | +|---|---|---|---| +| 取数 | `src/api.ts` | 313 | ChatGPT backend-api 客户端：token、列表分页、详情、附件签名 URL、全局限速与退避 | +| 转换 | `src/convert/*` | 833 | 纯 TS 零浏览器依赖，`bun test` 可测：线性化、公式、标题、引用、Canvas、代码感知 | +| 编排 | `src/main.ts` | 563 | 抓取 → 转换 → 附件下载 → 落地；两遍抓取 + 水位线推进 | +| 输出 | `src/output/*` | 140 | fflate 打 zip / File System Access 直写 vault | +| 状态 | `src/state.ts` | 128 | 增量水位线 + 设置持久化（GM 存储，回退 localStorage） | +| UI | `src/ui.ts` | 991 | Shadow DOM 液态玻璃面板、FAB 锚定、主题跟随 | +| 离线 | `cli/export.ts` | 370 | 官方导出 zip → Markdown，完全不碰内部 API | + +`PLAN.md` 的 P4 路线图第 3 项就写着 **「Claude/Gemini adapter」**，尚未动工——本文即为该项的前置评估。[码] + +### 这个项目真正的资产 + +不是「能导出对话」，市面上一堆脚本都能。是这三样： + +1. **失真控制的产品哲学**：未知内容类型 → 原始 JSON 塞折叠 callout，**永不静默丢内容**；引用还原不了 → 剥离，绝不留乱码；附件失败 → 正文留占位 + 完成文案报数。[码 `markdown.ts:default` 分支] +2. **实战限流画像**：列表 `total` 不可靠、列表索引会瞬时降级、突发桶型限流、条目级 429 vs 全局限流的区分、附件 `size` 元数据不可靠。这些写在 `PLAN.md` 的「实战经验」里，是 344 + 432 对话跑出来的。[码] +3. **两条管道一致性**：油猴端与离线 CLI 共用同一个 `assetLink` / `conversationToMarkdown`，保证产出一致。[码 `markdown.ts` 注释] + +移植的意义在于：把这三样资产复用到 Claude，而不是再写一个「能导出对话」的脚本。 + +--- + +## 二、两侧数据模型对照 + +| 维度 | ChatGPT（本项目现状 [码]） | claude.ai [开源，2026-07 记录] | +|---|---|---| +| 鉴权 | `GET /api/auth/session` 换 `accessToken`，再带 `Authorization: Bearer` | 同源 cookie 直接够用，**无需换 token**；只需 orgId（`lastActiveOrg` cookie 或 `GET /api/organizations`） | +| 列表 | `GET /backend-api/conversations?offset&limit=100&order=updated` | `GET /api/organizations/{org}/chat_conversations`（分页参数 `limit`/`offset` 见于部分实现，**是否必需/上限 [待测]**） | +| 详情 | `GET /backend-api/conversation/{id}` | `GET /api/organizations/{org}/chat_conversations/{id}?tree=True&rendering_mode=messages&render_all_tools=true` | +| 会话结构 | `mapping: Record` 树 | `chat_messages[]` 扁平数组 + `parent_message_uuid` 链 | +| 主线定位 | 从 `current_node` 沿 parent 回溯后 reverse | 从 `current_leaf_message_uuid` 沿 `parent_message_uuid` 回溯后 reverse（**同构**） | +| 角色 | `author.role: user/assistant/system/tool` | `sender: 'human' \| 'assistant'`（无 tool 角色，工具在 content 块里） | +| 消息内容 | `content.content_type` + `parts[]`（单一类型/条消息） | `content: [{type: 'text'\|'thinking'\|'tool_use'\|'tool_result', ...}]`（**一条消息多块、按序交错**） | +| 正文来源 | `parts[]` 拼接 | `content[].text`；注意顶层便利字段 `chat_messages[].text` 在此 rendering mode 下**为空** | +| 思维链 | `content_type: 'thoughts'` → `thoughts[{summary, content}]` | `type: 'thinking'` 块（**字段名 [待测]**） | +| 富文档 | Canvas：`recipient: canmore.*` + JSON payload，`update_textdoc` 走**正则 patch**，重放易失败 | Artifact：`tool_use` name=`artifacts`，`input.command: create/update/rewrite`，update 是**字面量 `old_str`→`new_str`**，`version_uuid` 标记最后一次编辑 | +| 引用 | 正文嵌 **PUA Unicode 标记**（U+E200 区段），靠 `metadata.content_references` 反查还原，官方导出包里连 `matched_text` 都没有，需按数量顺序配对 | 结构化 `citations` 数组（API 版形态见官方文档；**网页版实际形态 [待测]**）；web_search 结果 URL 会 `is_expired` | +| 上传文件 | `metadata.attachments[]` + `sediment://` 指针 → `GET /backend-api/files/{id}/download` 换签名 URL | `files[]`：image→`preview_url`、document→`document_asset.url`+`page_count`、blob→无 URL；另有 `attachments[]` 携带 **`extracted_content` 纯文本**（ChatGPT 无此项） | +| 生成图片 | `multimodal_text` 里的 `asset_pointer` | 走 artifact / 文件 [待测] | +| 时间戳 | epoch 秒（详情）/ ISO（列表）混用 | ISO 字符串 | +| 模型 | 消息级 `metadata.model_slug`（最后一条为准） | 会话级 `model`（可能为 null，老对话需按日期推断 [开源]） | +| 层级 | 无（Branch 对话靠 metadata 回链） | **Projects**：对话可归属 project，ChatGPT 侧无对应概念 [待测字段名] | +| 官方离线导出 | 账号设置导出 zip，含 `conversations.json`（tool/system 被剥离） | 设置 → Privacy → Export data，邮件发 zip，含 `conversations.json`（结构比页内 API 简化 [开源]） | + +### 三个关键判断 + +**① 主线定位同构。** 两侧都是「叶子 + parent 链」模型，`linearize.ts` 的算法（含防环 `seen` 集合、缺 leaf 时的兜底）逻辑照搬，只换字段名。约 30 行改动。[码 + 开源] + +**② 内容模型是唯一的结构性差异。** ChatGPT 是「一条消息一种 content_type」，`renderMessage` 用 `switch` 分发；Claude 是「一条消息多个 typed block 按序交错」。现有 `switch` 结构无法直接套用——但这恰恰是引入中间表示（IR）的动机，见第四节。 + +**③ Claude 侧的脏活更少。** PUA Unicode 引用还原（`citations.ts` 178 行，含「matched_text 可能是裸空格」「turn 号是另一套编号只能按数量配对」这类补丁）和 Canvas 正则 patch 重放（`canvas.ts` 150 行，重放失败要回退原始 JSON）——这两块合计 328 行的复杂度在 Claude 侧都不存在。Claude 的 artifact 折叠只需 30 行左右 [开源实现即约此量级]。 + +--- + +## 三、逐模块移植评估 + +| 模块 | 行数 | 判定 | 说明 | +|---|---|---|---| +| `convert/codeaware.ts` | 78 | **原样复用** | 纯 Markdown 代码块感知，零站点耦合 | +| `convert/headings.ts` | 32 | **原样复用** | ATX 标题降级/剥离，零耦合 | +| `convert/math.ts` | 20 | **原样复用**（行为待验） | `\(\)`→`$` 转换。Claude 输出的公式定界符形态 [待测]：若本就是 `$`，此模块变成幂等空转，无害 | +| `output/zip.ts` | 27 | **原样复用** | | +| `output/fsaccess.ts` | 113 | **原样复用** | | +| `state.ts` | 128 | **复用 + 小改** | 水位线键改 `inkstone:wm:claude:{kind}`；`selectChanged` 已泛型化，直接吃 `updated_at` | +| `api.ts` | 313 | **框架复用，端点重写** | 限速/退避/并发池/分页器/取消令牌**全部保留**；`getAccessToken` 变成 `resolveOrgId`；三个端点换 URL。限流常数需按 Claude 实测重调 | +| `convert/linearize.ts` | 72 | **算法复用，字段重写** | 见上「判断①」 | +| `convert/markdown.ts` | 391 | **拆分重构** | 站点无关部分（frontmatter、文件名净化、callout/fence、Sources 汇总、排版收尾、assetLink）抽成 `render.ts`；`renderMessage` 的 content_type 分发下沉到各站点 adapter | +| `convert/canvas.ts` | 150 | **不移植，另写** | Claude 侧新写 `artifacts.ts`（约 30–40 行，`old_str`→`new_str` 折叠 + `version_uuid` 定稿）。⚠️ 注意开源实现踩过的坑：`String.replace` 必须传**函数**替换器，否则 `new_str` 里的 `$&`/`` $` ``/`$$` 会被当替换模式，静默损坏内容 [开源] | +| `convert/citations.ts` | 178 | **不移植，另写** | Claude 侧按结构化 citations 直接生成链接 + Sources，无需 PUA 处理 | +| `main.ts` | 563 | **编排复用，取数调用改造** | 两遍抓取、水位线合并推进、保护性中止、附件缓存与占位——逻辑全站点无关，只需把 `fetchConversation` 等换成 adapter 调用 | +| `ui.ts` | 991 | **结构复用，锚定与配色重写** | 面板、设置、多选懒加载、进度全部无关站点；FAB 锚点（`[data-testid="share-chat-button"]` / `#prompt-textarea`）与 accent 探测（`html[data-chat-theme]` + `--{theme}-theme-submit-btn-bg`）是 ChatGPT 专属，需为 claude.ai 各写一套 [待测选择器] | +| `cli/export.ts` | 370 | **框架复用，解包重写** | Claude 官方导出 zip 的内部布局与附件命名规则 [待测] | + +按行数粗算：**约 40% 原样复用，35% 改造复用，25% 需新写**。 + +--- + +## 四、建议的架构改造：SiteAdapter + 中间表示 + +现在的耦合是隐式的——`ConversationDetail`、`Message` 这些类型名不带站点前缀，但字段全是 ChatGPT 的。硬加 Claude 支持会变成 `if (isClaude)` 遍地。建议先做一次结构性重构： + +``` +src/ + core/ + ir.ts # 站点无关的中间表示（新增） + render.ts # IR → Markdown（从 markdown.ts 抽出，站点无关） + math.ts headings.ts codeaware.ts # 原样迁入 + fetcher.ts # 限速/退避/并发池/取消（从 api.ts 抽出） + sites/ + index.ts # 按 location.host 选 adapter + chatgpt/{api,types,convert,canvas,citations}.ts + claude/{api,types,convert,artifacts}.ts + output/ state.ts ui/ +``` + +### 中间表示草案 + +```ts +export interface IRConversation { + source: 'chatgpt' | 'claude' + id: string + title: string + url: string + createdAt: string // ISO + updatedAt: string + model?: string + models?: string[] + extraFrontmatter?: Record // branched_from / project / … + turns: IRTurn[] +} + +export interface IRTurn { + role: 'user' | 'assistant' + blocks: IRBlock[] // 按序，允许交错 +} + +export type IRBlock = + | { kind: 'prose'; text: string; sources?: SourceLink[] } // 已还原引用的正文 + | { kind: 'thinking'; title?: string; text: string } // 受 thoughts 开关控制 + | { kind: 'tool'; label: string; body: string; lang?: string } // 受 toolTraces 开关控制 + | { kind: 'document'; title: string; docType: string; content: string } // Canvas / Artifact 终稿 + | { kind: 'asset'; ref: AssetRef } // 占位符，由编排层下载改链 + | { kind: 'raw'; label: string; json: unknown } // 未知类型兜底，永不丢内容 +``` + +这套 IR 让**现有的三个开关语义（`thoughts` / `toolTraces` / `assets`）在两侧完全对齐**： + +| 开关 | ChatGPT | Claude | +|---|---|---| +| `thoughts` | `content_type: 'thoughts'` | `type: 'thinking'` 块 | +| `toolTraces` | `content_type: 'code'` / `execution_output` / `recipient !== 'all'` | `type: 'tool_use'` / `tool_result`（artifacts 除外，它归 `document`） | +| `assets` | `attachments[]` + `asset_pointer` | `files[]` + `attachments[].extracted_content` | + +`render.ts` 只认 IR，不认站点。CLI 和油猴端继续共用它——两条管道一致性的保证不变。 + +### 一个 Claude 独有的增益 + +`attachments[].extracted_content` 直接给出上传文档的**纯文本抽取**。ChatGPT 侧没有这个，只能下载二进制原件。在 Claude 侧可以把它作为 `raw`/`prose` 块内联为引用块（开源实现的做法是整块 `>` 引用，让附件自身的标题不进文档大纲 [开源]）——这让导出的笔记对 RAG 和全文检索**自包含**，是 ChatGPT 版做不到的。值得作为 Claude 侧的差异化特性。 + +--- + +## 五、风险与待实测清单 + +### 风险（按严重度排序） + +1. **风控未知（最高）。** 本项目对 ChatGPT 的限流认知是实测出来的：突发桶型、~200 连发触发连环 429、列表索引会瞬时降级、旧对话会渐进式变 429→404 且**恢复要数小时** [码/PLAN.md]。Claude 侧**没有任何对应数据**。批量抓取整个历史是明显的异常行为模式，不能假设 ChatGPT 的参数（并发 2、间距 800ms、每 80 请求歇 25s）在 Claude 也安全。 + **应对**：首版把并发锁死 1、间距起步 1500ms 以上，先用 10 条 / 50 条分级试跑，观察 429 与 `Retry-After`，再逐步放宽。宁可慢，不可触发账号级限制。 +2. **内部 API 无稳定性保证。** 两侧同等风险，且 Claude 侧无历史稳定性数据。缓解方式与现有一致：未知字段从宽、未知类型不丢内容、失败不中断整条流水线。 +3. **附件 URL 的会话绑定。** `preview_url` / `document_asset.url` 是同源 claude.ai 地址，只在登录同账号时可取 [开源]。能否在脚本里直接 `fetch` 到字节 [待测]；若返回重定向到 CDN 且跨域，附件管道需要额外处理。 +4. **CSP。** claude.ai 的 CSP 会限制 `connect-src`；同源 API 调用不受影响，但 **vite dev 模式从 localhost 加载脚本可能被拦** [开源提到 CSP 无法在 CI 验证]。开发时要准备好直接用 build 产物在 Tampermonkey 里验证。 +5. **`is_expired` 的搜索结果。** Claude 的 web 搜索结果 URL 会被标记过期 [开源]。把过期链接写进笔记会制造死链——建议沿用现有哲学：能还原的还原，不能的剥离并在 callout 里说明，不留假链接。 + +### 待实测清单（建议按顺序打通） + +| # | 待确认 | 怎么测 | +|---|---|---| +| 1 | `chat_conversations` 列表是否分页、上限多少、是否支持 `order` | 带/不带 `limit`&`offset` 各调一次，比对返回条数与账号实际对话数 | +| 2 | 列表项字段名（`uuid` / `name` / `updated_at` / `project_uuid`） | dump 首项键名 | +| 3 | `thinking` 块的字段名与是否有 summary | 找一条开了扩展思考的对话 dump 该块的键 | +| 4 | 网页版 citations 的实际挂载位置与形态 | 找一条带 web 搜索的对话 dump text 块的全部键 | +| 5 | 附件 URL 能否直接取到字节、Content-Type 是否正确 | 对 `preview_url` 做一次 `fetch` 看 status 与 headers | +| 6 | Claude 输出里公式的定界符形态（`$` 还是 `\(`） | 找一条带公式的对话看 `content[].text` 原文 | +| 7 | Projects 归属字段名 | dump 一条项目内对话的会话级键 | +| 8 | 官方导出 zip 的内部布局与附件命名 | 申请一次数据导出，解包看目录 | +| 9 | claude.ai 的 FAB 锚点选择器与主题变量 | 在页面上找稳定的 share 按钮 / composer 容器；读 `:root` 的 CSS 变量 | + +### 探针脚本 + +打通 1–7 项，只需在 claude.ai 打开一条对话后，把下面这段贴进浏览器控制台。它**只打印字段名与类型骨架，不打印任何对话内容**： + +```js +(async () => { + const org = document.cookie.match(/lastActiveOrg=([^;]+)/)?.[1] + const id = location.pathname.split('/').pop() + const skeleton = (v, d = 0) => { + if (v === null) return 'null' + if (Array.isArray(v)) return v.length ? [skeleton(v[0], d + 1)] : [] + if (typeof v !== 'object') return typeof v + if (d > 4) return '…' + return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, skeleton(x, d + 1)])) + } + const j = async (u) => (await fetch(u, { credentials: 'include' })).json() + + const list = await j(`/api/organizations/${org}/chat_conversations?limit=5&offset=0`) + console.log('列表条数', Array.isArray(list) ? list.length : '非数组') + console.log('列表项骨架', skeleton(Array.isArray(list) ? list[0] : list)) + + const conv = await j( + `/api/organizations/${org}/chat_conversations/${id}?tree=True&rendering_mode=messages&render_all_tools=true`, + ) + console.log('会话级键', Object.keys(conv)) + console.log('消息级键', [...new Set(conv.chat_messages.flatMap(Object.keys))]) + console.log('内容块类型', [ + ...new Set(conv.chat_messages.flatMap((m) => (m.content || []).map((b) => b.type))), + ]) + console.log('tool_use 名称', [ + ...new Set( + conv.chat_messages.flatMap((m) => + (m.content || []).filter((b) => b.type === 'tool_use').map((b) => b.name), + ), + ), + ]) + for (const t of ['text', 'thinking', 'tool_use', 'tool_result']) { + const b = conv.chat_messages.flatMap((m) => m.content || []).find((x) => x.type === t) + if (b) console.log(`${t} 块骨架`, skeleton(b)) + } + const f = conv.chat_messages.flatMap((m) => m.files || [])[0] + if (f) console.log('files[0] 骨架', skeleton(f)) +})() +``` + +--- + +## 六、建议的实施路径 + +**P0 · 离线通道先行（1–2 天，零风控风险）** +用 Claude 官方数据导出的 zip 走 CLI 通道。不碰内部 API，不动 UI，纯粹验证转换层：Claude JSON → IR → Markdown。产出是可回归的 fixture 和一条能跑通的管道。这一步的价值是**把最不确定的转换质量问题，放在最没有风险的环境里解决**。 + +**P1 · 结构重构（3 天）** +落地 `core/` + `sites/` + IR，把现有 ChatGPT 逻辑迁进 `sites/chatgpt/`，`bun test` 全绿——即现有 8 个测试文件必须一个不改地通过（`markdown.test.ts` 等可能需改导入路径，但断言不变）。这是安全网：重构不改行为。 + +**P2 · Claude 取数层（2–3 天）** +`sites/claude/api.ts` 复用 `core/fetcher.ts`，端点换掉，限流参数保守起步。先只做单对话导出（风险最小），实测跑通后再开批量。 + +**P3 · UI 适配（1–2 天）** +`vite.config.ts` 的 `match` 加 `https://claude.ai/*`；FAB 锚点与 accent 探测按站点分派。现有的「找不到锚点就不出现、锚点消失 4s+ 才整体隐藏」策略直接沿用——它本来就是为「页面改版」设计的防御。 + +**P4 · 批量与增量（1–2 天）** +列表分页器 + 水位线，按 P2 实测出的限流画像调参。跑一次全量，把 Claude 侧的实战经验补进 `PLAN.md`——这份文档是本项目最值钱的部分之一，Claude 侧应当有对应的一节。 + +### 一个战术建议 + +先做 **P0 + 单对话导出**，发一个 `0.3.0-alpha`。这样能在真实用户的真实数据上暴露转换质量问题（公式、artifact、附件），而完全不触碰批量抓取的风控风险。批量放到最后，等限流画像清楚了再开。 + +--- + +## 七、参考 + +- 本仓库 `PLAN.md`（ChatGPT 侧实战经验，2026-07） +- [agarwalvishal/claude-chat-exporter](https://github.com/agarwalvishal/claude-chat-exporter) — 单文件控制台脚本，其 `CLAUDE.md` 记录了迄今最完整的 claude.ai 内部 API 响应契约，且明确区分「已验证」与「假设」，本文的 [开源] 结论多出自此 +- [socketteer/Claude-Conversation-Exporter](https://github.com/socketteer/Claude-Conversation-Exporter) — Chrome 扩展，含列表接口与批量导出实现 +- [Emnolope/claude-conversation-export](https://github.com/Emnolope/claude-conversation-export) — `tree=True` 全分支导出 +- [Claude Platform Docs · Web search tool](https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool) / [Citations](https://platform.claude.com/docs/en/build-with-claude/citations) — API 版引用结构（网页版是否一致待测） diff --git a/docs/claude-probe.js b/docs/claude-probe.js new file mode 100644 index 0000000..3ff0f4a --- /dev/null +++ b/docs/claude-probe.js @@ -0,0 +1,172 @@ +// claude.ai 结构 / 限流探针 +// +// 用法：在 claude.ai 打开任意一条对话，F12 打开控制台，整段粘贴回车。 +// +// 它做两件事： +// 1. 打印接口返回的**字段骨架**（键名 + 类型），不打印任何对话内容 +// 2. 以保守节奏做一次小规模限流试探，观察服务端反应 +// +// 请求预算：结构探测 2 个，限流试探最多 6 个，全程 ≤ 8 个请求，间隔不低于 1s， +// 一旦吃到 429 立刻停止。这是刻意的——Claude 侧的限流阈值尚无实测数据， +// 探针本身绝不能成为触发限制的原因。 +// +// 结果里带 [待测] 标记的字段，对应 docs/claude-adapter-feasibility.md 的清单。 + +;(async () => { + const log = (...a) => console.log('%c[inkstone]', 'color:#1e6b72;font-weight:bold', ...a) + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) + + // ---------- 会话上下文 ---------- + const orgFromCookie = document.cookie.match(/lastActiveOrg=([^;]+)/)?.[1] + let org = orgFromCookie && decodeURIComponent(orgFromCookie) + try { + const r = await fetch('/api/organizations', { credentials: 'include' }) + const d = await r.json() + const list = Array.isArray(d) ? d : (d?.organizations ?? []) + log('组织数', list.length, '| cookie 与接口是否一致：', list[0]?.uuid === org) + if (list[0]?.uuid) org = list[0].uuid + } catch (e) { + log('取组织接口失败，改用 cookie：', String(e)) + } + if (!org) return log('拿不到组织 id，请确认已登录') + + const convId = location.pathname.split('/').pop() + if (!convId || convId.length < 20 || !convId.includes('-')) { + return log('请先打开一条具体对话再运行（地址形如 /chat/）') + } + + // ---------- 字段骨架 ---------- + // 只保留键名与类型，值一律不打印——探针不该把对话内容抄到控制台里 + const skeleton = (v, d = 0) => { + if (v === null) return 'null' + if (Array.isArray(v)) return v.length ? [skeleton(v[0], d + 1)] : [] + if (typeof v !== 'object') return typeof v + if (d > 4) return '…' + return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, skeleton(x, d + 1)])) + } + const getJson = async (url) => { + const t = performance.now() + const res = await fetch(url, { credentials: 'include', headers: { Accept: 'application/json' } }) + const ms = Math.round(performance.now() - t) + if (!res.ok) throw Object.assign(new Error(`HTTP ${res.status}`), { status: res.status, ms, res }) + return { data: await res.json(), ms } + } + + log('—— 1. 列表接口 ——') + try { + const { data: list, ms } = await getJson( + `/api/organizations/${org}/chat_conversations?limit=5&offset=0`, + ) + const arr = Array.isArray(list) ? list : [] + log(`耗时 ${ms}ms | 请求 limit=5，实际返回 ${arr.length} 条`) + log('[待测 1] 分页是否生效：返回条数 == 5 说明 limit 被认；远大于 5 说明服务端忽略了分页') + log('[待测 2] 列表项骨架', skeleton(arr[0])) + } catch (e) { + log('列表接口失败', e.status ?? String(e)) + } + + await sleep(1200) + + log('—— 2. 对话详情 ——') + let conv + try { + const r = await getJson( + `/api/organizations/${org}/chat_conversations/${convId}` + + `?tree=True&rendering_mode=messages&render_all_tools=true`, + ) + conv = r.data + log(`耗时 ${r.ms}ms | 消息数 ${conv.chat_messages?.length ?? 0}`) + } catch (e) { + return log('详情接口失败，后续探测中止', e.status ?? String(e)) + } + + const msgs = conv.chat_messages ?? [] + const blocks = msgs.flatMap((m) => m.content ?? []) + log('会话级键', Object.keys(conv)) + log('[待测 7] Projects 归属字段：', Object.keys(conv).filter((k) => /project/i.test(k))) + log('消息级键', [...new Set(msgs.flatMap(Object.keys))]) + log('内容块类型', [...new Set(blocks.map((b) => b.type))]) + log('tool_use 名称', [ + ...new Set(blocks.filter((b) => b.type === 'tool_use').map((b) => b.name)), + ]) + + for (const t of ['text', 'thinking', 'tool_use', 'tool_result']) { + const b = blocks.find((x) => x.type === t) + if (b) log(`${t} 块骨架`, skeleton(b)) + } + log('[待测 3] thinking 正文在哪个键：看上面 thinking 块骨架里哪个字段是 string') + log('[待测 4] citations 挂载位置：', skeleton(blocks.find((b) => b.citations?.length)?.citations?.[0])) + + const file = msgs.flatMap((m) => m.files ?? [])[0] + if (file) log('files[0] 骨架', skeleton(file)) + const att = msgs.flatMap((m) => m.attachments ?? [])[0] + if (att) log('attachments[0] 骨架', skeleton(att)) + + // [待测 6] 公式定界符：只看形态，不打印正文 + const mathSample = blocks + .filter((b) => b.type === 'text' && typeof b.text === 'string') + .map((b) => b.text) + .join('\n') + log('[待测 6] 公式定界符：', { + '$...$': /(? o.status === 429) + log( + throttled + ? '结论：这个节奏已经会被限流，Inkstone 的起步间距应保持在 1500ms 以上' + : '结论：4 次小规模请求未触发限流。这只说明「不算激进」，不代表全量抓取安全——' + + '要放宽并发/间距，必须再做更大规模的分级试跑', + ) + log('把上面的输出贴进 issue 或 docs/claude-adapter-feasibility.md 的待测清单里') +})() diff --git a/src/api.ts b/src/api.ts index 4bc7198..e99525f 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,121 +1,50 @@ -import type { - ConversationDetail, - ConversationListItem, - ConversationListPage, - GizmoConversationsPage, - GizmoSidebarPage, - ProjectInfo, - SessionResponse, -} from './types' - -class ApiError extends Error { - constructor( - readonly status: number, - message: string, - ) { - super(message) - this.name = 'ApiError' - } -} - -export class CancelledError extends Error { - constructor() { - super('已取消') - this.name = 'CancelledError' - } -} - -export interface CancelToken { - cancelled: boolean +// ChatGPT backend-api 客户端。节奏控制在 core/fetcher，这里只负责端点与字段。 + +import { + ApiError, + CancelledError, + createFetcher, + ensureAlive, + fetchBinary as fetchBinaryWith, + jitter, + mapConcurrent, + SizeLimitError, + sleep, + type CancelToken, + type Fetcher, + type ThrottleConfig, +} from './core/fetcher' +import type { ConversationDetail, ConversationListItem, ConversationListPage, SessionResponse } from './types' + +export { + ApiError, + CancelledError, + SizeLimitError, + ensureAlive, + jitter, + mapConcurrent, + sleep, + type CancelToken, } -export const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) -const jitter = (base: number, spread = base): number => base + Math.random() * spread - -export function ensureAlive(cancel?: CancelToken): void { - if (cancel?.cancelled) throw new CancelledError() -} - -// ===== 全局限速 ===== // 后端是突发桶型限流，且持续高频抓取会触发账号级反滥用（实测：旧对话渐进式 -// 变 429→404、列表截断，恢复要数小时）。所以宁慢勿快： -// 1) 所有请求共享起跑间距；2) 一旦吃到 429，间距自适应放大且不回落； -// 3) 任何全局性 429 让全部 worker 共享冷却。 -const REQUEST_SPACING_BASE_MS = 800 -const REQUEST_SPACING_MAX_MS = 4000 -// 喘息暂停：贴合突发桶回填节奏，每 ~80 个请求整体歇一段 -const REST_EVERY_N_REQUESTS = 80 -const REST_DURATION_MS = 25_000 -let requestSpacingMs = REQUEST_SPACING_BASE_MS -let requestsSinceRest = 0 -let nextSlotAt = 0 -let cooldownUntil = 0 - -/** 429 后调用：全局节奏永久放慢（本次运行内不回落）。 */ -function slowDown(): void { - requestSpacingMs = Math.min(requestSpacingMs * 1.5, REQUEST_SPACING_MAX_MS) +// 变 429→404、列表截断，恢复要数小时）。所以宁慢勿快。 +export const CHATGPT_THROTTLE: ThrottleConfig = { + spacingBaseMs: 800, + spacingMaxMs: 4000, + // 喘息暂停：贴合突发桶回填节奏，每 ~80 个请求整体歇一段 + restEveryN: 80, + restDurationMs: 25_000, + maxAttempts: 7, } -async function acquireSlot(cancel?: CancelToken): Promise { - for (;;) { - ensureAlive(cancel) - const now = Date.now() - const target = Math.max(nextSlotAt, cooldownUntil) - if (now >= target) { - nextSlotAt = now + jitter(requestSpacingMs, requestSpacingMs * 0.4) - if (++requestsSinceRest >= REST_EVERY_N_REQUESTS) { - requestsSinceRest = 0 - cooldownUntil = Math.max(cooldownUntil, now + REST_DURATION_MS) - } - return - } - await sleep(Math.min(target - now, 500)) - } -} - -// 跨 URL 连续 429 计数：区分「全局限流」和「条目级 429」的关键信号 -let global429Streak = 0 +const fetcher: Fetcher = createFetcher(CHATGPT_THROTTLE) -// 429/5xx 指数退避重试；页内同源 fetch 自带登录 cookie。 -// 实测教训：部分对话会**永久性 429/404**（条目级问题，同一时刻其他请求全 200）， -// 把它们当全局限流会拖停整条流水线——所以： -// - 带 Retry-After 的 429 → 真全局信号，共享冷却 -// - 不带 Retry-After 的 429 → 条目级，快速放弃（结尾重试环节还有一次机会） -// - 跨 URL 连续多次 429 → 无头全局限流的兜底，短冷却 -async function backoffFetch(url: string, init: RequestInit = {}, cancel?: CancelToken): Promise { - let delay = 2000 - let headerless429s = 0 - for (let attempt = 0; ; attempt++) { - await acquireSlot(cancel) - const res = await fetch(url, { credentials: 'include', ...init }) - if (res.ok) { - global429Streak = 0 - return res - } - const retryable = res.status === 429 || res.status >= 500 - if (!retryable || attempt >= 7) throw new ApiError(res.status, `HTTP ${res.status}: ${url}`) - if (res.status === 429) { - global429Streak++ - slowDown() - const retryAfterMs = Number(res.headers.get('retry-after')) * 1000 - if (retryAfterMs > 0) { - cooldownUntil = Math.max(cooldownUntil, Date.now() + retryAfterMs) - } else if (global429Streak >= 5) { - cooldownUntil = Math.max(cooldownUntil, Date.now() + 15_000) - } else { - headerless429s++ - if (headerless429s > 1) throw new ApiError(429, `HTTP 429（条目级，快速放弃）: ${url}`) - await sleep(jitter(delay)) - } - } else { - await sleep(jitter(delay)) - } - delay = Math.min(delay * 2, 30_000) - } -} +/** 当前节奏快照（面板用来显示限流观测）。 */ +export const throttleStats = (): ReturnType => fetcher.stats() export async function getAccessToken(cancel?: CancelToken): Promise { - const res = await backoffFetch(`${location.origin}/api/auth/session`, {}, cancel) + const res = await fetcher.request(`${location.origin}/api/auth/session`, {}, cancel) const data = (await res.json()) as SessionResponse if (!data.accessToken) throw new Error('拿不到 accessToken：请确认已登录 ChatGPT 后重试') return data.accessToken @@ -123,255 +52,114 @@ export async function getAccessToken(cancel?: CancelToken): Promise { const auth = (token: string) => ({ Authorization: `Bearer ${token}` }) -async function listConversationsPage( +export async function listConversationsPage( token: string, offset: number, limit: number, cancel?: CancelToken, ): Promise { const url = `${location.origin}/backend-api/conversations?offset=${offset}&limit=${limit}&order=updated` - const res = await backoffFetch(url, { headers: auth(token) }, cancel) + const res = await fetcher.request(url, { headers: auth(token) }, cancel) return (await res.json()) as ConversationListPage } -// ===== Projects（gizmo）===== -// 主列表接口只返回侧栏「Chats」那份平铺列表，project 里的会话必须按 project -// 逐个走 gizmos 接口拿。两个坐标系完全不同：主列表是 offset，gizmos 是字符串游标。 -const PROJECT_PAGE_LIMIT = 50 // gizmos 接口的 limit 上限比主列表小 - -// gizmo_id → project 名：对话详情只带 gizmo_id，名字要靠 sidebar 接口换； -// listProjects 每拉一次就往这里补，读的一方不用关心何时拉的 -const projectNames = new Map() - -/** 已知的 project 名（需先 listProjects 拉过）；不在 project 侧栏里的 gizmo 返回 undefined。 */ -export const projectNameOf = (gizmoId: string | null | undefined): string | undefined => - gizmoId ? projectNames.get(gizmoId) : undefined - -// 面板打开时「来源」下拉与归并分页器会几乎同时要这份列表，并发的合成一次请求； -// 只合并「正在飞」的，不缓存已完成的结果，避免新建项目后拉到陈数据 -let projectsInFlight: Promise | null = null - -/** 拉全部 project（顺带填好 gizmo_id → 名字的映射）。 */ -export function listProjects(token: string, cancel?: CancelToken): Promise { - projectsInFlight ??= fetchProjects(token, cancel).finally(() => { - projectsInFlight = null - }) - return projectsInFlight -} - -async function fetchProjects(token: string, cancel?: CancelToken): Promise { - const out: ProjectInfo[] = [] - let cursor: number | null = null +export async function listAllConversations( + token: string, + onProgress?: (fetched: number) => void, + cancel?: CancelToken, +): Promise { + const all: ConversationListItem[] = [] + let offset = 0 + let limit = 100 + let emptyRetries = 0 + // 注意：接口的 total 字段不可靠（实测翻页途中返回 offset+len+1）， + // 终止条件只认「空页」或「不足一页」。 for (;;) { ensureAlive(cancel) - // conversations_per_gizmo=0：只要 project 本身，别顺带回一堆用不上的会话 - const url = - `${location.origin}/backend-api/gizmos/snorlax/sidebar?conversations_per_gizmo=0` + - (cursor == null ? '' : `&cursor=${encodeURIComponent(String(cursor))}`) - const res = await backoffFetch(url, { headers: auth(token) }, cancel) - const page = (await res.json()) as GizmoSidebarPage - for (const entry of page.items ?? []) { - const g = entry.gizmo?.gizmo - if (!g?.id) continue - const name = (g.display?.name ?? '').trim() || '未命名项目' - projectNames.set(g.id, name) - out.push({ id: g.id, name }) + let page: ConversationListPage + try { + page = await listConversationsPage(token, offset, limit, cancel) + } catch (e) { + // limit 上限历史上收紧过；非限流的 4xx 先降到 50 重试一次 + if (e instanceof ApiError && e.status >= 400 && e.status < 500 && e.status !== 429 && limit > 50) { + limit = 50 + continue + } + throw e } - cursor = page.cursor ?? null - if (cursor == null) return out + const items = page.items ?? [] + all.push(...items) + onProgress?.(all.length) + // 服务端可能按自己的上限截页（返回数 < 请求 limit 不代表到底），只认空页； + // 而且列表索引实测会瞬时降级、提前返回空页/短列表（对话本身还在）， + // 所以空页也不轻信，隔几秒重试确认，连续空 3 次才算到底。 + if (items.length === 0) { + if (all.length === 0 || emptyRetries >= 2) break + emptyRetries++ + await sleep(4000 * emptyRetries) + continue + } + emptyRetries = 0 + offset += items.length } -} - -async function listProjectConversationsPage( - token: string, - gizmoId: string, - cursor: string, - cancel?: CancelToken, -): Promise { - const url = - `${location.origin}/backend-api/gizmos/${encodeURIComponent(gizmoId)}/conversations` + - `?cursor=${encodeURIComponent(cursor)}&limit=${PROJECT_PAGE_LIMIT}` - const res = await backoffFetch(url, { headers: auth(token) }, cancel) - return (await res.json()) as GizmoConversationsPage + return all } export interface ConversationPager { - /** 拉下一页；done=true 表示所有来源都到底（此后再调直接返回空页 + done） */ + /** 拉下一页；done=true 表示已确认到底（此后再调直接返回空页 + done） */ next(): Promise<{ items: ConversationListItem[]; done: boolean }> } -// 分页来源的两个固定值，其余按 gizmo id 视为单个 project。 -// 值必须与 ui.ts 面板里「来源」下拉的 option value 一致。 -const SOURCE_ALL = 'all' -const SOURCE_MAIN = 'main' - -/** 归一到 epoch 秒：列表接口给 ISO 字符串，详情接口给数字；取不到时间的排最后。 */ -function timeOf(i: ConversationListItem): number { - const t = i.update_time ?? i.create_time - if (typeof t === 'number') return t - if (typeof t === 'string') { - const ms = Date.parse(t) - return Number.isNaN(ms) ? -Infinity : ms / 1000 - } - return -Infinity -} - -/** 一个来源的页流：内部缓冲一页，对外只看得到队头。 */ -interface SourceStream { - done: boolean - peek(): ConversationListItem | undefined - take(): ConversationListItem - /** 补一页；拿不到新页就置 done */ - fill(): Promise -} - -/** nextPage 返回 null 表示该源到底（空数组只是本页无内容，还要再问一次）。 */ -function makeStream(nextPage: () => Promise): SourceStream { - const buf: ConversationListItem[] = [] - const stream: SourceStream = { - done: false, - peek: () => buf[0], - take: () => buf.shift()!, - async fill() { - const page = await nextPage() - if (page == null) stream.done = true - else buf.push(...page) - }, - } - return stream -} - /** - * 多源惰性分页器：主列表（offset 分页）与每个 project（字符串游标）各是一条流， - * 按 update_time 做 k 路归并，交出一条严格时间倒序的列表（前提：各源自身按更新 - * 时间倒序，两个接口都是）。代价是首屏要先把每个源都拉一页（N+2 个请求）， - * 之后大多数 next() 只消耗缓冲区。 - * source 可把范围收窄到单一来源（面板上的「来源」下拉）；指定单个 project 时 - * 连 project 列表都不用拉，归并退化成单条流。 - * 跨源按 id 去重（主列表哪天开始包含 project 会话也不会重复导出）。 - * 主列表的终止条件只认空页（服务端会按自己的上限截页，短页不代表到底），且 - * 列表索引实测会瞬时降级提前返回空页，所以空页要隔几秒重试，连续空 3 次才算到底。 + * 惰性分页器：把 listAllConversations 的翻页与防御逻辑逐页化，供「选择对话」 + * 的懒加载使用（切到「选择」不再一次性翻完全部页）。终止条件与全量版一致： + * 只认空页，且空页要隔几秒重试确认，连续空 3 次才算到底。 */ -export function createConversationPager( - token: string, - cancel?: CancelToken, - source: string = SOURCE_ALL, -): ConversationPager { - const onlyProject = source === SOURCE_ALL || source === SOURCE_MAIN ? null : source - const seen = new Set() +export function createConversationPager(token: string, cancel?: CancelToken): ConversationPager { let offset = 0 let limit = 100 let emptyRetries = 0 - let streams: SourceStream[] | null = null let done = false - - /** 主列表一页；null = 主列表到底 */ - async function mainPage(): Promise { - for (;;) { - ensureAlive(cancel) - let page: ConversationListPage - try { - page = await listConversationsPage(token, offset, limit, cancel) - } catch (e) { - // limit 上限历史上收紧过；非限流的 4xx 先降到 50 重试一次 - if (e instanceof ApiError && e.status >= 400 && e.status < 500 && e.status !== 429 && limit > 50) { - limit = 50 - continue - } - throw e - } - const items = page.items ?? [] - if (items.length === 0) { - // 首页即空 = 账号真没对话；否则可能是列表索引瞬时降级，隔几秒重试确认 - if (offset === 0 || emptyRetries >= 2) return null - emptyRetries++ - await sleep(4000 * emptyRetries) - continue - } - emptyRetries = 0 - offset += items.length - return items - } - } - - /** 某个 project 的页源（自带游标）；null = 该 project 到底 */ - function projectPages(gizmoId: string): () => Promise { - let cursor: string | null = '0' - return async () => { - if (cursor == null) return null - ensureAlive(cancel) - const page = await listProjectConversationsPage(token, gizmoId, cursor, cancel) - cursor = page.cursor ?? null - // gizmos 接口的条目不保证带 gizmo_id，补上才能在下游认出归属 - return (page.items ?? []).map((i) => ({ ...i, gizmo_id: i.gizmo_id ?? gizmoId })) - } - } - - async function buildStreams(): Promise { - if (onlyProject != null) return [makeStream(projectPages(onlyProject))] - if (source === SOURCE_MAIN) return [makeStream(mainPage)] - const projects = await listProjects(token, cancel) - return [makeStream(mainPage), ...projects.map((p) => makeStream(projectPages(p.id)))] - } - return { async next() { if (done) return { items: [], done: true } - streams ??= await buildStreams() for (;;) { - // 每个未到底的源都得有队头，否则没法知道谁才是全局最新的那条 - for (const s of streams) { - while (!s.done && s.peek() === undefined) await s.fill() - } - const out: ConversationListItem[] = [] - for (;;) { - let best: SourceStream | undefined - for (const s of streams) { - const head = s.peek() - if (head === undefined) continue - if (best === undefined || timeOf(head) > timeOf(best.peek()!)) best = s + ensureAlive(cancel) + let page: ConversationListPage + try { + page = await listConversationsPage(token, offset, limit, cancel) + } catch (e) { + if (e instanceof ApiError && e.status >= 400 && e.status < 500 && e.status !== 429 && limit > 50) { + limit = 50 + continue } - if (best === undefined) { + throw e + } + const items = page.items ?? [] + if (items.length === 0) { + // 首页即空 = 账号真没对话；否则可能是列表索引瞬时降级，隔几秒重试确认 + if (offset === 0 || emptyRetries >= 2) { done = true - break + return { items: [], done: true } } - // offset 翻页 + order=updated 期间列表会漂移，加上跨源重叠，统一在这里去重 - const item = best.take() - if (!seen.has(item.id)) { - seen.add(item.id) - out.push(item) - } - // 队头空了又没到底：再取就得发请求，这一页到此为止，保住惰性加载 - if (best.peek() === undefined && !best.done) break + emptyRetries++ + await sleep(4000 * emptyRetries) + continue } - // 整页都是重复时继续翻，不把空页交给调用方（会被当成到底） - if (out.length > 0 || done) return { items: out, done } + emptyRetries = 0 + offset += items.length + return { items, done: false } } }, } } -export async function listAllConversations( - token: string, - onProgress?: (fetched: number) => void, - cancel?: CancelToken, -): Promise { - const pager = createConversationPager(token, cancel) - const all: ConversationListItem[] = [] - for (;;) { - const { items, done } = await pager.next() - all.push(...items) - if (items.length > 0) onProgress?.(all.length) - if (done) return all - } -} - export async function fetchConversation( token: string, id: string, cancel?: CancelToken, ): Promise { - const res = await backoffFetch( + const res = await fetcher.request( `${location.origin}/backend-api/conversation/${id}`, { headers: auth(token) }, cancel, @@ -390,7 +178,7 @@ export async function resolveFileDownload( fileId: string, cancel?: CancelToken, ): Promise { - const res = await backoffFetch( + const res = await fetcher.request( `${location.origin}/backend-api/files/${fileId}/download`, { headers: auth(token) }, cancel, @@ -406,57 +194,10 @@ export async function resolveFileDownload( return { url: data.download_url, filename } } -export class SizeLimitError extends Error { - constructor(readonly actualBytes: number) { - super(`附件大小 ${actualBytes} 字节超出上限`) - this.name = 'SizeLimitError' - } -} - -/** 附件元数据里的 size 不可靠（library 文件报 0），上限以实际传输为准。 */ -export async function fetchBinary( +export function fetchBinary( url: string, cancel?: CancelToken, maxBytes?: number, ): Promise<{ bytes: Uint8Array; contentType: string | null }> { - const res = await backoffFetch(url, {}, cancel) - const declared = Number(res.headers.get('content-length')) - if (maxBytes != null && declared > maxBytes) { - try { - await res.body?.cancel() - } catch { - /* 取消流失败无所谓 */ - } - throw new SizeLimitError(declared) - } - const bytes = new Uint8Array(await res.arrayBuffer()) - if (maxBytes != null && bytes.length > maxBytes) throw new SizeLimitError(bytes.length) - return { bytes, contentType: res.headers.get('content-type') } -} - -// 简易并发池：fn 抛错即整体中止（逐条的容错由调用方在 fn 里自己 catch） -export async function mapConcurrent( - items: readonly T[], - concurrency: number, - fn: (item: T, index: number) => Promise, - cancel?: CancelToken, -): Promise { - let next = 0 - let aborted: unknown = null - const n = Math.max(1, Math.min(concurrency, items.length)) - const worker = async (): Promise => { - while (aborted == null && !cancel?.cancelled) { - const i = next++ - if (i >= items.length) return - try { - await fn(items[i]!, i) - } catch (e) { - aborted = e - return - } - } - } - await Promise.all(Array.from({ length: n }, () => worker())) - if (aborted != null) throw aborted - ensureAlive(cancel) + return fetchBinaryWith(fetcher, url, cancel, maxBytes) } diff --git a/src/convert/citations.ts b/src/convert/citations.ts index 10ad1a3..61b98f1 100644 --- a/src/convert/citations.ts +++ b/src/convert/citations.ts @@ -1,5 +1,8 @@ +import type { SourceLink } from '../core/ir' import type { ContentReference, ContentReferenceItem } from '../types' +export type { SourceLink } + // ChatGPT 用私有区 Unicode（U+E200 区段）包裹引用标记，如 citeturn0search1。 // 源码里不能出现这些不可见字面量（编辑器/工具链会悄悄弄坏它们），统一用码点构造。 const cp = (n: number) => String.fromCharCode(n) @@ -9,11 +12,6 @@ const PUA_SEP = cp(0xe202) const PUA_ANY = new RegExp(`[${cp(0xe000)}-${cp(0xf8ff)}]`, 'g') const LEGACY_CITATION = /【[^【】\n]*†[^【】\n]*】/g // 【12†source】 -export interface SourceLink { - title: string - url: string -} - export interface RestoreResult { text: string sources: SourceLink[] diff --git a/src/convert/markdown.ts b/src/convert/markdown.ts index 4204916..4440309 100644 --- a/src/convert/markdown.ts +++ b/src/convert/markdown.ts @@ -1,397 +1,34 @@ -import type { - AttachmentMeta, - ContentReference, - ConversationDetail, - ImageAssetPart, - Message, - MessageContent, -} from '../types' -import { groupTurns, linearize, type Turn } from './linearize' -import { convertMath } from './math' -import { transformHeadings, type HeadingMode } from './headings' -import { restoreCitations, stripResidualMarkers, type SourceLink } from './citations' -import { mapTextSegmentsOutsideCode } from './codeaware' -import { replayCanvas, type CanvasOp } from './canvas' - -export interface AssetRef { - fileId: string - kind: 'image' | 'file' - name?: string - sizeBytes?: number - mime?: string -} - -export interface ConvertResult { - markdown: string - title: string - /** 正文里以 assetToken 占位，待下载后由调用方替换成真实链接 */ - assets: AssetRef[] -} - -export interface ConvertOptions { - /** 是否把思维链（thoughts）写入导出，默认不写入（打开后折叠 callout） */ - thoughts?: boolean - /** 消息内标题处理：demote 整体降一级（默认）/ strip 全部剥离为加粗行 */ - headingMode?: HeadingMode - /** 是否写入工具运行痕迹（发给工具的代码/搜索请求与运行输出），默认不写入 */ - toolTraces?: boolean - /** 所属 project 名（仅 project 会话有）；写进 frontmatter 供 Dataview 分组 */ - projectName?: string -} - -export type LinkStyle = 'wikilink' | 'markdown' - -/** 附件链接统一出口：油猴端与离线 CLI 共用，保证两条管道产出一致。 */ -export function assetLink( - style: LinkStyle, - path: string, - opts: { label?: string; embed?: boolean } = {}, -): string { - if (style === 'markdown') { - const label = escapeLinkLabel(opts.label ?? path.split('/').pop() ?? path) - return `${opts.embed ? '!' : ''}[${label}](${encodeURI(path)})` - } - if (opts.embed) return `![[${path}]]` - // wikilink 别名里 |、] 会破坏链接语法 - const alias = opts.label?.replace(/[[\]|]/g, '-') - return `[[${path}${alias ? `|${alias}` : ''}]]` -} - -/** 附件占位符：转换层不做网络请求，下载与链接改写由调用方完成。 */ -export const assetToken = (fileId: string): string => `%%INKSTONE-ASSET-${fileId}%%` - -// 控制字符用码点构造，避免源码里出现不可见字面量 -const CONTROL_CHARS = new RegExp(`[${String.fromCharCode(0)}-${String.fromCharCode(31)}]`, 'g') - -interface RenderCtx { - sources: SourceLink[] - assets: AssetRef[] - thoughts: boolean - toolTraces: boolean - headingMode: HeadingMode - /** msgId → 重放成功的 Canvas 操作；重放失败的 canmore 消息走原始 JSON 兜底 */ - canvas: Map -} +// ChatGPT 侧的转换入口（兼容壳）。 +// +// 实现已拆成两段：`sites/chatgpt/convert` 把 backend-api JSON 解成站点无关的 IR， +// `core/render` 把 IR 渲染成 Markdown。这里只保留原有的导出签名， +// 让离线 CLI 与既有测试无需改动地继续工作。 + +import { conversationToIR } from '../sites/chatgpt/convert' +import { + renderConversation, + type ConvertOptions, + type ConvertResult, + type LinkStyle, +} from '../core/render' +import type { ConversationDetail } from '../types' + +export { + assetLink, + assetToken, + filenameFor, + sanitizeName, + sanitizeSubdir, + type ConvertOptions, + type ConvertResult, + type LinkStyle, +} from '../core/render' +export type { AssetRef } from '../core/ir' export function conversationToMarkdown( conv: ConversationDetail, fallbackId = '', copts: ConvertOptions = {}, ): ConvertResult { - const convId = String(conv.conversation_id ?? conv.id ?? fallbackId) - const title = (conv.title ?? '').trim() || 'Untitled' - const messages = linearize(conv) - // 消息级 model_slug 才是实际生成回复的模型（default_model_slug 只是对话的默认档位，仅作回退）； - // 中途切换过模型时以最后一条为准——须在去重前取（Set 保留首现顺序，A→B→A 会错取 B）， - // 去重序列只用于 models 列表 - const rawSlugs = messages - .filter((m) => m.author.role === 'assistant' && m.metadata?.model_slug) - .map((m) => m.metadata!.model_slug!) - const modelSlugs = [...new Set(rawSlugs)] - const model = rawSlugs[rawSlugs.length - 1] ?? conv.default_model_slug - - const ctx: RenderCtx = { - sources: [], - assets: [], - thoughts: copts.thoughts === true, - toolTraces: copts.toolTraces === true, - headingMode: copts.headingMode ?? 'demote', - canvas: replayCanvas(messages), - } - const turns = groupTurns(messages) - let body = turns - .map((t) => renderTurn(t, ctx)) - .filter((s): s is string => s != null) - .join('\n\n') - - const sources = dedupeSources(ctx.sources) - if (sources.length > 0) { - body += `\n\n# Sources\n\n${sources.map((s) => `- [${escapeLinkLabel(s.title)}](${s.url})`).join('\n')}` - } - - // Branch · 对话：链接回父对话的导出文件（文件名规则可预测），Obsidian 图谱直接连起来 - const branchMeta = messages.map((m) => m.metadata).find((md) => md?.branching_from_conversation_id) - const branchedFrom = branchMeta - ? filenameFor( - branchMeta.branching_from_conversation_title ?? '', - branchMeta.branching_from_conversation_id!, - ).replace(/\.md$/, '') - : null - - // 收尾排版：正文里 3 连以上空行压成 1 个空行（代码块内不动） - const tidied = mapTextSegmentsOutsideCode(body, (s) => s.replace(/\n{3,}/g, '\n\n')) - - // project / 自定义 GPT 的会话网址带 gizmo 段，写平铺版会丢掉归属信息 - const gizmoId = conv.gizmo_id ?? null - - const fm = [ - '---', - `title: ${yamlQuote(title)}`, - `chat_id: ${convId}`, - `url: https://chatgpt.com${gizmoId ? `/g/${gizmoId}` : ''}/c/${convId}`, - copts.projectName ? `project: ${yamlQuote(copts.projectName)}` : null, - `created: ${toIso(conv.create_time)}`, - `updated: ${toIso(conv.update_time)}`, - model ? `model: ${model}` : null, - modelSlugs.length > 1 ? `models:\n${modelSlugs.map((s) => ` - ${s}`).join('\n')}` : null, - branchedFrom ? `branched_from: ${yamlQuote(`[[${branchedFrom}]]`)}` : null, - branchMeta - ? `branched_from_url: https://chatgpt.com/c/${branchMeta.branching_from_conversation_id}` - : null, - 'tags:', - ' - chatgpt', - '---', - ] - .filter((l): l is string => l != null) - .join('\n') - - return { markdown: `${fm}\n\n${tidied.trim()}\n`, title, assets: ctx.assets } -} - -/** `标题-短id.md`：防重名，且 id 稳定保证增量重导时覆盖同一文件。 */ -export function filenameFor(title: string, convId: string): string { - const safe = sanitizeName(title).slice(0, 80).replace(/-+$/, '') || 'Untitled' - return `${safe}-${convId.slice(0, 8)}.md` -} - -/** 文件名净化：非法字符（跨平台 + Obsidian 链接敏感）与空白统一归一为 `-`，不留空格。 */ -export function sanitizeName(name: string): string { - return name - .replace(CONTROL_CHARS, '') - .replace(/[/\\:*?"<>|#^[\]\s]+/g, '-') - .replace(/-{2,}/g, '-') - .replace(/^[-.]+|-+$/g, '') -} - -/** - * 子文件夹设置净化：按 `/` 分段逐段过 sanitizeName（`.`/`..` 被清成空段丢弃，防目录逃逸）， - * 允许 `a/b` 嵌套；返回 `''` 表示不套子文件夹。 - */ -export function sanitizeSubdir(input: string): string { - return input - .split('/') - .map((seg) => sanitizeName(seg)) - .filter((seg) => seg !== '') - .join('/') -} - -function renderTurn(turn: Turn, ctx: RenderCtx): string | null { - const rendered = turn.messages - .map((m) => renderMessage(m, ctx)) - .filter((s): s is string => s != null && s.trim() !== '') - if (rendered.length === 0) return null - const heading = turn.role === 'user' ? '# User' : '# ChatGPT' - return [heading, ...rendered].join('\n\n') -} - -function renderMessage(msg: Message, ctx: RenderCtx): string | null { - const c = msg.content - const recipient = msg.recipient ?? 'all' - const refs = msg.metadata?.content_references - const blocks: string[] = [] - const inlineImageIds = new Set() - - // canmore 工具的确认回执（role=tool）：内容已由重放侧呈现，不重复 - if (msg.author.role === 'tool' && (msg.author.name ?? '').startsWith('canmore.')) return null - - switch (c.content_type) { - case 'text': { - const raw = joinTextParts(c) - if (msg.author.role === 'assistant' && recipient.startsWith('canmore.')) { - // Canvas：patch 重放还原终稿；重放失败回退原始 JSON 折叠嵌入 - const op = ctx.canvas.get(msg.id) - if (op) { - const rendered = renderCanvasOp(op, ctx) - if (rendered != null) blocks.push(rendered) - } else { - blocks.push( - callout('example', `工具调用 → \`${recipient}\``, fence(stripResidualMarkers(raw)), true), - ) - } - } else if (msg.author.role === 'assistant' && recipient !== 'all') { - // 联网等其他工具调用载荷（多为 JSON）：默认不写入，toolTraces 打开时整块折叠嵌入 - if (ctx.toolTraces) { - blocks.push(callout('example', `工具调用 → \`${recipient}\``, fence(stripResidualMarkers(raw)), true)) - } - } else { - blocks.push(renderProse(raw, refs, ctx)) - } - break - } - case 'multimodal_text': - for (const p of c.parts ?? []) { - if (typeof p === 'string') { - const s = renderProse(p, refs, ctx) - if (s.trim() !== '') blocks.push(s) - } else { - const rendered = renderImageAsset(p, ctx) - blocks.push(rendered.block) - if (rendered.fileId) inlineImageIds.add(rendered.fileId) - } - } - break - case 'code': - // content_type=code 都是工具调用载荷（代码解释器 python、联网检索 search_query/open/click 等）， - // 随 toolTraces 开关；折叠 callout 包裹，与其他工具痕迹一致 - if (ctx.toolTraces) { - blocks.push( - callout( - 'example', - `工具调用 → \`${recipient}\``, - fence(c.text ?? '', codeLanguage(c, recipient)), - true, - ), - ) - } - break - case 'execution_output': - if (ctx.toolTraces) { - blocks.push(callout('note', '运行输出', fence(stripResidualMarkers(c.text ?? '')), true)) - } - break - case 'thoughts': { - if (!ctx.thoughts) break - const t = renderThoughts(c, refs, ctx) - if (t != null) blocks.push(t) - break - } - default: - // 未知类型：原始 JSON 塞进折叠 callout，永不静默丢内容 - blocks.push( - callout( - 'warning', - `未识别的内容类型 \`${c.content_type}\`（原始 JSON）`, - fence(JSON.stringify(c, null, 2), 'json'), - true, - ), - ) - } - - // 用户上传的附件（图片已在正文里内联的不重复列出） - const attachments = (msg.metadata?.attachments ?? []).filter((a) => a?.id && !inlineImageIds.has(a.id)) - if (attachments.length > 0) { - blocks.push(attachments.map((a) => `- ${registerFileAsset(a, ctx)}`).join('\n')) - } - - const out = blocks.filter((s) => s.trim() !== '').join('\n\n') - return out === '' ? null : out -} - -function renderProse(raw: string, refs: ContentReference[] | undefined, ctx: RenderCtx): string { - const { text, sources } = restoreCitations(raw, refs) - ctx.sources.push(...sources) - return transformHeadings(convertMath(text), ctx.headingMode) -} - -/** Canvas 操作的呈现：终稿整块嵌入（document 走排版管道，code 走围栏），中间版本一行说明。 */ -function renderCanvasOp(op: CanvasOp, ctx: RenderCtx): string | null { - if (op.kind === 'comment') { - const body = (op.comments ?? []).map((c) => `- ${c.comment}`).join('\n') - return callout('example', `Canvas 批注${op.docName ? ` · ${op.docName}` : ''}`, body, true) - } - if (op.finalContent != null) { - const lang = op.docType.startsWith('code/') ? op.docType.slice('code/'.length) : '' - const body = op.docType.startsWith('code/') - ? fence(op.finalContent, lang) - : transformHeadings(convertMath(op.finalContent), ctx.headingMode) - return callout('abstract', `Canvas · ${op.docName}`, body) - } - return op.kind === 'create' ? `*(Canvas 创建「${op.docName}」，终稿见后)*` : `*(Canvas 更新「${op.docName}」，终稿见后)*` -} - -function renderThoughts( - c: MessageContent, - refs: ContentReference[] | undefined, - ctx: RenderCtx, -): string | null { - const blocks = (c.thoughts ?? []) - .map((t) => { - const head = t.summary?.trim() ? `**${t.summary.trim()}**\n\n` : '' - return head + renderProse(t.content ?? '', refs, ctx) - }) - .filter((s) => s.trim() !== '') - if (blocks.length === 0) return null - return callout('quote', '思考过程', blocks.join('\n\n'), true) -} - -function renderImageAsset(p: ImageAssetPart, ctx: RenderCtx): { block: string; fileId: string | null } { - const pointer = typeof p.asset_pointer === 'string' ? p.asset_pointer : '' - const fileId = pointer.split('//')[1] ?? '' - if (!fileId) { - // 没有可下载指针的多模态 part（音频等）：塞原始 JSON，不丢内容 - return { - block: callout( - 'warning', - `未识别的多模态 part \`${p.content_type}\`（原始 JSON）`, - fence(JSON.stringify(p, null, 2), 'json'), - true, - ), - fileId: null, - } - } - ctx.assets.push({ - fileId, - kind: 'image', - sizeBytes: typeof p.size_bytes === 'number' ? p.size_bytes : undefined, - }) - return { block: assetToken(fileId), fileId } -} - -function registerFileAsset(a: AttachmentMeta, ctx: RenderCtx): string { - ctx.assets.push({ - fileId: a.id, - kind: 'file', - name: a.name ?? undefined, - sizeBytes: typeof a.size === 'number' ? a.size : undefined, - mime: a.mime_type ?? undefined, - }) - return assetToken(a.id) -} - -function joinTextParts(c: MessageContent): string { - return (c.parts ?? []).filter((p): p is string => typeof p === 'string').join('\n') -} - -function codeLanguage(c: MessageContent, recipient: string): string { - const lang = (c.language ?? '').trim() - if (lang && lang !== 'unknown') return lang - return recipient === 'python' ? 'python' : '' -} - -function fence(code: string, lang = ''): string { - // 围栏比内容里最长的反引号串再长一格，避免被内容截断 - const longest = (code.match(/`+/g) ?? []).reduce((n, run) => Math.max(n, run.length), 2) - const f = '`'.repeat(Math.max(3, longest + 1)) - return `${f}${lang}\n${code.replace(/\n$/, '')}\n${f}` -} - -function callout(type: string, title: string, body: string, folded = false): string { - const head = `> [!${type}]${folded ? '-' : ''} ${title}` - const quoted = body - .split('\n') - .map((l) => (l === '' ? '>' : `> ${l}`)) - .join('\n') - return `${head}\n${quoted}` -} - -function dedupeSources(sources: SourceLink[]): SourceLink[] { - const seen = new Map() - for (const s of sources) { - if (!seen.has(s.url)) seen.set(s.url, s) - } - return [...seen.values()] -} - -function escapeLinkLabel(s: string): string { - return s.replace(/([[\]])/g, '\\$1') -} - -function yamlQuote(s: string): string { - return `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"` -} - -function toIso(t: number | string | null | undefined): string { - if (t == null || t === '') return '' - const d = typeof t === 'number' ? new Date(t * 1000) : new Date(t) - return Number.isNaN(d.getTime()) ? '' : d.toISOString() + return renderConversation(conversationToIR(conv, fallbackId), copts) } diff --git a/src/core/fetcher.ts b/src/core/fetcher.ts new file mode 100644 index 0000000..62a9b09 --- /dev/null +++ b/src/core/fetcher.ts @@ -0,0 +1,224 @@ +// 站点无关的取数内核：限速、退避、并发池、取消。 +// +// 这一层是本项目最贵的资产——ChatGPT 侧 344 + 432 对话实测踩出来的教训 +// （见 PLAN.md 实战经验）。换站点时端点会变、字段会变，但下面这套节奏控制 +// 的形状不变，所以它值得站点无关： +// 1) 所有请求共享起跑间距，宁慢勿快 +// 2) 一旦吃到 429，间距自适应放大且本次运行内不回落 +// 3) 带 Retry-After 的 429 = 真全局信号，共享冷却 +// 不带 Retry-After 的 429 = 条目级问题，快速放弃（别拖停整条流水线） +// 跨 URL 连续多次 429 = 无头全局限流的兜底，短冷却 +// +// 每个站点持有自己的 Fetcher 实例：节奏参数互不干扰，一边的限流不拖累另一边。 + +export class ApiError extends Error { + constructor( + readonly status: number, + message: string, + ) { + super(message) + this.name = 'ApiError' + } +} + +export class CancelledError extends Error { + constructor() { + super('已取消') + this.name = 'CancelledError' + } +} + +export class SizeLimitError extends Error { + constructor(readonly actualBytes: number) { + super(`附件大小 ${actualBytes} 字节超出上限`) + this.name = 'SizeLimitError' + } +} + +export interface CancelToken { + cancelled: boolean +} + +export const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) +export const jitter = (base: number, spread = base): number => base + Math.random() * spread + +export function ensureAlive(cancel?: CancelToken): void { + if (cancel?.cancelled) throw new CancelledError() +} + +export interface ThrottleConfig { + /** 请求之间的起跑间距 */ + spacingBaseMs: number + /** 吃 429 后间距放大的上限 */ + spacingMaxMs: number + /** 每这么多个请求整体歇一次（贴合突发桶回填节奏）；0 表示不歇 */ + restEveryN: number + restDurationMs: number + /** 单个请求的最大重试次数 */ + maxAttempts: number +} + +/** + * 观测快照：小规模试探时，节奏是否被服务端推着变慢，全看这里。 + * 未知站点的限流画像只能实测得来，所以先让它可见，再谈调参。 + */ +export interface FetchStats { + /** 当前起跑间距（会被 429 推大） */ + spacingMs: number + requests: number + /** 命中 429 的次数 */ + hits429: number + /** 服务端明确给出 Retry-After 的次数（真全局限流信号） */ + retryAfterHits: number + /** 还需冷却多久 */ + cooldownMs: number + /** 观察到的最大 Retry-After 秒数 */ + maxRetryAfterSec: number +} + +export interface Fetcher { + request(url: string, init?: RequestInit, cancel?: CancelToken): Promise + stats(): FetchStats +} + +export function createFetcher(cfg: ThrottleConfig): Fetcher { + let spacingMs = cfg.spacingBaseMs + let requestsSinceRest = 0 + let nextSlotAt = 0 + let cooldownUntil = 0 + // 跨 URL 连续 429 计数：区分「全局限流」和「条目级 429」的关键信号 + let global429Streak = 0 + let requests = 0 + let hits429 = 0 + let retryAfterHits = 0 + let maxRetryAfterSec = 0 + + /** 429 后调用：全局节奏永久放慢（本次运行内不回落）。 */ + const slowDown = (): void => { + spacingMs = Math.min(spacingMs * 1.5, cfg.spacingMaxMs) + } + + const acquireSlot = async (cancel?: CancelToken): Promise => { + for (;;) { + ensureAlive(cancel) + const now = Date.now() + const target = Math.max(nextSlotAt, cooldownUntil) + if (now >= target) { + nextSlotAt = now + jitter(spacingMs, spacingMs * 0.4) + if (cfg.restEveryN > 0 && ++requestsSinceRest >= cfg.restEveryN) { + requestsSinceRest = 0 + cooldownUntil = Math.max(cooldownUntil, now + cfg.restDurationMs) + } + return + } + await sleep(Math.min(target - now, 500)) + } + } + + // 429/5xx 指数退避重试；页内同源 fetch 自带登录 cookie。 + // 实测教训：部分对话会**永久性 429/404**（条目级问题，同一时刻其他请求全 200）， + // 把它们当全局限流会拖停整条流水线。 + const request = async ( + url: string, + init: RequestInit = {}, + cancel?: CancelToken, + ): Promise => { + let delay = 2000 + let headerless429s = 0 + for (let attempt = 0; ; attempt++) { + await acquireSlot(cancel) + requests++ + const res = await fetch(url, { credentials: 'include', ...init }) + if (res.ok) { + global429Streak = 0 + return res + } + const retryable = res.status === 429 || res.status >= 500 + if (!retryable || attempt >= cfg.maxAttempts) { + throw new ApiError(res.status, `HTTP ${res.status}: ${url}`) + } + if (res.status === 429) { + hits429++ + global429Streak++ + slowDown() + const retryAfterSec = Number(res.headers.get('retry-after')) + const retryAfterMs = retryAfterSec * 1000 + if (retryAfterMs > 0) { + retryAfterHits++ + maxRetryAfterSec = Math.max(maxRetryAfterSec, retryAfterSec) + cooldownUntil = Math.max(cooldownUntil, Date.now() + retryAfterMs) + } else if (global429Streak >= 5) { + cooldownUntil = Math.max(cooldownUntil, Date.now() + 15_000) + } else { + headerless429s++ + if (headerless429s > 1) throw new ApiError(429, `HTTP 429（条目级，快速放弃）: ${url}`) + await sleep(jitter(delay)) + } + } else { + await sleep(jitter(delay)) + } + delay = Math.min(delay * 2, 30_000) + } + } + + return { + request, + stats: () => ({ + spacingMs: Math.round(spacingMs), + requests, + hits429, + retryAfterHits, + cooldownMs: Math.max(0, cooldownUntil - Date.now()), + maxRetryAfterSec, + }), + } +} + +/** 附件元数据里的 size 不可靠（ChatGPT library 文件报 0），上限以实际传输为准。 */ +export async function fetchBinary( + fetcher: Fetcher, + url: string, + cancel?: CancelToken, + maxBytes?: number, +): Promise<{ bytes: Uint8Array; contentType: string | null }> { + const res = await fetcher.request(url, {}, cancel) + const declared = Number(res.headers.get('content-length')) + if (maxBytes != null && declared > maxBytes) { + try { + await res.body?.cancel() + } catch { + /* 取消流失败无所谓 */ + } + throw new SizeLimitError(declared) + } + const bytes = new Uint8Array(await res.arrayBuffer()) + if (maxBytes != null && bytes.length > maxBytes) throw new SizeLimitError(bytes.length) + return { bytes, contentType: res.headers.get('content-type') } +} + +// 简易并发池：fn 抛错即整体中止（逐条的容错由调用方在 fn 里自己 catch） +export async function mapConcurrent( + items: readonly T[], + concurrency: number, + fn: (item: T, index: number) => Promise, + cancel?: CancelToken, +): Promise { + let next = 0 + let aborted: unknown = null + const n = Math.max(1, Math.min(concurrency, items.length)) + const workers = Array.from({ length: n }, async () => { + while (aborted == null && !cancel?.cancelled) { + const i = next++ + if (i >= items.length) return + try { + await fn(items[i]!, i) + } catch (e) { + aborted = e + return + } + } + }) + await Promise.all(workers) + if (aborted != null) throw aborted + ensureAlive(cancel) +} diff --git a/src/core/ir.ts b/src/core/ir.ts new file mode 100644 index 0000000..c9c084d --- /dev/null +++ b/src/core/ir.ts @@ -0,0 +1,81 @@ +// 站点无关的中间表示（IR）。 +// +// 各站点的 adapter 负责把自家的对话 JSON 解成 IR，core/render 只认 IR。 +// 这条分界线的意义：ChatGPT 与 Claude 的原始结构差异极大（一条消息一种 +// content_type vs 一条消息多个 typed block 按序交错），但**渲染出的 Markdown +// 形态是同一套**——轮次标题、折叠 callout、围栏、附件占位、frontmatter。 +// 把后者收敛到一处，两个站点才可能共用排版、共用设置、共用离线 CLI。 +// +// 设计约束：IR 里的 prose 是「已还原引用、但未做公式/标题变换」的文本。 +// 引用还原是站点特定的（ChatGPT 的私有区标记 vs Claude 的结构化数组）， +// 公式与标题变换是通用的，所以前者归 adapter，后者归 render。 + +export interface SourceLink { + title: string + url: string +} + +export interface AssetRef { + fileId: string + kind: 'image' | 'file' + name?: string + sizeBytes?: number + mime?: string + /** 站点特定的下载线索：ChatGPT 走 files 接口换签名 URL，Claude 直接给同源地址 */ + url?: string +} + +export type IRBlock = + /** 正文：走公式 + 标题变换管道。sources 在块被渲染时并入文末汇总 */ + | { kind: 'prose'; text: string; sources?: SourceLink[] } + /** 思维链：受 thoughts 开关控制，渲染为折叠 callout。关掉时其 sources 也不收集 */ + | { kind: 'thinking'; items: Array<{ summary?: string; text: string }>; sources?: SourceLink[] } + /** + * 工具痕迹：默认受 toolTraces 开关控制（gated: false 则无条件写入—— + * Canvas 重放失败的原始 JSON 兜底走这条，属于「不丢内容」而非「工具痕迹」）。 + * fenced: false 时 body 原样进 callout，不套围栏。 + */ + | { + kind: 'tool' + title: string + body: string + lang?: string + tone?: 'example' | 'note' + gated?: boolean + fenced?: boolean + } + /** 富文档终稿（ChatGPT Canvas / Claude Artifact）：展开的 callout */ + | { kind: 'document'; label: string; docType: string; content: string } + /** 单个附件占位（图片内联） */ + | { kind: 'asset'; ref: AssetRef } + /** 附件清单（每行一个，带 `- ` 前缀） */ + | { kind: 'assetList'; refs: AssetRef[] } + /** 原样输出的短说明，不走任何变换 */ + | { kind: 'note'; text: string } + /** 未识别内容的兜底：原始 JSON 塞进折叠 callout，永不静默丢内容 */ + | { kind: 'raw'; label: string; json: unknown } + +export interface IRTurn { + role: 'user' | 'assistant' + blocks: IRBlock[] +} + +export interface IRConversation { + source: 'chatgpt' | 'claude' + id: string + title: string + /** frontmatter 里的原对话地址 */ + url: string + /** ISO 字符串，取不到时空串（frontmatter 仍保留该行，与既有行为一致） */ + created: string + updated: string + model?: string + /** 中途切换过模型时的完整列表（长度 > 1 才写进 frontmatter） */ + models?: string[] + /** 额外 frontmatter 行，值须是已成形的 YAML 片段；插在 models 与 tags 之间 */ + extra?: Array<[key: string, value: string]> + tags: string[] + /** assistant 轮次的标题文字：ChatGPT / Claude */ + assistantHeading: string + turns: IRTurn[] +} diff --git a/src/core/render.ts b/src/core/render.ts new file mode 100644 index 0000000..8d39ce5 --- /dev/null +++ b/src/core/render.ts @@ -0,0 +1,215 @@ +// IR → Markdown。站点无关：这里不认识 ChatGPT 的 content_type，也不认识 +// Claude 的 content block，只认 core/ir 的 IRBlock。 +// +// 排版契约（两个站点、油猴端与离线 CLI 四条路径共用）： +// - `# User` / `# ChatGPT`|`# Claude` 作为轮次分隔，消息内标题整体降一级 +// - 折叠 callout 承载思维链、工具痕迹与未识别内容——永不静默丢内容 +// - 附件在正文里留 assetToken 占位，下载与链接改写由调用方完成 + +import { mapTextSegmentsOutsideCode } from '../convert/codeaware' +import { transformHeadings, type HeadingMode } from '../convert/headings' +import { convertMath } from '../convert/math' +import type { AssetRef, IRBlock, IRConversation, IRTurn, SourceLink } from './ir' + +export interface ConvertOptions { + /** 是否把思维链写入导出，默认不写入（打开后折叠 callout） */ + thoughts?: boolean + /** 消息内标题处理：demote 整体降一级（默认）/ strip 全部剥离为加粗行 */ + headingMode?: HeadingMode + /** 是否写入工具运行痕迹（发给工具的代码/搜索请求与运行输出），默认不写入 */ + toolTraces?: boolean +} + +export interface ConvertResult { + markdown: string + title: string + /** 正文里以 assetToken 占位，待下载后由调用方替换成真实链接 */ + assets: AssetRef[] +} + +export type LinkStyle = 'wikilink' | 'markdown' + +/** 附件链接统一出口：油猴端与离线 CLI 共用，保证两条管道产出一致。 */ +export function assetLink( + style: LinkStyle, + path: string, + opts: { label?: string; embed?: boolean } = {}, +): string { + if (style === 'markdown') { + const label = escapeLinkLabel(opts.label ?? path.split('/').pop() ?? path) + return `${opts.embed ? '!' : ''}[${label}](${encodeURI(path)})` + } + if (opts.embed) return `![[${path}]]` + // wikilink 别名里 |、] 会破坏链接语法 + const alias = opts.label?.replace(/[[\]|]/g, '-') + return `[[${path}${alias ? `|${alias}` : ''}]]` +} + +/** 附件占位符：转换层不做网络请求，下载与链接改写由调用方完成。 */ +export const assetToken = (fileId: string): string => `%%INKSTONE-ASSET-${fileId}%%` + +// 控制字符用码点构造，避免源码里出现不可见字面量 +const CONTROL_CHARS = new RegExp(`[${String.fromCharCode(0)}-${String.fromCharCode(31)}]`, 'g') + +/** `标题-短id.md`：防重名，且 id 稳定保证增量重导时覆盖同一文件。 */ +export function filenameFor(title: string, convId: string): string { + const safe = sanitizeName(title).slice(0, 80).replace(/-+$/, '') || 'Untitled' + return `${safe}-${convId.slice(0, 8)}.md` +} + +/** 文件名净化：非法字符（跨平台 + Obsidian 链接敏感）与空白统一归一为 `-`，不留空格。 */ +export function sanitizeName(name: string): string { + return name + .replace(CONTROL_CHARS, '') + .replace(/[/\\:*?"<>|#^[\]\s]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^[-.]+|-+$/g, '') +} + +/** + * 子文件夹设置净化：按 `/` 分段逐段过 sanitizeName（`.`/`..` 被清成空段丢弃，防目录逃逸）， + * 允许 `a/b` 嵌套；返回 `''` 表示不套子文件夹。 + */ +export function sanitizeSubdir(input: string): string { + return input + .split('/') + .map((seg) => sanitizeName(seg)) + .filter((seg) => seg !== '') + .join('/') +} + +export function renderConversation(conv: IRConversation, copts: ConvertOptions = {}): ConvertResult { + const thoughts = copts.thoughts === true + const toolTraces = copts.toolTraces === true + const headingMode: HeadingMode = copts.headingMode ?? 'demote' + + const sources: SourceLink[] = [] + const assets: AssetRef[] = [] + const prose = (text: string): string => transformHeadings(convertMath(text), headingMode) + + const renderBlock = (b: IRBlock): string | null => { + switch (b.kind) { + case 'prose': + // 空正文也要收 sources：原文可能只剩引用标记，还原后为空但来源真实存在 + if (b.sources) sources.push(...b.sources) + return prose(b.text) + + case 'thinking': { + if (!thoughts) return null + if (b.sources) sources.push(...b.sources) + const items = b.items + .map((t) => (t.summary?.trim() ? `**${t.summary.trim()}**\n\n` : '') + prose(t.text)) + .filter((s) => s.trim() !== '') + return items.length === 0 ? null : callout('quote', '思考过程', items.join('\n\n'), true) + } + + case 'tool': { + if (b.gated !== false && !toolTraces) return null + const body = b.fenced === false ? b.body : fence(b.body, b.lang ?? '') + return callout(b.tone ?? 'example', b.title, body, true) + } + + case 'document': { + const isCode = b.docType.startsWith('code/') + const body = isCode ? fence(b.content, b.docType.slice('code/'.length)) : prose(b.content) + return callout('abstract', b.label, body) + } + + case 'asset': + assets.push(b.ref) + return assetToken(b.ref.fileId) + + case 'assetList': + assets.push(...b.refs) + return b.refs.map((r) => `- ${assetToken(r.fileId)}`).join('\n') + + case 'note': + return b.text + + case 'raw': + return callout('warning', b.label, fence(JSON.stringify(b.json, null, 2), 'json'), true) + } + } + + const renderTurn = (turn: IRTurn): string | null => { + const rendered = turn.blocks + .map(renderBlock) + .filter((s): s is string => s != null && s.trim() !== '') + if (rendered.length === 0) return null + const heading = turn.role === 'user' ? '# User' : `# ${conv.assistantHeading}` + return [heading, ...rendered].join('\n\n') + } + + let body = conv.turns + .map(renderTurn) + .filter((s): s is string => s != null) + .join('\n\n') + + const deduped = dedupeSources(sources) + if (deduped.length > 0) { + body += `\n\n# Sources\n\n${deduped.map((s) => `- [${escapeLinkLabel(s.title)}](${s.url})`).join('\n')}` + } + + // 收尾排版：正文里 3 连以上空行压成 1 个空行（代码块内不动） + const tidied = mapTextSegmentsOutsideCode(body, (s) => s.replace(/\n{3,}/g, '\n\n')) + + const fm = [ + '---', + `title: ${yamlQuote(conv.title)}`, + `chat_id: ${conv.id}`, + `url: ${conv.url}`, + `created: ${conv.created}`, + `updated: ${conv.updated}`, + conv.model ? `model: ${conv.model}` : null, + conv.models && conv.models.length > 1 + ? `models:\n${conv.models.map((s) => ` - ${s}`).join('\n')}` + : null, + ...(conv.extra ?? []).map(([k, v]) => `${k}: ${v}`), + 'tags:', + ...conv.tags.map((t) => ` - ${t}`), + '---', + ] + .filter((l): l is string => l != null) + .join('\n') + + return { markdown: `${fm}\n\n${tidied.trim()}\n`, title: conv.title, assets } +} + +export function fence(code: string, lang = ''): string { + // 围栏比内容里最长的反引号串再长一格，避免被内容截断 + const longest = (code.match(/`+/g) ?? []).reduce((n, run) => Math.max(n, run.length), 2) + const f = '`'.repeat(Math.max(3, longest + 1)) + return `${f}${lang}\n${code.replace(/\n$/, '')}\n${f}` +} + +export function callout(type: string, title: string, body: string, folded = false): string { + const head = `> [!${type}]${folded ? '-' : ''} ${title}` + const quoted = body + .split('\n') + .map((l) => (l === '' ? '>' : `> ${l}`)) + .join('\n') + return `${head}\n${quoted}` +} + +function dedupeSources(sources: SourceLink[]): SourceLink[] { + const seen = new Map() + for (const s of sources) { + if (!seen.has(s.url)) seen.set(s.url, s) + } + return [...seen.values()] +} + +export function escapeLinkLabel(s: string): string { + return s.replace(/([[\]])/g, '\\$1') +} + +export function yamlQuote(s: string): string { + return `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"` +} + +/** epoch 秒 / ISO 字符串 → ISO 字符串；取不到时空串（frontmatter 仍保留该行）。 */ +export function toIso(t: number | string | null | undefined): string { + if (t == null || t === '') return '' + const d = typeof t === 'number' ? new Date(t * 1000) : new Date(t) + return Number.isNaN(d.getTime()) ? '' : d.toISOString() +} diff --git a/src/main.ts b/src/main.ts index ad2d683..3875e40 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,28 +1,25 @@ import { CancelledError, ensureAlive, - fetchBinary, - fetchConversation, - createConversationPager, - getAccessToken, - listAllConversations, - listProjects, - projectNameOf, - type ConversationPager, mapConcurrent, - resolveFileDownload, SizeLimitError, sleep, type CancelToken, -} from './api' +} from './core/fetcher' +import type { AssetRef } from './core/ir' import { assetLink, assetToken, - conversationToMarkdown, filenameFor, + renderConversation, sanitizeName, - type AssetRef, -} from './convert/markdown' +} from './core/render' +import { + resolveAdapter, + type SiteAdapter, + type SiteConversationItem, + type SitePager, +} from './sites' import { downloadBlob, makeZip, strToU8, type ZipEntries } from './output/zip' import { acquireVaultDir, @@ -40,44 +37,59 @@ import { type Watermark, } from './state' import { mountPanel, type ExportFormat, type ExportOptions, type PanelHandle, type PickerItem } from './ui' -import type { ConversationListItem } from './types' // 图片始终下载，上限只防异常；文件类附件的上限由面板设置（opts.maxFileMB） const MAX_IMAGE_BYTES = 30 * 1024 * 1024 +// @match 已限定域名，理论上必命中；万一命中不了就整个不挂载，页面上不留痕迹 +let site!: SiteAdapter +const detected = resolveAdapter() + let activeCancel: CancelToken | null = null // 「选择对话…」的列表缓存：懒加载逐页追加，导出所选时直接用，不重复拉列表 -let pickedList: ConversationListItem[] = [] +let pickedList: SiteConversationItem[] = [] const pickedIds = new Set() -let pager: ConversationPager | null = null +let pager: SitePager | null = null // 代际号：重新拉取后，旧分页器迟到的响应一律丢弃 let pagerGen = 0 -mountPanel({ - onExport(scope, format, ids, panel, opts) { - void dispatchExport(scope, format, ids, panel, opts) - }, - onPickList(panel, source) { - void loadPickList(panel, source) - }, - onPickMore(panel) { - void loadNextPage(panel) - }, - onCancel() { - if (activeCancel) activeCancel.cancelled = true - }, - onResetWatermark() { - clearWatermarks(['markdown', 'json']) - }, - onForgetFolder() { - void forgetVaultDir() - }, - settings: { - values: loadSettings(), - supportsFolder: supportsDirectoryPicker(), - onSettingsChange: (patch) => saveSettings(patch), - }, -}) +/** 水位线按站点分开存：两边的对话 id 空间互不相干，共用一张表会互相污染。 */ +const wmKey = (kind: string): string => `${site.id}:${kind}` + +if (detected) { + site = detected + mount() +} + +function mount(): void { + mountPanel({ + site: { id: site.id, label: site.label, supportsBatch: site.supportsBatch }, + siteUi: site.ui, + onExport(scope, format, ids, panel, opts) { + void dispatchExport(scope, format, ids, panel, opts) + }, + onPickList(panel) { + void loadPickList(panel) + }, + onPickMore(panel) { + void loadNextPage(panel) + }, + onCancel() { + if (activeCancel) activeCancel.cancelled = true + }, + onResetWatermark() { + clearWatermarks([wmKey('markdown'), wmKey('json')]) + }, + onForgetFolder() { + void forgetVaultDir() + }, + settings: { + values: loadSettings(), + supportsFolder: supportsDirectoryPicker(), + onSettingsChange: (patch) => saveSettings(patch), + }, + }) +} /** 统一入口：folder 目标先在用户手势链路里拿目录句柄，再分发到各导出流程。 */ async function dispatchExport( @@ -87,6 +99,12 @@ async function dispatchExport( panel: PanelHandle, opts: ExportOptions, ): Promise { + // 界面已按 supportsBatch 隐藏了批量入口，这里是第二道闸：能力没实测过就不放行 + if (scope !== 'current' && !site.supportsBatch) { + panel.setStatus(`${site.label} 暂时只支持导出当前对话`) + panel.finish() + return + } let sink: OutputSink | null = null if (opts.target === 'folder') { try { @@ -112,22 +130,16 @@ async function dispatchExport( * 重置分页并拉第一页。注意这里**不碰** activeCancel / panel.finish()—— * 懒加载不占用「运行中」状态，取消按钮只属于导出流程。 */ -async function loadPickList(panel: PanelHandle, source: string): Promise { +async function loadPickList(panel: PanelHandle): Promise { const gen = ++pagerGen pager = null pickedList = [] pickedIds.clear() try { panel.setStatus('获取登录态…') - const token = await getAccessToken() + const session = await site.prepare() if (gen !== pagerGen) return - pager = createConversationPager(token, undefined, source) - // 来源下拉的 project 选项后台补上，不阻塞第一页；拿不到就只留「全部/主列表」 - void listProjects(token) - .then((ps) => { - if (gen === pagerGen) panel.setPickerProjects(ps) - }) - .catch(() => {}) + pager = site.batch!.createPager(session) panel.setStatus('拉取对话列表…') await loadNextPage(panel, gen) } catch (e) { @@ -150,9 +162,8 @@ async function loadNextPage(panel: PanelHandle, gen: number = pagerGen): Promise pickedList.push(...fresh) const picked: PickerItem[] = fresh.map((i) => ({ id: i.id, - title: i.title ?? '', + title: i.title, updated: shortDate(i.update_time), - project: projectNameOf(i.gizmo_id), })) panel.appendPicker(picked, done) panel.setStatus( @@ -184,8 +195,8 @@ async function exportSelection( return } panel.setStatus('获取登录态…') - const token = await getAccessToken(cancel) - await exportItems(format, items, 0, token, cancel, panel, opts, sink) + const session = await site.prepare(cancel) + await exportItems(format, items, 0, session, cancel, panel, opts, sink) } catch (e) { panel.setStatus(e instanceof CancelledError ? '已取消' : `出错：${String(e)}`) } finally { @@ -194,6 +205,20 @@ async function exportSelection( } } +/** + * 限流观测后缀：吃到 429 时如实报出来。 + * + * 未知站点的节奏只能靠实测看清，而实测的第一手材料就是「这次跑下来被推慢了多少」。 + * 没有 429 时保持安静，不给正常导出添噪音。 + */ +function throttleNote(): string { + const s = site.throttleStats() + if (s.hits429 === 0) return '' + const parts = [`限流 ${s.hits429} 次`, `间距已放慢到 ${s.spacingMs}ms`] + if (s.maxRetryAfterSec > 0) parts.push(`服务端最长要求等待 ${s.maxRetryAfterSec}s`) + return `（${parts.join('，')}）` +} + function shortDate(t: string | number | null | undefined): string { if (t == null) return '' const d = typeof t === 'number' ? new Date(t * 1000) : new Date(t) @@ -250,7 +275,7 @@ function folderSink(dir: FileSystemDirectoryHandle): OutputSink { /** 共享处理器：全量 / 所选 / 单对话导出都用它。 */ function createProcessor( kind: ExportFormat, - token: string, + session: string, cancel: CancelToken, panel: PanelHandle, opts: ExportOptions, @@ -276,9 +301,8 @@ function createProcessor( replacement = skippedNote(a, a.sizeBytes!, cap) } else { try { - const target = await resolveFileDownload(token, a.fileId, cancel) - const { bytes, contentType } = await fetchBinary(target.url, cancel, cap) - const name = assetFileName(a, target.filename, contentType) + const { bytes, filename, contentType } = await site.fetchAsset(session, a, cancel, cap) + const name = assetFileName(a, filename, contentType) // 链接相对 .md 所在目录，落盘再套上笔记目录前缀 const linkPath = `${attachPrefix}${a.fileId.slice(-8)}-${name}` await sink.put(`${notesPrefix}${linkPath}`, bytes, { precompressed: true }) @@ -313,36 +337,17 @@ function createProcessor( return `*(附件未下载：${a.name ?? a.fileId}，${fmtSize(actual)} 超过 ${fmtSize(cap)} 上限)*` } - // 全量/所选导出走分页器，名字早就缓存好了；只有单对话导出会落到这里补拉一次 - let projectsPass: Promise | null = null - - /** 会话详情只带 gizmo_id，project 名要靠 projects 列表换（一次导出最多补拉一次）。 */ - async function projectNameFor(gizmoId: string | null | undefined): Promise { - if (!gizmoId) return undefined - const known = projectNameOf(gizmoId) - if (known) return known - projectsPass ??= listProjects(token, cancel) - try { - await projectsPass - } catch (e) { - if (e instanceof CancelledError) throw e - return undefined // 拿不到项目名不影响正文 - } - return projectNameOf(gizmoId) - } - - async function processConversation(item: ConversationListItem): Promise<{ path: string }> { - const conv = await fetchConversation(token, item.id, cancel) + async function processConversation(item: SiteConversationItem): Promise<{ path: string }> { + const raw = await site.fetchRaw(session, item.id, cancel) if (kind === 'json') { const path = `raw/${item.id}.json` - await sink.put(path, strToU8(JSON.stringify(conv, null, 2))) + await sink.put(path, strToU8(JSON.stringify(raw, null, 2))) return { path } } - const { markdown, title, assets } = conversationToMarkdown(conv, item.id, { + const { markdown, title, assets } = renderConversation(site.toIR(raw, item.id), { thoughts: opts.thoughts, toolTraces: opts.toolTraces, headingMode: opts.headingMode, - projectName: await projectNameFor(conv.gizmo_id), }) let md = markdown let assetIdx = 0 @@ -354,7 +359,7 @@ function createProcessor( } // 附件多的对话一磨几分钟，进度要有反馈，否则像卡死 if (assets.length > 3 && assetIdx % 5 === 0) { - panel.setStatus(`「${(item.title ?? title).slice(0, 14)}」附件 ${assetIdx}/${assets.length}…`) + panel.setStatus(`「${(item.title || title).slice(0, 14)}」附件 ${assetIdx}/${assets.length}…`) } md = md.split(assetToken(a.fileId)).join(await resolveAsset(a)) } @@ -376,40 +381,42 @@ async function exportSingle( const cancel: CancelToken = { cancelled: false } activeCancel = cancel try { - const m = /\/c\/([0-9a-f][0-9a-f-]{10,})/i.exec(location.pathname) - if (!m) { - panel.setStatus('请先打开要导出的对话（网址需含 /c/…）') + const convId = site.currentConversationId() + if (!convId) { + panel.setStatus('请先打开要导出的对话') return } panel.setStatus('获取登录态…') - const token = await getAccessToken(cancel) + const session = await site.prepare(cancel) panel.setStatus('抓取当前对话…') if (format === 'json' && sink == null) { // zip 目标的 json 单对话：裸 .json 下载 - const conv = await fetchConversation(token, m[1]!, cancel) - const name = filenameFor((conv.title ?? '').trim() || 'Untitled', m[1]!).replace(/\.md$/, '.json') - downloadBlob(name, strToU8(JSON.stringify(conv, null, 2)), 'application/json') + const raw = await site.fetchRaw(session, convId, cancel) + const name = filenameFor(site.toIR(raw, convId).title, convId).replace(/\.md$/, '.json') + downloadBlob(name, strToU8(JSON.stringify(raw, null, 2)), 'application/json') panel.setStatus(`完成：${name}`) return } const zs = sink == null ? zipSink() : null - const proc = createProcessor(format, token, cancel, panel, opts, zs ?? sink!) - const { path } = await proc.processConversation({ id: m[1]!, title: null }) + const proc = createProcessor(format, session, cancel, panel, opts, zs ?? sink!) + const { path } = await proc.processConversation({ id: convId, title: '', update_time: null }) const baseName = path.split('/').pop()! if (zs != null) { const hasAttachments = Object.keys(zs.entries).some((p) => p !== path) if (hasAttachments) { - panel.setStatus((await zs.close(panel, baseName.replace(/\.md$/, '.zip'))) + proc.assetSummary()) + panel.setStatus( + (await zs.close(panel, baseName.replace(/\.md$/, '.zip'))) + proc.assetSummary() + throttleNote(), + ) } else { const entry = zs.entries[path]! downloadBlob(baseName, entry instanceof Uint8Array ? entry : entry[0], 'text/markdown') - panel.setStatus(`完成：${baseName}${proc.assetSummary()}`) + panel.setStatus(`完成：${baseName}${proc.assetSummary()}${throttleNote()}`) } } else { - panel.setStatus(`完成：${await sink!.close(panel, '')}${proc.assetSummary()}`) + panel.setStatus(`完成：${await sink!.close(panel, '')}${proc.assetSummary()}${throttleNote()}`) } } catch (e) { panel.setStatus(e instanceof CancelledError ? '已取消' : `出错：${String(e)}`) @@ -429,11 +436,11 @@ async function startExport( activeCancel = cancel try { panel.setStatus('获取登录态…') - const token = await getAccessToken(cancel) + const session = await site.prepare(cancel) panel.setStatus('拉取对话列表…') - const fullList = await listAllConversations( - token, + const fullList = await site.batch!.listAll( + session, (n) => panel.setStatus(`拉取对话列表… 已 ${n} 条`), cancel, ) @@ -443,7 +450,7 @@ async function startExport( } // 增量：跳过 update_time 与上次导出一致的对话——重负载的全量抓取一辈子只需一次 - const list = opts.incremental ? selectChanged(fullList, loadWatermark(kind)) : fullList + const list = opts.incremental ? selectChanged(fullList, loadWatermark(wmKey(kind))) : fullList const skipped = fullList.length - list.length if (list.length === 0) { panel.setStatus(`没有变化：${fullList.length} 条对话都与上次导出一致`) @@ -451,7 +458,7 @@ async function startExport( } if (skipped > 0) panel.setStatus(`跳过未变化 ${skipped} 条，导出 ${list.length} 条…`) - await exportItems(kind, list, skipped, token, cancel, panel, opts, sink) + await exportItems(kind, list, skipped, session, cancel, panel, opts, sink) } catch (e) { panel.setStatus(e instanceof CancelledError ? '已取消' : `出错：${String(e)}`) } finally { @@ -463,9 +470,9 @@ async function startExport( /** 全量 / 增量 / 所选 共用的导出主体：两遍抓取 + 落地 + 水位线推进。 */ async function exportItems( kind: ExportFormat, - list: ConversationListItem[], + list: SiteConversationItem[], skipped: number, - token: string, + session: string, cancel: CancelToken, panel: PanelHandle, opts: ExportOptions, @@ -473,22 +480,22 @@ async function exportItems( ): Promise { const sink = sinkIn ?? zipSink() // 水位线合并推进：导出成功的对话记下 update_time，其余保持原状 - const wmDraft: Watermark = { ...loadWatermark(kind) } - const proc = createProcessor(kind, token, cancel, panel, opts, sink) + const wmDraft: Watermark = { ...loadWatermark(wmKey(kind)) } + const proc = createProcessor(kind, session, cancel, panel, opts, sink) // 单条失败不中断，收集后统一重试；失败过多则保护性中止（防止触发/加重账号级反滥用）， // 已抓取的内容照常落地 async function runPass( - items: readonly ConversationListItem[], + items: readonly SiteConversationItem[], concurrency: number, label: string, ): Promise<{ - failed: ConversationListItem[] - untried: ConversationListItem[] + failed: SiteConversationItem[] + untried: SiteConversationItem[] aborted: boolean }> { - const failed: ConversationListItem[] = [] - const untried: ConversationListItem[] = [] + const failed: SiteConversationItem[] = [] + const untried: SiteConversationItem[] = [] let done = 0 let aborted = false await mapConcurrent( @@ -538,12 +545,12 @@ async function exportItems( const failures: Failure[] = [ ...failedItems.map((i) => ({ id: i.id, - title: i.title ?? '', + title: i.title, error: '多次重试后仍失败（限流隔离或对话不可用）', })), ...untriedItems.map((i) => ({ id: i.id, - title: i.title ?? '', + title: i.title, error: '保护性中止，本次未尝试（下次增量导出会自动补上）', })), ] @@ -552,15 +559,16 @@ async function exportItems( } const stamp = new Date().toISOString().slice(0, 16).replace(/[T:]/g, '-') - const doneDesc = await sink.close(panel, `chatgpt-export-${kind}-${stamp}.zip`) + const doneDesc = await sink.close(panel, `${site.id}-export-${kind}-${stamp}.zip`) // 水位线只在产物真正落地后推进：取消/崩溃的运行不记，避免下次增量漏数据 - saveWatermark(kind, wmDraft) + saveWatermark(wmKey(kind), wmDraft) panel.setStatus( `${safetyAborted ? '保护性中止（失败过多，防止触发服务端限制）。' : '完成：'}` + `${list.length - failures.length} 个对话，${doneDesc}` + (skipped > 0 ? `（另跳过未变化 ${skipped} 条）` : '') + (failures.length ? `，${failures.length} 个失败（见 _failures.json）` : '') + - proc.assetSummary(), + proc.assetSummary() + + throttleNote(), ) } diff --git a/src/sites/chatgpt/convert.ts b/src/sites/chatgpt/convert.ts new file mode 100644 index 0000000..29e36be --- /dev/null +++ b/src/sites/chatgpt/convert.ts @@ -0,0 +1,256 @@ +// ChatGPT backend-api JSON → IR。 +// +// 这里是 ChatGPT 数据模型的全部知识所在：content_type 分发、recipient 语义、 +// canmore（Canvas）重放、私有区引用标记还原。core/render 对这些一无所知。 + +import { replayCanvas, type CanvasOp } from '../../convert/canvas' +import { restoreCitations, stripResidualMarkers } from '../../convert/citations' +import { groupTurns, linearize } from '../../convert/linearize' +import type { AssetRef, IRBlock, IRConversation, IRTurn, SourceLink } from '../../core/ir' +import { filenameFor, toIso, yamlQuote } from '../../core/render' +import type { + AttachmentMeta, + ContentReference, + ConversationDetail, + ImageAssetPart, + Message, + MessageContent, +} from '../../types' + +interface Ctx { + /** msgId → 重放成功的 Canvas 操作；重放失败的 canmore 消息走原始 JSON 兜底 */ + canvas: Map +} + +export function conversationToIR(conv: ConversationDetail, fallbackId = ''): IRConversation { + const convId = String(conv.conversation_id ?? conv.id ?? fallbackId) + const title = (conv.title ?? '').trim() || 'Untitled' + const messages = linearize(conv) + + // 消息级 model_slug 才是实际生成回复的模型（default_model_slug 只是对话的默认档位，仅作回退）； + // 中途切换过模型时以最后一条为准——须在去重前取（Set 保留首现顺序，A→B→A 会错取 B）， + // 去重序列只用于 models 列表 + const rawSlugs = messages + .filter((m) => m.author.role === 'assistant' && m.metadata?.model_slug) + .map((m) => m.metadata!.model_slug!) + const modelSlugs = [...new Set(rawSlugs)] + const model = rawSlugs[rawSlugs.length - 1] ?? conv.default_model_slug + + const ctx: Ctx = { canvas: replayCanvas(messages) } + const turns: IRTurn[] = groupTurns(messages).map((t) => ({ + role: t.role, + blocks: t.messages.flatMap((m) => messageBlocks(m, ctx)), + })) + + // Branch · 对话：链接回父对话的导出文件（文件名规则可预测），Obsidian 图谱直接连起来 + const branchMeta = messages.map((m) => m.metadata).find((md) => md?.branching_from_conversation_id) + const extra: Array<[string, string]> = [] + if (branchMeta) { + const branchedFrom = filenameFor( + branchMeta.branching_from_conversation_title ?? '', + branchMeta.branching_from_conversation_id!, + ).replace(/\.md$/, '') + extra.push(['branched_from', yamlQuote(`[[${branchedFrom}]]`)]) + extra.push([ + 'branched_from_url', + `https://chatgpt.com/c/${branchMeta.branching_from_conversation_id}`, + ]) + } + + return { + source: 'chatgpt', + id: convId, + title, + url: `https://chatgpt.com/c/${convId}`, + created: toIso(conv.create_time), + updated: toIso(conv.update_time), + model: model || undefined, + models: modelSlugs, + extra, + tags: ['chatgpt'], + assistantHeading: 'ChatGPT', + turns, + } +} + +function messageBlocks(msg: Message, ctx: Ctx): IRBlock[] { + const c = msg.content + const recipient = msg.recipient ?? 'all' + const refs = msg.metadata?.content_references + const blocks: IRBlock[] = [] + const inlineImageIds = new Set() + + // canmore 工具的确认回执（role=tool）：内容已由重放侧呈现，不重复 + if (msg.author.role === 'tool' && (msg.author.name ?? '').startsWith('canmore.')) return [] + + switch (c.content_type) { + case 'text': { + const raw = joinTextParts(c) + if (msg.author.role === 'assistant' && recipient.startsWith('canmore.')) { + // Canvas：patch 重放还原终稿；重放失败回退原始 JSON 折叠嵌入 + const op = ctx.canvas.get(msg.id) + if (op) { + blocks.push(canvasBlock(op)) + } else { + blocks.push({ + kind: 'tool', + title: `工具调用 → \`${recipient}\``, + body: stripResidualMarkers(raw), + gated: false, // 兜底不丢内容，不随 toolTraces 开关 + }) + } + } else if (msg.author.role === 'assistant' && recipient !== 'all') { + // 联网等其他工具调用载荷（多为 JSON）：默认不写入，toolTraces 打开时整块折叠嵌入 + blocks.push({ + kind: 'tool', + title: `工具调用 → \`${recipient}\``, + body: stripResidualMarkers(raw), + }) + } else { + blocks.push(proseBlock(raw, refs)) + } + break + } + + case 'multimodal_text': + for (const p of c.parts ?? []) { + if (typeof p === 'string') { + blocks.push(proseBlock(p, refs)) + } else { + const { block, fileId } = imageAssetBlock(p) + blocks.push(block) + if (fileId) inlineImageIds.add(fileId) + } + } + break + + case 'code': + // content_type=code 都是工具调用载荷（代码解释器 python、联网检索 search_query/open/click 等）， + // 随 toolTraces 开关；折叠 callout 包裹，与其他工具痕迹一致 + blocks.push({ + kind: 'tool', + title: `工具调用 → \`${recipient}\``, + body: c.text ?? '', + lang: codeLanguage(c, recipient), + }) + break + + case 'execution_output': + blocks.push({ + kind: 'tool', + tone: 'note', + title: '运行输出', + body: stripResidualMarkers(c.text ?? ''), + }) + break + + case 'thoughts': { + const sources: SourceLink[] = [] + const items = (c.thoughts ?? []).map((t) => { + const restored = restoreCitations(t.content ?? '', refs) + sources.push(...restored.sources) + return { summary: t.summary, text: restored.text } + }) + blocks.push({ kind: 'thinking', items, sources }) + break + } + + default: + // 未知类型：原始 JSON 塞进折叠 callout，永不静默丢内容 + blocks.push({ + kind: 'raw', + label: `未识别的内容类型 \`${c.content_type}\`（原始 JSON）`, + json: c, + }) + } + + // 用户上传的附件（图片已在正文里内联的不重复列出） + const attachments = (msg.metadata?.attachments ?? []).filter( + (a) => a?.id && !inlineImageIds.has(a.id), + ) + if (attachments.length > 0) { + blocks.push({ kind: 'assetList', refs: attachments.map(fileAssetRef) }) + } + + return blocks +} + +function proseBlock(raw: string, refs: ContentReference[] | undefined): IRBlock { + const { text, sources } = restoreCitations(raw, refs) + return { kind: 'prose', text, sources } +} + +/** Canvas 操作的呈现：终稿整块嵌入，中间版本一行说明。 */ +function canvasBlock(op: CanvasOp): IRBlock { + if (op.kind === 'comment') { + return { + kind: 'tool', + title: `Canvas 批注${op.docName ? ` · ${op.docName}` : ''}`, + body: (op.comments ?? []).map((c) => `- ${c.comment}`).join('\n'), + gated: false, + fenced: false, + } + } + if (op.finalContent != null) { + return { + kind: 'document', + label: `Canvas · ${op.docName}`, + docType: op.docType, + content: op.finalContent, + } + } + return { + kind: 'note', + text: + op.kind === 'create' + ? `*(Canvas 创建「${op.docName}」，终稿见后)*` + : `*(Canvas 更新「${op.docName}」，终稿见后)*`, + } +} + +function imageAssetBlock(p: ImageAssetPart): { block: IRBlock; fileId: string | null } { + const pointer = typeof p.asset_pointer === 'string' ? p.asset_pointer : '' + const fileId = pointer.split('//')[1] ?? '' + if (!fileId) { + // 没有可下载指针的多模态 part（音频等）：塞原始 JSON，不丢内容 + return { + block: { + kind: 'raw', + label: `未识别的多模态 part \`${p.content_type}\`（原始 JSON）`, + json: p, + }, + fileId: null, + } + } + return { + block: { + kind: 'asset', + ref: { + fileId, + kind: 'image', + sizeBytes: typeof p.size_bytes === 'number' ? p.size_bytes : undefined, + }, + }, + fileId, + } +} + +function fileAssetRef(a: AttachmentMeta): AssetRef { + return { + fileId: a.id, + kind: 'file', + name: a.name ?? undefined, + sizeBytes: typeof a.size === 'number' ? a.size : undefined, + mime: a.mime_type ?? undefined, + } +} + +function joinTextParts(c: MessageContent): string { + return (c.parts ?? []).filter((p): p is string => typeof p === 'string').join('\n') +} + +function codeLanguage(c: MessageContent, recipient: string): string { + const lang = (c.language ?? '').trim() + if (lang && lang !== 'unknown') return lang + return recipient === 'python' ? 'python' : '' +} diff --git a/src/sites/chatgpt/index.ts b/src/sites/chatgpt/index.ts new file mode 100644 index 0000000..8e785fc --- /dev/null +++ b/src/sites/chatgpt/index.ts @@ -0,0 +1,98 @@ +import { + createConversationPager, + fetchBinary, + fetchConversation, + getAccessToken, + listAllConversations, + resolveFileDownload, + throttleStats, +} from '../../api' +import type { AssetRef } from '../../core/ir' +import type { CancelToken } from '../../core/fetcher' +import type { ConversationDetail, ConversationListItem } from '../../types' +import type { AssetPayload, SiteAdapter, SiteConversationItem } from '../types' +import { conversationToIR } from './convert' + +const toItem = (i: ConversationListItem): SiteConversationItem => ({ + id: i.id, + title: i.title ?? '', + update_time: i.update_time ?? null, +}) + +export const chatgptAdapter: SiteAdapter = { + id: 'chatgpt', + label: 'ChatGPT', + supportsBatch: true, + + matches: () => /(^|\.)chatgpt\.com$|(^|\.)chat\.openai\.com$/.test(location.hostname), + + currentConversationId() { + const m = /\/c\/([0-9a-f][0-9a-f-]{10,})/i.exec(location.pathname) + return m ? m[1]! : null + }, + + prepare: (cancel) => getAccessToken(cancel), + + fetchRaw: (session, id, cancel) => fetchConversation(session, id, cancel), + + toIR: (raw, fallbackId) => conversationToIR(raw as ConversationDetail, fallbackId), + + async fetchAsset( + session: string, + ref: AssetRef, + cancel?: CancelToken, + maxBytes?: number, + ): Promise { + // ChatGPT 的附件要先用 file id 换一个签名下载地址（fn 参数带原始文件名） + const target = await resolveFileDownload(session, ref.fileId, cancel) + const { bytes, contentType } = await fetchBinary(target.url, cancel, maxBytes) + return { bytes, filename: target.filename, contentType } + }, + + throttleStats, + + batch: { + async listAll(session, onProgress, cancel) { + return (await listAllConversations(session, onProgress, cancel)).map(toItem) + }, + createPager(session, cancel) { + const pager = createConversationPager(session, cancel) + return { + async next() { + const { items, done } = await pager.next() + return { items: items.map(toItem), done } + }, + } + }, + }, + + ui: { + headerAnchor: () => + (document.querySelector('[data-testid="share-chat-button"]') ?? + document.querySelector('#conversation-header-actions')) as HTMLElement | null, + + composerAnchor: () => + (document.querySelector('#prompt-textarea')?.closest('form') ?? + document.querySelector('form[data-type="unified-composer"]')) as HTMLElement | null, + + isDark: () => document.documentElement.classList.contains('dark'), + + themeAttributes: ['class', 'data-chat-theme'], + + // ChatGPT 的 accent 方案（2026-07 实测）：html[data-chat-theme="purple"] + + // 每主题一族变量 --{theme}-theme-submit-btn-bg/-text 与 --{theme}-theme-entity-accent。 + // 直接读当前主题的发送键配色作主色；变量消失（改版）时返回 null，交给通用兜底。 + accent(parse) { + const rootEl = document.documentElement + const cs = getComputedStyle(rootEl) + const theme = rootEl.getAttribute('data-chat-theme') || 'default' + const bg = parse(cs.getPropertyValue(`--${theme}-theme-submit-btn-bg`)) + if (!bg) return null + return { + bg, + fg: parse(cs.getPropertyValue(`--${theme}-theme-submit-btn-text`)), + ring: parse(cs.getPropertyValue(`--${theme}-theme-entity-accent`)), + } + }, + }, +} diff --git a/src/sites/claude/api.ts b/src/sites/claude/api.ts new file mode 100644 index 0000000..e0c60e8 --- /dev/null +++ b/src/sites/claude/api.ts @@ -0,0 +1,189 @@ +// claude.ai 内部 API 客户端。节奏控制在 core/fetcher，这里只负责端点与字段。 +// +// ⚠️ 限流画像未知。ChatGPT 侧的参数是 344 + 432 对话实测调出来的，Claude 侧 +// 一条实测数据都没有，现有的开源 claude.ai 导出器也没有一个实现了退避 +// （最激进的是 3 并发 + 固定 200ms 间隔，且不看 429）。所以这里的起步值刻意比 +// ChatGPT 侧慢一倍，且每 40 个请求就歇一次：宁可慢，不可触发账号级限制。 +// +// 调参依据只能来自实测——fetcher.stats() 会记下间距被推大的过程、429 次数与 +// 服务端给出的最大 Retry-After，面板把它显示出来。等有了真实数据再谈放宽。 + +import { + ApiError, + createFetcher, + ensureAlive, + fetchBinary as fetchBinaryWith, + sleep, + type CancelToken, + type Fetcher, + type ThrottleConfig, +} from '../../core/fetcher' +import type { ClaudeConversation, ClaudeConversationListItem, ClaudeOrganization } from './types' + +export const CLAUDE_THROTTLE: ThrottleConfig = { + spacingBaseMs: 1500, + spacingMaxMs: 8000, + restEveryN: 40, + restDurationMs: 30_000, + maxAttempts: 6, +} + +const fetcher: Fetcher = createFetcher(CLAUDE_THROTTLE) + +/** 当前节奏快照（面板用来显示限流观测）。 */ +export const throttleStats = (): ReturnType => fetcher.stats() + +const api = (path: string): string => `${location.origin}${path}` + +/** + * 拿组织 id。优先问接口——cookie 里的 lastActiveOrg 在多组织账号下会随最近活跃 + * 组织变化，而接口给的是真实归属；接口不可用时才回退 cookie。 + */ +export async function resolveOrgId(cancel?: CancelToken): Promise { + try { + const res = await fetcher.request(api('/api/organizations'), {}, cancel) + const data: unknown = await res.json() + const list = Array.isArray(data) + ? (data as ClaudeOrganization[]) + : ((data as { organizations?: ClaudeOrganization[] })?.organizations ?? []) + const uuid = list.find((o) => typeof o?.uuid === 'string')?.uuid + if (uuid) return uuid + } catch { + /* 落到 cookie 兜底 */ + } + const fromCookie = /(?:^|;\s*)lastActiveOrg=([^;]+)/.exec(document.cookie)?.[1] + if (fromCookie) return decodeURIComponent(fromCookie) + throw new Error('拿不到组织 id：请确认已登录 claude.ai 后重试') +} + +/** 从地址栏取当前对话 id；不在对话页时返回 null。 */ +export function currentConversationId(): string | null { + const m = /\/chat\/([0-9a-f-]{20,})/i.exec(location.pathname) + return m ? m[1]! : null +} + +export async function fetchConversation( + orgId: string, + id: string, + cancel?: CancelToken, +): Promise { + // tree=True 拿完整消息树（分支靠 parent 链自己走），render_all_tools 保证 + // artifact / 文件 / widget 的 tool_use 载荷不被服务端裁掉 + const url = api( + `/api/organizations/${orgId}/chat_conversations/${id}` + + `?tree=True&rendering_mode=messages&render_all_tools=true`, + ) + const res = await fetcher.request(url, { headers: { Accept: 'application/json' } }, cancel) + return (await res.json()) as ClaudeConversation +} + +// ——— 以下是批量导出的地基，当前版本的 UI 不暴露 ——— +// 单对话导出只需要上面两个端点。列表接口先按分页写好（形状与 ChatGPT 侧一致， +// 便于后续复用同一套编排），但在限流画像实测清楚之前不接进界面。 + +export async function listConversationsPage( + orgId: string, + offset: number, + limit: number, + cancel?: CancelToken, +): Promise { + const url = api(`/api/organizations/${orgId}/chat_conversations?limit=${limit}&offset=${offset}`) + const res = await fetcher.request(url, { headers: { Accept: 'application/json' } }, cancel) + const data: unknown = await res.json() + // [待测] 分页参数是否被服务端认。若不认，这里会一次性拿回全部——调用方靠 + // 「返回数 < limit」判断到底会误判，所以终止条件同样只认空页。 + return Array.isArray(data) ? (data as ClaudeConversationListItem[]) : [] +} + +export interface ConversationPager { + next(): Promise<{ items: ClaudeConversationListItem[]; done: boolean }> +} + +export type FetchPage = ( + orgId: string, + offset: number, + limit: number, + cancel?: CancelToken, +) => Promise + +export interface PagerOptions { + /** 翻页请求的实现；默认打真实接口，测试可注入假页 */ + fetchPage?: FetchPage + /** 空页重试的等待基数（第 n 次等 n 倍）；设 0 即不等待 */ + emptyRetryBaseMs?: number +} + +export function createConversationPager( + orgId: string, + cancel?: CancelToken, + opts: PagerOptions = {}, +): ConversationPager { + const fetchPage = opts.fetchPage ?? listConversationsPage + const emptyRetryBaseMs = opts.emptyRetryBaseMs ?? 4000 + let offset = 0 + let limit = 50 + let emptyRetries = 0 + let done = false + const seen = new Set() + + return { + async next() { + if (done) return { items: [], done: true } + for (;;) { + ensureAlive(cancel) + let items: ClaudeConversationListItem[] + try { + items = await fetchPage(orgId, offset, limit, cancel) + } catch (e) { + if (e instanceof ApiError && e.status >= 400 && e.status < 500 && e.status !== 429 && limit > 20) { + limit = 20 + continue + } + throw e + } + + // 真空页：可能只是服务端瞬时抖动（ChatGPT 侧实测过列表索引会短暂降级）， + // 隔几秒重试确认，连续空 3 次才认到底 + if (items.length === 0) { + if (offset === 0 || emptyRetries >= 2) { + done = true + return { items: [], done: true } + } + emptyRetries++ + if (emptyRetryBaseMs > 0) await sleep(emptyRetryBaseMs * emptyRetries) + continue + } + + // 有返回、但全是见过的：要么服务端忽略了分页参数每次给同一批，要么已到底。 + // 两种都不该重试——再问一次只会拿到同样的东西。 + const fresh = items.filter((i) => typeof i?.uuid === 'string' && !seen.has(i.uuid)) + if (fresh.length === 0) { + done = true + return { items: [], done: true } + } + for (const i of fresh) seen.add(i.uuid) + + emptyRetries = 0 + offset += items.length + return { items: fresh, done: false } + } + }, + } +} + +/** Claude 的附件地址是同源相对路径，登录态直接可取，不需要先换签名 URL。 */ +export function fetchBinary( + url: string, + cancel?: CancelToken, + maxBytes?: number, +): Promise<{ bytes: Uint8Array; contentType: string | null }> { + return fetchBinaryWith(fetcher, absolute(url), cancel, maxBytes) +} + +function absolute(url: string): string { + try { + return new URL(url, location.origin).href + } catch { + return url + } +} diff --git a/src/sites/claude/artifacts.ts b/src/sites/claude/artifacts.ts new file mode 100644 index 0000000..2229369 --- /dev/null +++ b/src/sites/claude/artifacts.ts @@ -0,0 +1,131 @@ +// Artifact 折叠：把 create / update / rewrite 序列还原成终稿。 +// +// 与 ChatGPT 的 Canvas 重放同构，但简单得多——Canvas 的 update 是 Python 风格 +// 正则 patch（语义与 JS 存疑，得逐条判断能不能翻译），Artifact 的 update 是 +// 字面量 old_str → new_str，可靠得多。失真控制沿用 canvas.ts 的规矩： +// 任何一步匹配不上就放弃该文档的后续重放，由调用方回退原始 JSON 折叠嵌入。 + +import type { ClaudeContentBlock, ClaudeMessage } from './types' + +export type ArtifactOpKind = 'create' | 'update' | 'rewrite' + +export interface ArtifactOp { + kind: ArtifactOpKind + title: string + /** claude 的 artifact type，如 application/vnd.ant.code、text/markdown */ + mime: string + language?: string + /** 仅当本条是该 artifact 最后一次成功的内容变更：重放出的终稿 */ + finalContent?: string +} + +interface DocState { + title: string + mime: string + language?: string + content: string + /** 某次 update 匹配失败后不再信任后续状态，终稿停在最后一次成功处 */ + broken: boolean + lastGoodKey: string +} + +/** 块定位键：一条消息可能含多个 tool_use 块，光靠 msg.uuid 不够。 */ +export const blockKey = (msgUuid: string, blockIndex: number): string => `${msgUuid}#${blockIndex}` + +/** + * 沿主线重放全部 artifacts 工具调用。 + * 返回 blockKey → ArtifactOp；不在 map 里的 artifacts 块表示重放失败，调用方走原始 JSON 兜底。 + */ +export function replayArtifacts(messages: readonly ClaudeMessage[]): Map { + const ops = new Map() + const docs = new Map() + + for (const msg of messages) { + const blocks = msg.content ?? [] + for (let i = 0; i < blocks.length; i++) { + const block = blocks[i]! + if (block.type !== 'tool_use' || block.name !== 'artifacts') continue + const input = block.input ?? {} + const id = str(input['id']) || '__artifact__' + const key = blockKey(msg.uuid, i) + const command = str(input['command']) + + let doc = docs.get(id) + if (!doc) { + doc = { title: '', mime: '', content: '', broken: false, lastGoodKey: key } + docs.set(id, doc) + } + // 元数据挂在 create 块上，后续 update 不重复携带 + if (str(input['title'])) doc.title = str(input['title']) + if (str(input['type'])) doc.mime = str(input['type']) + if (str(input['language'])) doc.language = str(input['language']) + + if (command === 'update') { + if (doc.broken) continue + const oldStr = input['old_str'] + const newStr = input['new_str'] + if (typeof oldStr !== 'string' || typeof newStr !== 'string' || !doc.content.includes(oldStr)) { + doc.broken = true + continue + } + // 必须传函数替换器：字符串形式会把 new_str 里的 $&、$` 、$$ 当成替换模式， + // 静默损坏内容——这些序列在真实的 JS / shell / CSS 代码里很常见。 + doc.content = doc.content.replace(oldStr, () => newStr) + doc.lastGoodKey = key + ops.set(key, { kind: 'update', title: doc.title, mime: doc.mime, language: doc.language }) + } else if (command === 'create' || command === 'rewrite') { + const content = input['content'] + if (typeof content !== 'string') { + doc.broken = true + continue + } + doc.content = content + doc.broken = false // 全量写入重新奠定基线，此前的失配不再影响后续 + doc.lastGoodKey = key + ops.set(key, { + kind: command, + title: doc.title, + mime: doc.mime, + language: doc.language, + }) + } + // 未知 command：不入 ops，调用方走原始 JSON 兜底 + } + } + + // 每个 artifact 最后一次成功的内容变更处嵌入终稿 + for (const doc of docs.values()) { + if (doc.content === '') continue + const op = ops.get(doc.lastGoodKey) + if (op) op.finalContent = doc.content + } + return ops +} + +/** + * artifact 的 mime + language → IR 的 docType。 + * `code/*` 会被渲染成围栏代码块，其余走正文排版管道（标题降级、公式转换）。 + */ +export function artifactDocType(mime: string, language?: string): string { + switch (mime) { + case 'application/vnd.ant.react': + return 'code/jsx' + case 'text/html': + return 'code/html' + case 'image/svg+xml': + return 'code/svg' + case 'application/vnd.ant.mermaid': + return 'code/mermaid' + case 'application/vnd.ant.code': + return `code/${language ?? ''}` + case 'text/markdown': + return 'document' + default: + // 未知类型当文档处理：宁可让正文原样出现，也不套错围栏 + return 'document' + } +} + +function str(v: unknown): string { + return typeof v === 'string' ? v : '' +} diff --git a/src/sites/claude/convert.ts b/src/sites/claude/convert.ts new file mode 100644 index 0000000..8465157 --- /dev/null +++ b/src/sites/claude/convert.ts @@ -0,0 +1,355 @@ +// claude.ai 内部 API JSON → IR。 +// +// 与 ChatGPT adapter 的结构性差异：那边一条消息只有一种 content_type，这边一条 +// 消息是多个 typed block 按序交错（text / thinking / tool_use / tool_result）， +// 所以分发发生在块级而不是消息级。产出的 IR 形态两边完全一致。 + +import type { AssetRef, IRBlock, IRConversation, IRTurn, SourceLink } from '../../core/ir' +import { toIso, yamlQuote } from '../../core/render' +import { artifactDocType, blockKey, replayArtifacts, type ArtifactOp } from './artifacts' +import type { ClaudeContentBlock, ClaudeConversation, ClaudeMessage } from './types' + +interface Ctx { + /** blockKey → 重放成功的 artifact 操作；不在表里的走原始 JSON 兜底 */ + artifacts: Map +} + +export function conversationToIR(conv: ClaudeConversation, fallbackId = ''): IRConversation { + const convId = String(conv.uuid ?? fallbackId) + const title = (conv.name ?? '').trim() || 'Untitled' + const messages = linearize(conv) + const ctx: Ctx = { artifacts: replayArtifacts(messages) } + + const turns: IRTurn[] = groupTurns(messages).map((t) => ({ + role: t.role, + blocks: t.messages.flatMap((m) => messageBlocks(m, ctx)), + })) + + // Projects：ChatGPT 侧没有的层级，写进 frontmatter 供 Dataview 分组 + const extra: Array<[string, string]> = [] + const projectName = (conv.project?.name ?? '').trim() + if (projectName) extra.push(['project', yamlQuote(projectName)]) + + return { + source: 'claude', + id: convId, + title, + url: `https://claude.ai/chat/${convId}`, + created: toIso(conv.created_at), + updated: toIso(conv.updated_at), + model: conv.model || undefined, + extra, + tags: ['claude'], + assistantHeading: 'Claude', + turns, + } +} + +/** + * 沿 current_leaf_message_uuid 的 parent 链回溯，得到网页上实际可见的主线 + * （重新生成的旧分支不含在内）。leaf 缺失时退回 index 排序——分支对话下可能不准， + * 但总好过丢消息。 + */ +export function linearize(conv: ClaudeConversation): ClaudeMessage[] { + const all = conv.chat_messages ?? [] + if (all.length === 0) return [] + + const byUuid = new Map(all.map((m) => [m.uuid, m])) + const leaf = conv.current_leaf_message_uuid + let chain: ClaudeMessage[] | null = null + + if (leaf && byUuid.has(leaf)) { + const path: ClaudeMessage[] = [] + const seen = new Set() + let cur: ClaudeMessage | undefined = byUuid.get(leaf) + while (cur && !seen.has(cur.uuid)) { + seen.add(cur.uuid) + path.push(cur) + cur = cur.parent_message_uuid ? byUuid.get(cur.parent_message_uuid) : undefined + } + if (path.length > 0) chain = path.reverse() + } + + if (!chain) chain = [...all].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)) + // 只按 sender 过滤：接口也会返回 UI 从不显示的消息，但内容层面的取舍留给块级， + // 空块由 render 统一丢弃——不在这里提前判断「有没有内容」，免得误杀 + return chain.filter((m) => m.sender === 'human' || m.sender === 'assistant') +} + +interface Turn { + role: 'user' | 'assistant' + messages: ClaudeMessage[] +} + +/** 相邻同侧消息合并成轮次。 */ +export function groupTurns(messages: readonly ClaudeMessage[]): Turn[] { + const turns: Turn[] = [] + for (const msg of messages) { + const role: Turn['role'] = msg.sender === 'human' ? 'user' : 'assistant' + const last = turns[turns.length - 1] + if (last && last.role === role) last.messages.push(msg) + else turns.push({ role, messages: [msg] }) + } + return turns +} + +function messageBlocks(msg: ClaudeMessage, ctx: Ctx): IRBlock[] { + const blocks: IRBlock[] = [] + const content = msg.content ?? [] + + for (let i = 0; i < content.length; i++) { + const b = content[i]! + switch (b.type) { + case 'text': + blocks.push(proseBlock(b)) + break + case 'thinking': + blocks.push(thinkingBlock(b)) + break + case 'tool_use': + blocks.push(...toolUseBlocks(b, msg.uuid, i, ctx)) + break + case 'tool_result': + blocks.push(toolResultBlock(b)) + break + default: + // 未知类型：原始 JSON 塞进折叠 callout，永不静默丢内容 + blocks.push({ + kind: 'raw', + label: `未识别的内容块 \`${b.type}\`（原始 JSON）`, + json: b, + }) + } + } + + blocks.push(...attachmentBlocks(msg)) + + // 未完成的回复如实标注，但不替读者下判断：user_canceled 是正常结束（用户自己停的）， + // truncated 的触发条件未经证实，只陈述服务端给了这个标记 + if (msg.stop_reason === 'user_canceled') { + blocks.push({ kind: 'note', text: '*(这条回复被用户中止)*' }) + } + if (msg.truncated === true) { + blocks.push({ kind: 'note', text: '*(服务端将这条消息标记为 truncated)*' }) + } + + return blocks +} + +/** + * 正文 + 引用。 + * + * 首版只把引用汇总进文末 Sources，正文一字不动：Claude 通常自己就在正文里写了 + * Markdown 链接，行内再插一遍会重复；而 citations 的字符级锚定字段尚未实测， + * 猜着插入的风险高于收益。等实测确认锚定形态后再做行内还原。 + */ +function proseBlock(b: ClaudeContentBlock): IRBlock { + const sources: SourceLink[] = [] + for (const c of b.citations ?? []) { + // 过期的搜索结果不写进笔记——死链比没有链接更糟 + if (c.is_expired === true) continue + const url = str(c.url) + if (!url) continue + sources.push({ title: str(c.title) || hostOf(url) || url, url }) + } + return { kind: 'prose', text: str(b.text), sources } +} + +function thinkingBlock(b: ClaudeContentBlock): IRBlock { + const text = str(b.thinking) || str(b.text) + const summary = + (b.summaries ?? []) + .map((s) => str(s?.summary)) + .filter((s) => s !== '') + .join(' · ') || undefined + return { kind: 'thinking', items: [{ summary, text }] } +} + +function toolUseBlocks( + b: ClaudeContentBlock, + msgUuid: string, + index: number, + ctx: Ctx, +): IRBlock[] { + const name = str(b.name) + const input = b.input ?? {} + + if (name === 'artifacts') { + const op = ctx.artifacts.get(blockKey(msgUuid, index)) + if (!op) { + // 重放失败：原始 JSON 兜底，不随 toolTraces 开关——这是「不丢内容」而非工具痕迹 + return [ + { + kind: 'tool', + title: '工具调用 → `artifacts`（重放失败，原始 JSON）', + body: JSON.stringify(b, null, 2), + lang: 'json', + gated: false, + }, + ] + } + if (op.finalContent != null) { + return [ + { + kind: 'document', + label: `Artifact · ${op.title || 'untitled'}`, + docType: artifactDocType(op.mime, op.language), + content: op.finalContent, + }, + ] + } + return [ + { + kind: 'note', + text: + op.kind === 'create' + ? `*(Artifact 创建「${op.title || 'untitled'}」，终稿见后)*` + : `*(Artifact 更新「${op.title || 'untitled'}」，终稿见后)*`, + }, + ] + } + + if (name === 'create_file' && typeof input['file_text'] === 'string') { + const path = str(input['path']) || 'file' + return [ + { + kind: 'document', + label: `文件 · ${path}`, + docType: `code/${langOfPath(path)}`, + content: input['file_text'], + }, + ] + } + + if (name === 'visualize:show_widget' && typeof input['widget_code'] === 'string') { + return [ + { + kind: 'document', + label: `Widget · ${str(input['title']) || 'untitled'}`, + docType: 'code/jsx', + content: input['widget_code'], + }, + ] + } + + // 其余工具（web 搜索、bash、文件读写…）：工具痕迹，随 toolTraces 开关 + return [ + { + kind: 'tool', + title: `工具调用 → \`${name || 'unknown'}\``, + body: JSON.stringify(input, null, 2), + lang: 'json', + }, + ] +} + +function toolResultBlock(b: ClaudeContentBlock): IRBlock { + const isText = typeof b.content === 'string' + return { + kind: 'tool', + tone: 'note', + title: b.is_error === true ? '工具返回（错误）' : '工具返回', + body: isText ? (b.content as string) : JSON.stringify(b.content ?? null, null, 2), + lang: isText ? '' : 'json', + } +} + +/** + * 附件两处来源，按各自实际提供的东西处理： + * files[] —— 上传的原件。图片有 preview_url（内联嵌入），文档有 + * document_asset.url（列为链接），blob 类没有可用地址（留说明） + * attachments[] —— 文本抽取件（.md/.docx/…）。没有地址，但正文就在 + * extracted_content 里，整块嵌进展开的 callout —— 附件自身的 + * 标题因此不会进文档大纲，导出的笔记对全文检索是自包含的 + */ +function attachmentBlocks(msg: ClaudeMessage): IRBlock[] { + const out: IRBlock[] = [] + const fileRefs: AssetRef[] = [] + + for (const f of msg.files ?? []) { + const name = str(f.file_name) || 'file' + const size = typeof f.size_bytes === 'number' ? f.size_bytes : undefined + const kind = str(f.file_kind) + + if (kind === 'image') { + const url = str(f.preview_url) || str(f.preview_asset?.url) + if (url) { + out.push({ kind: 'asset', ref: { fileId: assetId(f, url), kind: 'image', name, url, sizeBytes: size } }) + continue + } + } else if (kind === 'document') { + const url = str(f.document_asset?.url) + if (url) { + fileRefs.push({ fileId: assetId(f, url), kind: 'file', name, url, sizeBytes: size }) + continue + } + } + // blob（音频等）与拿不到地址的：留名字，不假装能下载 + out.push({ kind: 'note', text: `*(附件：${name}${kind ? ` · ${kind}` : ''} — 无可下载地址)*` }) + } + + if (fileRefs.length > 0) out.push({ kind: 'assetList', refs: fileRefs }) + + for (const a of msg.attachments ?? []) { + const name = str(a.file_name) || str(a.name) || 'attachment' + const content = str(a.extracted_content).trim() + if (content === '') { + out.push({ kind: 'note', text: `*(附件：${name})*` }) + continue + } + out.push({ + kind: 'document', + label: `附件 · ${name}（文本抽取）`, + docType: 'document', + content, + }) + } + + return out +} + +/** 附件在正文里的占位键：优先用服务端 uuid，缺失时用地址兜底（同一附件只下载一次）。 */ +function assetId(f: { file_uuid?: string | null; uuid?: string | null }, url: string): string { + return str(f.file_uuid) || str(f.uuid) || url +} + +const EXT_LANG: Record = { + py: 'python', + js: 'javascript', + jsx: 'jsx', + ts: 'typescript', + tsx: 'tsx', + md: 'markdown', + html: 'html', + css: 'css', + json: 'json', + sh: 'bash', + yml: 'yaml', + yaml: 'yaml', + sql: 'sql', + java: 'java', + rb: 'ruby', + go: 'go', + rs: 'rust', + c: 'c', + cpp: 'cpp', + txt: '', +} + +function langOfPath(path: string): string { + const file = path.split('/').pop() ?? path + const ext = file.includes('.') ? file.split('.').pop()!.toLowerCase() : '' + return EXT_LANG[ext] ?? '' +} + +function hostOf(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, '') + } catch { + return '' + } +} + +function str(v: unknown): string { + return typeof v === 'string' ? v : '' +} diff --git a/src/sites/claude/index.ts b/src/sites/claude/index.ts new file mode 100644 index 0000000..05202ad --- /dev/null +++ b/src/sites/claude/index.ts @@ -0,0 +1,83 @@ +import type { AssetRef } from '../../core/ir' +import type { CancelToken } from '../../core/fetcher' +import type { AssetPayload, Rgb, SiteAdapter } from '../types' +import { + currentConversationId, + fetchBinary, + fetchConversation, + resolveOrgId, + throttleStats, +} from './api' +import { conversationToIR } from './convert' +import type { ClaudeConversation } from './types' + +export const claudeAdapter: SiteAdapter = { + id: 'claude', + label: 'Claude', + // 首版只做「导出当前对话」。批量的地基（分页器、水位线、并发池）都在， + // 但在 Claude 的限流画像实测清楚之前不开——见 sites/claude/api.ts 的说明。 + supportsBatch: false, + + matches: () => /(^|\.)claude\.ai$/.test(location.hostname), + + currentConversationId, + + prepare: (cancel) => resolveOrgId(cancel), + + fetchRaw: (session, id, cancel) => fetchConversation(session, id, cancel), + + toIR: (raw, fallbackId) => conversationToIR(raw as ClaudeConversation, fallbackId), + + async fetchAsset( + _session: string, + ref: AssetRef, + cancel?: CancelToken, + maxBytes?: number, + ): Promise { + // Claude 的附件地址就在消息里，同源、登录态直接可取，不需要先换签名 URL + if (!ref.url) throw new Error(`附件 ${ref.name ?? ref.fileId} 没有可下载地址`) + const { bytes, contentType } = await fetchBinary(ref.url, cancel, maxBytes) + return { bytes, filename: null, contentType } + }, + + throttleStats, + + ui: { + // [待测] 以下选择器需要在真实页面上确认。找不到锚点时 FAB 不显示（既有防御）， + // 所以候选写宽是安全的：宁可多试几个，也不要挂在会被本地化的 aria-label 文案上。 + headerAnchor: () => + (document.querySelector('[data-testid="share-button"]') ?? + document.querySelector('[data-testid="chat-menu-trigger"]') ?? + document.querySelector('header button[aria-haspopup="menu"]')) as HTMLElement | null, + + composerAnchor: () => + (document.querySelector('fieldset div[contenteditable="true"]')?.closest('fieldset') ?? + document.querySelector('div[contenteditable="true"][role="textbox"]')?.closest('fieldset') ?? + document.querySelector('div.ProseMirror[contenteditable="true"]')?.parentElement ?? + null) as HTMLElement | null, + + // 明暗判定不赌 class 名：先看常见标记，再退回背景色亮度——任何改版都还成立 + isDark: () => { + const root = document.documentElement + if (root.classList.contains('dark')) return true + if (root.getAttribute('data-mode') === 'dark') return true + const rgb = /(\d+)[,\s]+(\d+)[,\s]+(\d+)/.exec(getComputedStyle(document.body).backgroundColor) + if (!rgb) return false + const [r, g, b] = [Number(rgb[1]), Number(rgb[2]), Number(rgb[3])] + return (0.2126 * r + 0.7152 * g + 0.0722 * b) / 255 < 0.5 + }, + + themeAttributes: ['class', 'data-mode', 'data-theme'], + + // [待测] Claude 的主色变量名未确认。先试几个常见命名，命中不了就返回 null， + // 交给界面层的通用兜底（扫描含 accent 的自定义属性，取最饱和的那个）。 + accent(parse: (raw: string) => Rgb | null) { + const cs = getComputedStyle(document.documentElement) + for (const name of ['--accent-main-000', '--accent-main-100', '--accent-brand', '--brand']) { + const bg = parse(cs.getPropertyValue(name)) + if (bg) return { bg, fg: null, ring: null } + } + return null + }, + }, +} diff --git a/src/sites/claude/types.ts b/src/sites/claude/types.ts new file mode 100644 index 0000000..ea4ef63 --- /dev/null +++ b/src/sites/claude/types.ts @@ -0,0 +1,106 @@ +// claude.ai 内部 API 是非官方接口，字段随时可能变：类型一律从宽，未知字段用索引签名兜住。 +// +// 契约来源：现有开源导出器的实现与其维护文档（2026-07 记录），尚未在本项目里 +// 端到端实测。凡是本文件里带 “[待测]” 的字段，都要在真实会话上确认后再收紧类型。 + +export interface ClaudeOrganization { + uuid: string + name?: string | null + [k: string]: unknown +} + +export interface ClaudeConversationListItem { + uuid: string + name?: string | null + created_at?: string | null + updated_at?: string | null + model?: string | null + [k: string]: unknown +} + +export interface ClaudeCitation { + url?: string | null + title?: string | null + /** 被引用的原文片段（最多约 150 字符） */ + cited_text?: string | null + metadata?: { site_domain?: string; site_name?: string; [k: string]: unknown } | null + /** 搜索结果 URL 可能已失效——过期的不写进笔记，避免制造死链 */ + is_expired?: boolean + [k: string]: unknown +} + +export interface ClaudeContentBlock { + /** text | thinking | tool_use | tool_result，以及未来可能新增的类型 */ + type: string + text?: string | null + /** thinking 块的正文 [待测：字段名可能是 thinking 或 text] */ + thinking?: string | null + /** thinking 块的分段摘要 [待测] */ + summaries?: Array<{ summary?: string | null; [k: string]: unknown }> | null + /** tool_use 的工具名：artifacts / create_file / visualize:show_widget / web_search … */ + name?: string | null + input?: Record | null + /** tool_result 的载荷，形态随工具而异 */ + content?: unknown + is_error?: boolean + citations?: ClaudeCitation[] | null + [k: string]: unknown +} + +export interface ClaudeFile { + file_kind?: string | null + file_name?: string | null + file_uuid?: string | null + uuid?: string | null + /** 图片预览地址（同源 claude.ai，登录态绑定） */ + preview_url?: string | null + preview_asset?: { url?: string | null; [k: string]: unknown } | null + document_asset?: { url?: string | null; page_count?: number | null; [k: string]: unknown } | null + size_bytes?: number | null + [k: string]: unknown +} + +/** 文本抽取型附件：没有下载地址，但正文本身就在 extracted_content 里 */ +export interface ClaudeAttachment { + file_name?: string | null + name?: string | null + file_type?: string | null + file_size?: number | null + extracted_content?: string | null + [k: string]: unknown +} + +export interface ClaudeMessage { + uuid: string + parent_message_uuid?: string | null + /** current_leaf 缺失时的兜底排序 */ + index?: number | null + sender: string + created_at?: string | null + updated_at?: string | null + /** 该 rendering mode 下顶层 text 为空，正文一律读 content 块 */ + text?: string | null + content?: ClaudeContentBlock[] | null + files?: ClaudeFile[] | null + attachments?: ClaudeAttachment[] | null + /** 服务端标记；触发条件未知，出现时在正文里如实标注，不臆断内容缺失 */ + truncated?: boolean + /** assistant 专有；'user_canceled' = 用户主动停止，属正常结束而非异常 */ + stop_reason?: string | null + [k: string]: unknown +} + +export interface ClaudeConversation { + uuid?: string + name?: string | null + model?: string | null + created_at?: string | null + updated_at?: string | null + /** 当前分支的末端，驱动主线定位 */ + current_leaf_message_uuid?: string | null + chat_messages?: ClaudeMessage[] + /** Projects 归属 [待测：字段名] */ + project_uuid?: string | null + project?: { uuid?: string | null; name?: string | null } | null + [k: string]: unknown +} diff --git a/src/sites/index.ts b/src/sites/index.ts new file mode 100644 index 0000000..27793fd --- /dev/null +++ b/src/sites/index.ts @@ -0,0 +1,23 @@ +// 站点分派：按当前域名选适配器。 + +import { chatgptAdapter } from './chatgpt' +import { claudeAdapter } from './claude' +import type { SiteAdapter } from './types' + +export type { + AssetPayload, + Rgb, + SiteAdapter, + SiteBatch, + SiteConversationItem, + SiteId, + SitePager, + SiteUi, +} from './types' + +export const adapters: readonly SiteAdapter[] = [chatgptAdapter, claudeAdapter] + +/** 当前页面对应的适配器；都不匹配时返回 null（脚本不挂载任何界面）。 */ +export function resolveAdapter(): SiteAdapter | null { + return adapters.find((a) => a.matches()) ?? null +} diff --git a/src/sites/types.ts b/src/sites/types.ts new file mode 100644 index 0000000..b0d60bf --- /dev/null +++ b/src/sites/types.ts @@ -0,0 +1,88 @@ +// 站点适配器接口：编排层（main.ts）与界面层（ui.ts）只认这个契约， +// 不认任何一家的端点、字段或 DOM。新增一个站点 = 新增一个实现，不改编排。 + +import type { AssetRef, IRConversation } from '../core/ir' +import type { CancelToken, FetchStats } from '../core/fetcher' + +export type SiteId = 'chatgpt' | 'claude' + +export interface AssetPayload { + bytes: Uint8Array + /** 服务端给出的原始文件名，取不到时 null（调用方回退 AssetRef.name） */ + filename: string | null + contentType: string | null +} + +export type Rgb = [number, number, number] + +/** 站点专属的界面锚定与配色探测——唯一需要认识对方 DOM 的地方。 */ +export interface SiteUi { + /** 顶栏锚点（贴在分享按钮左侧，面板向下展开）；找不到返回 null */ + headerAnchor(): HTMLElement | null + /** 输入框锚点（贴在输入框旁，面板向上展开）；找不到返回 null */ + composerAnchor(): HTMLElement | null + /** 页面是否处于暗色 */ + isDark(): boolean + /** 需要监听的根元素属性，变化时重新同步主题 */ + themeAttributes: string[] + /** + * 站点专属主色。返回 null 时由界面层走通用兜底（扫描含 accent 的自定义属性）。 + * bg = 主色底，fg = 其上的文字色（null 表示让界面层按对比度自选），ring = 焦点环色。 + */ + accent(parse: (raw: string) => Rgb | null): { bg: Rgb; fg: Rgb | null; ring: Rgb | null } | null +} + +/** 列表项的站点无关形状（字段名沿用既有水位线逻辑，直接喂 selectChanged）。 */ +export interface SiteConversationItem { + id: string + title: string + update_time: string | number | null +} + +export interface SitePager { + /** 拉下一页；done=true 表示已确认到底 */ + next(): Promise<{ items: SiteConversationItem[]; done: boolean }> +} + +/** 批量导出能力。supportsBatch 为 true 时必须提供。 */ +export interface SiteBatch { + listAll( + session: string, + onProgress?: (fetched: number) => void, + cancel?: CancelToken, + ): Promise + createPager(session: string, cancel?: CancelToken): SitePager +} + +export interface SiteAdapter { + id: SiteId + /** 面板与轮次标题里显示的名字 */ + label: string + /** + * 是否开放批量 / 全量导出。 + * Claude 首版为 false：限流画像尚无实测数据，先只做当前对话。 + */ + supportsBatch: boolean + /** 当前页面是否属于这个站点 */ + matches(): boolean + /** 当前打开的对话 id；不在对话页时 null */ + currentConversationId(): string | null + /** 建立会话上下文（ChatGPT 换 accessToken，Claude 取 orgId），返回值透传给后续调用 */ + prepare(cancel?: CancelToken): Promise + /** 原始 JSON（raw 导出与 IR 转换共用同一次抓取） */ + fetchRaw(session: string, id: string, cancel?: CancelToken): Promise + /** 原始 JSON → IR */ + toIR(raw: unknown, fallbackId: string): IRConversation + /** 取附件字节 */ + fetchAsset( + session: string, + ref: AssetRef, + cancel?: CancelToken, + maxBytes?: number, + ): Promise + /** 限流观测快照：未知站点的节奏只能靠实测看清 */ + throttleStats(): FetchStats + /** 批量导出的实现；supportsBatch 为 false 时可缺省 */ + batch?: SiteBatch + ui: SiteUi +} diff --git a/src/ui.ts b/src/ui.ts index 44cd40e..ab884e7 100644 --- a/src/ui.ts +++ b/src/ui.ts @@ -1,4 +1,5 @@ -import { sanitizeSubdir } from './convert/markdown' +import { sanitizeSubdir } from './core/render' +import type { SiteUi } from './sites' export type ExportFormat = 'markdown' | 'json' export type ExportScope = 'current' | 'all' | 'selection' @@ -42,8 +43,6 @@ export interface PickerItem { id: string title: string updated: string - /** 所属 project 名；缺省 = 主列表会话 */ - project?: string } export interface PanelHandle { @@ -54,13 +53,15 @@ export interface PanelHandle { appendPicker(items: PickerItem[], done: boolean): void /** 清空多选列表（重新拉取前调用） */ clearPicker(): void - /** 填「来源」下拉里的 project 选项（当前选中项会保留） */ - setPickerProjects(projects: { id: string; name: string }[]): void /** 某一页拉取失败：解除加载中状态，允许再次触发 */ pickerLoadFailed(): void } export interface PanelCallbacks { + /** 当前站点：决定标题文案与批量入口是否出现 */ + site: { id: string; label: string; supportsBatch: boolean } + /** 站点专属的锚点与配色探测 */ + siteUi: SiteUi /** ids 仅在 scope === 'selection' 时有意义 */ onExport( scope: ExportScope, @@ -69,11 +70,8 @@ export interface PanelCallbacks { panel: PanelHandle, opts: ExportOptions, ): void - /** - * 首次切到「选择」/ 点重新拉取 / 切换「来源」：回调负责重置分页并拉第一页。 - * source 为 `all` / `main` / project 的 gizmo id。 - */ - onPickList(panel: PanelHandle, source: string): void + /** 首次切到「选择」或点重新拉取：回调负责重置分页并拉第一页 */ + onPickList(panel: PanelHandle): void /** 列表滚到底部：回调负责拉下一页并调用 panel.appendPicker */ onPickMore(panel: PanelHandle): void onCancel(): void @@ -232,18 +230,9 @@ const STYLE = ` .picker { display: none; margin-top: 8px; } .picker.open { display: block; } - .picker .srcrow { display: flex; gap: 4px; margin-bottom: 6px; } - .picker select.src { - flex-shrink: 0; max-width: 108px; padding: 6px 7px; border: 1px solid var(--border); - border-radius: 8px; font-size: 12px; background: transparent; color: var(--fg); - outline: none; cursor: pointer; transition: border-color .15s var(--ease); - } - .picker select.src:focus { border-color: var(--accent); } - /* 弹出的 option 在系统层渲染，不继承面板的透明背景，得给实色 */ - .picker select.src option { background: Canvas; color: CanvasText; } .picker input[type="search"] { - flex: 1; min-width: 0; padding: 6px 9px; border: 1px solid var(--border); border-radius: 8px; - font-size: 12px; background: transparent; color: var(--fg); outline: none; + width: 100%; padding: 6px 9px; border: 1px solid var(--border); border-radius: 8px; + font-size: 12px; margin-bottom: 6px; background: transparent; color: var(--fg); outline: none; transition: border-color .15s var(--ease); } .picker input[type="search"]::placeholder { color: var(--muted); } @@ -261,10 +250,6 @@ const STYLE = ` .picker .row:hover { background: var(--hover); } .picker .row.hidden { display: none; } .picker .row .t { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } - .picker .row .p { - flex-shrink: 0; max-width: 84px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; - color: var(--muted); font-size: 10px; padding: 1px 5px; border-radius: 999px; border: 1px solid var(--border); - } .picker .row .d { color: var(--muted); font-size: 10px; flex-shrink: 0; font-variant-numeric: tabular-nums; } .picker .sentinel { padding: 7px 0; text-align: center; color: var(--muted); font-size: 11px; } .picker .sentinel:empty { padding: 0; } @@ -442,15 +427,10 @@ export function mountPanel(cb: PanelCallbacks): void { let accentScanTick = 0 const detectAccent = (force = false): void => { const rootEl = document.documentElement - const rootCS = getComputedStyle(rootEl) - const theme = rootEl.getAttribute('data-chat-theme') || 'default' - const bg = parseColor(rootCS.getPropertyValue(`--${theme}-theme-submit-btn-bg`)) - if (bg) { - applyAccent( - bg, - parseColor(rootCS.getPropertyValue(`--${theme}-theme-submit-btn-text`)), - parseColor(rootCS.getPropertyValue(`--${theme}-theme-entity-accent`)), - ) + // 站点专属探测优先（各家主色变量命名不同）；探测不到再走下面的通用扫描 + const hit = cb.siteUi.accent(parseColor) + if (hit) { + applyAccent(hit.bg, hit.fg, hit.ring) return } // 改版兜底：扫 html/body 上含 accent 的自定义属性，取最饱和的可解析颜色 @@ -476,15 +456,15 @@ export function mountPanel(cb: PanelCallbacks): void { if (best) applyAccent(best, null, null) } - // 跟随 ChatGPT 主题（html.dark class）与 accent 设置（html[data-chat-theme]） + // 跟随页面的明暗与主题色设置（判定方式由站点适配器给） const syncTheme = () => { - host.dataset['theme'] = document.documentElement.classList.contains('dark') ? 'dark' : 'light' + host.dataset['theme'] = cb.siteUi.isDark() ? 'dark' : 'light' detectAccent(true) // 明暗/主题色切换时 accent 值跟着变 } syncTheme() new MutationObserver(syncTheme).observe(document.documentElement, { attributes: true, - attributeFilter: ['class', 'data-chat-theme'], + attributeFilter: cb.siteUi.themeAttributes, }) const style = document.createElement('style') @@ -493,36 +473,31 @@ export function mountPanel(cb: PanelCallbacks): void { const fab = document.createElement('button') fab.className = 'fab' - fab.title = 'Inkstone — 导出对话' - fab.setAttribute('aria-label', 'Inkstone — 导出对话') + fab.title = `Inkstone — 导出 ${cb.site.label} 对话` + fab.setAttribute('aria-label', `Inkstone — 导出 ${cb.site.label} 对话`) fab.setAttribute('aria-haspopup', 'dialog') fab.setAttribute('aria-expanded', 'false') // 关闭态 = 下载图标；打开态 = 向下箭头（收起面板），两层交叉淡出 - // pi-lens-ignore: ast-grep:no-inner-html, no-inner-html fab.innerHTML = `${ICON_DOWNLOAD}${ICON_ARROW_DOWN}` const panel = document.createElement('div') panel.className = 'panel' panel.setAttribute('role', 'dialog') panel.setAttribute('aria-label', '导出对话') - // pi-lens-ignore: ast-grep:no-inner-html, no-inner-html + // 批量能力未开放的站点直接不出现「全部 / 选择…」——按钮存在但点不动， + // 比它根本不出现更让人困惑 + const batchAttr = cb.site.supportsBatch ? '' : ' hidden' panel.innerHTML = ` -
导出对话
+
导出 ${cb.site.label} 对话
范围
- - + +
-
- - -
+
@@ -593,11 +568,7 @@ export function mountPanel(cb: PanelCallbacks): void { let curBottom = -1 let curPanelTop = -1 const findAnchor = (): HTMLElement | null => - (mode === 'header' - ? (document.querySelector('[data-testid="share-chat-button"]') ?? - document.querySelector('#conversation-header-actions')) - : (document.querySelector('#prompt-textarea')?.closest('form') ?? - document.querySelector('form[data-type="unified-composer"]'))) as HTMLElement | null + mode === 'header' ? cb.siteUi.headerAnchor() : cb.siteUi.composerAnchor() let anchor: HTMLElement | null = null const syncPos = (): void => { if (!anchor?.isConnected) return // 没有锚点：位置保持原样，藏与不藏由 rebindAnchor 决定 @@ -709,7 +680,6 @@ export function mountPanel(cb: PanelCallbacks): void { const pickerList = pickerEl.querySelector('.list')! const sentinel = pickerList.querySelector('.sentinel')! const pickerSearch = pickerEl.querySelector('input[type="search"]')! - const pickerSrc = pickerEl.querySelector('select.src')! const pickerEmpty = pickerEl.querySelector('.empty')! const pickerCount = pickerEl.querySelector('.count')! const segButtons = [...panel.querySelectorAll('.seg button')] @@ -867,8 +837,7 @@ export function mountPanel(cb: PanelCallbacks): void { for (const item of items) { const row = document.createElement('label') row.className = 'row' - // title 属性兼任 tooltip 与搜索词源，带上 project 名才能搜到项目下的会话 - row.title = item.project ? `${item.title}（${item.project}）` : item.title + row.title = item.title const box = document.createElement('input') box.type = 'checkbox' box.dataset['id'] = item.id @@ -878,14 +847,7 @@ export function mountPanel(cb: PanelCallbacks): void { const d = document.createElement('span') d.className = 'd' d.textContent = item.updated - if (item.project) { - const p = document.createElement('span') - p.className = 'p' - p.textContent = item.project - row.append(box, t, p, d) - } else { - row.append(box, t, d) - } + row.append(box, t, d) // 始终插在哨兵之前，哨兵保持在列表末尾 sentinel.before(row) } @@ -899,19 +861,6 @@ export function mountPanel(cb: PanelCallbacks): void { // 需要主动续拉，否则懒加载会停在第一页。 if (!done) queueMicrotask(maybeAutoFill) }, - setPickerProjects: (projects) => { - // 「全部」「主列表」两个固定项之后全量重建 project 选项，选中项按 value 复原 - const keep = pickerSrc.value - while (pickerSrc.options.length > 2) pickerSrc.remove(2) - for (const p of projects) { - const opt = document.createElement('option') - opt.value = p.id - opt.textContent = p.name - pickerSrc.add(opt) - } - // 项目被删掉时选中项会消失，退回「全部」——但不重拉，列表内容仍是有效的 - pickerSrc.value = [...pickerSrc.options].some((o) => o.value === keep) ? keep : 'all' - }, clearPicker: () => { for (const r of rows()) r.remove() listLoaded = false @@ -956,17 +905,15 @@ export function mountPanel(cb: PanelCallbacks): void { sentinel.addEventListener('click', requestMore) - /** 重置并拉第一页（首次进入「选择」/ 点重新拉取 / 切换来源） */ + /** 重置并拉第一页（首次进入「选择」/ 点重新拉取按钮） */ function loadList(): void { handle.clearPicker() pickerSearch.value = '' listLoading = true sentinel.textContent = '加载中…' - cb.onPickList(handle, pickerSrc.value) + cb.onPickList(handle) } - pickerSrc.addEventListener('change', loadList) - for (const btn of segButtons) { btn.addEventListener('click', () => { const group = btn.parentElement!.dataset['seg']! diff --git a/test/claude-pager.test.ts b/test/claude-pager.test.ts new file mode 100644 index 0000000..f5c41b9 --- /dev/null +++ b/test/claude-pager.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, test } from 'bun:test' +import { ApiError } from '../src/core/fetcher' +import { createConversationPager, type FetchPage } from '../src/sites/claude/api' +import type { ClaudeConversationListItem } from '../src/sites/claude/types' + +// api.ts 拼 URL 要用 location.origin，bun 环境里补一个 +;(globalThis as unknown as { location: { origin: string } }).location = { origin: 'https://claude.ai' } + +const items = (n: number, from = 0): ClaudeConversationListItem[] => + Array.from({ length: n }, (_, i) => ({ uuid: `c${from + i}` })) + +/** 记录每次翻页请求的 offset/limit，并按脚本返回结果 */ +function pages(script: (call: { offset: number; limit: number }, n: number) => ClaudeConversationListItem[]) { + const calls: { offset: number; limit: number }[] = [] + const fetchPage: FetchPage = (_org, offset, limit) => { + calls.push({ offset, limit }) + return Promise.resolve(script({ offset, limit }, calls.length)) + } + return { calls, fetchPage } +} + +const drain = async (pager: { next(): Promise<{ items: ClaudeConversationListItem[]; done: boolean }> }) => { + const all: ClaudeConversationListItem[] = [] + for (;;) { + const { items: page, done } = await pager.next() + all.push(...page) + if (done) return all + } +} + +describe('claude 分页器', () => { + test('按 offset 逐页翻；空页要连续确认 3 次才认到底', async () => { + // 空页不轻信是继承自 ChatGPT 侧的防御：那边实测过列表索引会瞬时降级， + // 提前返回空页而对话其实都还在。宁可多问两次，也不要漏掉半个历史。 + const { calls, fetchPage } = pages(({ offset }) => (offset < 100 ? items(50, offset) : [])) + const all = await drain(createConversationPager('org', undefined, { fetchPage, emptyRetryBaseMs: 0 })) + expect(all.length).toBe(100) + expect(calls.map((c) => c.offset)).toEqual([0, 50, 100, 100, 100]) + }) + + test('服务端忽略分页参数、每次返回同一批时立即收尾，不空转', async () => { + // 这是 claude.ai 侧尚未实测的分支：limit/offset 可能根本不被认。 + // 一旦发生，第二页会原样返回第一页——只能就此收尾，再问也是同样的东西。 + const { calls, fetchPage } = pages(() => items(30)) + const all = await drain(createConversationPager('org', undefined, { fetchPage, emptyRetryBaseMs: 0 })) + expect(all.length).toBe(30) + expect(calls.length).toBe(2) + }) + + test('部分重复时只保留新条目', async () => { + const { fetchPage } = pages((_c, n) => (n === 1 ? items(10) : n === 2 ? items(10, 5) : [])) + const all = await drain(createConversationPager('org', undefined, { fetchPage, emptyRetryBaseMs: 0 })) + expect(all.map((i) => i.uuid)).toEqual( + Array.from({ length: 15 }, (_, i) => `c${i}`), + ) + }) + + test('首页即空 = 账号没有对话，不重试', async () => { + const { calls, fetchPage } = pages(() => []) + const all = await drain(createConversationPager('org', undefined, { fetchPage, emptyRetryBaseMs: 0 })) + expect(all.length).toBe(0) + expect(calls.length).toBe(1) + }) + + test('非限流的 4xx 先降 limit 再试一次', async () => { + const { calls, fetchPage } = pages(({ limit }, n) => { + if (n === 1 && limit > 20) throw new ApiError(400, 'limit too large') + return n === 1 ? [] : items(5) + }) + const pager = createConversationPager('org', undefined, { fetchPage, emptyRetryBaseMs: 0 }) + const first = await pager.next() + expect(first.items.length).toBe(5) + expect(calls.map((c) => c.limit)).toEqual([50, 20]) + }) + + test('限流错误直接抛出，不降级也不吞掉', async () => { + const { fetchPage } = pages(() => { + throw new ApiError(429, 'rate limited') + }) + const pager = createConversationPager('org', undefined, { fetchPage, emptyRetryBaseMs: 0 }) + await expect(pager.next()).rejects.toThrow('rate limited') + }) + + test('取消后不再发请求', async () => { + const cancel = { cancelled: true } + const { calls, fetchPage } = pages(() => items(5)) + const pager = createConversationPager('org', cancel, { fetchPage, emptyRetryBaseMs: 0 }) + await expect(pager.next()).rejects.toThrow('已取消') + expect(calls.length).toBe(0) + }) + + test('收尾后再调直接返回 done，不重复请求', async () => { + const { calls, fetchPage } = pages(() => []) + const pager = createConversationPager('org', undefined, { fetchPage, emptyRetryBaseMs: 0 }) + await pager.next() + const again = await pager.next() + expect(again).toEqual({ items: [], done: true }) + expect(calls.length).toBe(1) + }) +}) diff --git a/test/claude.test.ts b/test/claude.test.ts new file mode 100644 index 0000000..ac29d14 --- /dev/null +++ b/test/claude.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, test } from 'bun:test' +import { renderConversation } from '../src/core/render' +import { replayArtifacts } from '../src/sites/claude/artifacts' +import { conversationToIR, linearize } from '../src/sites/claude/convert' +import type { ClaudeConversation, ClaudeMessage } from '../src/sites/claude/types' +import fixtureJson from './fixtures/claude-basic.json' + +const fixture = fixtureJson as unknown as ClaudeConversation + +const md = (conv: ClaudeConversation, opts = {}): string => + renderConversation(conversationToIR(conv, 'fallback'), opts).markdown + +/** 只有 assistant 一轮的最小对话，用来单点验证块渲染。 */ +function withBlocks(blocks: unknown[]): ClaudeConversation { + return { + uuid: 'c1', + name: 'T', + current_leaf_message_uuid: 'a1', + chat_messages: [ + { + uuid: 'a1', + parent_message_uuid: null, + sender: 'assistant', + content: blocks, + } as unknown as ClaudeMessage, + ], + } +} + +describe('linearize', () => { + test('沿 current_leaf 回溯主线，丢弃被重新生成的旧分支', () => { + const ids = linearize(fixture).map((m) => m.uuid) + expect(ids).toEqual(['m1', 'm2', 'm3', 'm4']) + expect(ids).not.toContain('m2-abandoned') + }) + + test('leaf 缺失时退回 index 排序而不是丢消息', () => { + const ids = linearize({ ...fixture, current_leaf_message_uuid: null }).map((m) => m.uuid) + expect(ids.length).toBe(5) + expect(ids[0]).toBe('m1') + }) + + test('parent 链成环也能收敛', () => { + const conv: ClaudeConversation = { + current_leaf_message_uuid: 'x', + chat_messages: [ + { uuid: 'x', parent_message_uuid: 'y', sender: 'human' }, + { uuid: 'y', parent_message_uuid: 'x', sender: 'assistant' }, + ] as ClaudeMessage[], + } + expect(linearize(conv).map((m) => m.uuid).sort()).toEqual(['x', 'y']) + }) + + test('非 human/assistant 的消息不参与轮次', () => { + const conv: ClaudeConversation = { + current_leaf_message_uuid: 'b', + chat_messages: [ + { uuid: 'a', parent_message_uuid: null, sender: 'system' }, + { uuid: 'b', parent_message_uuid: 'a', sender: 'human' }, + ] as ClaudeMessage[], + } + expect(linearize(conv).map((m) => m.uuid)).toEqual(['b']) + }) +}) + +describe('artifact 折叠', () => { + test('create + update 折叠成终稿，只在最后一次编辑处出现', () => { + const out = md(fixture) + expect(out).toContain('function a() {') + expect(out).toContain('return 2') + expect(out).not.toContain('return 1') + // 终稿整块出现一次；创建处只留一行指引 + expect(out.match(/> \[!abstract\] Artifact · 工具函数/g)?.length).toBe(1) + expect(out).toContain('*(Artifact 创建「工具函数」，终稿见后)*') + }) + + test('new_str 里的 $& 等替换模式按字面量写入，不被 replace 语义吞掉', () => { + // 字符串形式的 replace 会把 $& 展开成匹配到的文本，静默损坏代码 + const conv = withBlocks([ + { + type: 'tool_use', + name: 'artifacts', + input: { command: 'create', id: 'a', type: 'application/vnd.ant.code', content: 'X', version_uuid: 'v1' }, + }, + { + type: 'tool_use', + name: 'artifacts', + input: { command: 'update', id: 'a', old_str: 'X', new_str: 'cost($&) + $` + $$9', version_uuid: 'v2' }, + }, + ]) + expect(md(conv)).toContain('cost($&) + $` + $$9') + }) + + test('old_str 匹配不上就放弃重放，原始 JSON 兜底且不受 toolTraces 开关', () => { + const conv = withBlocks([ + { + type: 'tool_use', + name: 'artifacts', + input: { command: 'create', id: 'a', content: 'hello', version_uuid: 'v1' }, + }, + { + type: 'tool_use', + name: 'artifacts', + input: { command: 'update', id: 'a', old_str: '不存在的文本', new_str: 'x', version_uuid: 'v2' }, + }, + ]) + const out = md(conv) // 注意：没开 toolTraces + expect(out).toContain('重放失败，原始 JSON') + expect(out).toContain('不存在的文本') + // 失配之前的终稿仍要留下，不能因为一次失败就整份丢掉 + expect(out).toContain('hello') + }) + + test('rewrite 重新奠定基线，之前的失配不影响后续', () => { + const messages = [ + { + uuid: 'm', + sender: 'assistant', + content: [ + { type: 'tool_use', name: 'artifacts', input: { command: 'create', id: 'a', content: 'v1', version_uuid: '1' } }, + { type: 'tool_use', name: 'artifacts', input: { command: 'update', id: 'a', old_str: 'zzz', new_str: 'q', version_uuid: '2' } }, + { type: 'tool_use', name: 'artifacts', input: { command: 'rewrite', id: 'a', content: '全新内容', version_uuid: '3' } }, + ], + }, + ] as unknown as ClaudeMessage[] + const ops = replayArtifacts(messages) + expect(ops.get('m#2')?.finalContent).toBe('全新内容') + }) + + test('artifact 类型决定是否套围栏', () => { + const codeOut = md( + withBlocks([ + { + type: 'tool_use', + name: 'artifacts', + input: { + command: 'create', + id: 'a', + title: 'demo', + type: 'application/vnd.ant.code', + language: 'python', + content: 'print(1)', + version_uuid: 'v1', + }, + }, + ]), + ) + expect(codeOut).toContain('```python') + + const mdOut = md( + withBlocks([ + { + type: 'tool_use', + name: 'artifacts', + input: { + command: 'create', + id: 'a', + title: 'doc', + type: 'text/markdown', + content: '# 文档标题', + version_uuid: 'v1', + }, + }, + ]), + ) + // Markdown artifact 走正文管道：标题降一级，不套围栏 + expect(mdOut).toContain('## 文档标题') + expect(mdOut).not.toContain('```') + }) +}) + +describe('内容块分发', () => { + test('thinking 默认不写入，开关打开后进折叠 callout', () => { + expect(md(fixture)).not.toContain('用户要一个最小函数') + const withThoughts = md(fixture, { thoughts: true }) + expect(withThoughts).toContain('> [!quote]- 思考过程') + expect(withThoughts).toContain('**分析需求**') + expect(withThoughts).toContain('用户要一个最小函数') + }) + + test('工具痕迹默认不写入，开关打开后进折叠 callout', () => { + const out = md(fixture) + expect(out).not.toContain('web_search') + expect(out).not.toContain('搜索返回了三条结果') + + const traced = md(fixture, { toolTraces: true }) + expect(traced).toContain('工具调用 → `web_search`') + expect(traced).toContain('搜索返回了三条结果') + }) + + test('未识别的块类型留原始 JSON，永不静默丢内容', () => { + const out = md(withBlocks([{ type: 'future_block', payload: { keep: '这段不能丢' } }])) + expect(out).toContain('未识别的内容块 `future_block`') + expect(out).toContain('这段不能丢') + }) + + test('用户中止与 truncated 如实标注', () => { + const conv = withBlocks([{ type: 'text', text: '半句话' }]) + conv.chat_messages![0]!.stop_reason = 'user_canceled' + conv.chat_messages![0]!.truncated = true + const out = md(conv) + expect(out).toContain('*(这条回复被用户中止)*') + expect(out).toContain('*(服务端将这条消息标记为 truncated)*') + }) + + test('create_file 与 widget 作为内容承载块写入', () => { + const out = md( + withBlocks([ + { type: 'tool_use', name: 'create_file', input: { path: 'src/a.py', file_text: 'print(1)' } }, + { type: 'tool_use', name: 'visualize:show_widget', input: { title: '图表', widget_code: '
' } }, + ]), + ) + expect(out).toContain('> [!abstract] 文件 · src/a.py') + expect(out).toContain('```python') + expect(out).toContain('> [!abstract] Widget · 图表') + }) +}) + +describe('附件', () => { + test('图片留占位符，文本抽取件整块内联', () => { + const out = md(fixture) + expect(out).toContain('%%INKSTONE-ASSET-img-1%%') + expect(out).toContain('> [!abstract] 附件 · 说明.md（文本抽取）') + // 附件自带的标题被降级并包在 callout 里，不污染文档大纲 + expect(out).toContain('> ## 附件自带的标题') + }) + + test('图片附件带上下载地址交给编排层', () => { + const { assets } = renderConversation(conversationToIR(fixture, '')) + const img = assets.find((a) => a.fileId === 'img-1') + expect(img?.kind).toBe('image') + expect(img?.url).toBe('/api/files/img-1/preview') + }) + + test('拿不到地址的附件留名字，不假装能下载', () => { + const conv = withBlocks([{ type: 'text', text: 'x' }]) + conv.chat_messages![0]!.files = [{ file_kind: 'blob', file_name: '录音.m4a' }] + const out = md(conv) + expect(out).toContain('*(附件：录音.m4a · blob — 无可下载地址)*') + }) +}) + +describe('引用与 frontmatter', () => { + test('引用汇总进 Sources，过期来源不写入', () => { + const out = md(fixture) + expect(out).toContain('# Sources') + expect(out).toContain('- [参考 A](https://example.com/a)') + expect(out).not.toContain('expired.example.com') + }) + + test('frontmatter 带站点标记、项目与地址', () => { + const out = md(fixture) + expect(out).toContain('chat_id: 3f9a1c20-77b4-4e0d-9f21-0a5b6c7d8e9f') + expect(out).toContain('url: https://claude.ai/chat/3f9a1c20-77b4-4e0d-9f21-0a5b6c7d8e9f') + expect(out).toContain('created: 2026-08-01T10:00:00.000Z') + expect(out).toContain('model: claude-opus-5') + expect(out).toContain('project: "研究笔记"') + expect(out).toContain(' - claude') + }) + + test('轮次标题用 Claude 而不是 ChatGPT', () => { + const out = md(fixture) + expect(out).toContain('# User') + expect(out).toContain('# Claude') + expect(out).not.toContain('# ChatGPT') + }) + + test('公式定界符照样转换（与 ChatGPT 侧共用同一段管道）', () => { + expect(md(fixture)).toContain('$x^2$') + }) + + test('标题为空时回退 Untitled', () => { + expect(md({ ...fixture, name: ' ' })).toContain('title: "Untitled"') + }) +}) diff --git a/test/fixtures/claude-basic.json b/test/fixtures/claude-basic.json new file mode 100644 index 0000000..b082356 --- /dev/null +++ b/test/fixtures/claude-basic.json @@ -0,0 +1,119 @@ +{ + "uuid": "3f9a1c20-77b4-4e0d-9f21-0a5b6c7d8e9f", + "name": "测试对话", + "model": "claude-opus-5", + "created_at": "2026-08-01T10:00:00.000Z", + "updated_at": "2026-08-02T11:30:00.000Z", + "current_leaf_message_uuid": "m4", + "project": { "uuid": "p1", "name": "研究笔记" }, + "chat_messages": [ + { + "uuid": "m1", + "parent_message_uuid": null, + "index": 0, + "sender": "human", + "created_at": "2026-08-01T10:00:00.000Z", + "text": "", + "content": [{ "type": "text", "text": "帮我写个返回 1 的函数" }] + }, + { + "uuid": "m2", + "parent_message_uuid": "m1", + "index": 1, + "sender": "assistant", + "created_at": "2026-08-01T10:00:20.000Z", + "content": [ + { + "type": "thinking", + "thinking": "用户要一个最小函数，直接给 artifact。", + "summaries": [{ "summary": "分析需求" }] + }, + { "type": "text", "text": "好的，见下面的 artifact。" }, + { + "type": "tool_use", + "name": "artifacts", + "input": { + "command": "create", + "id": "a1", + "title": "工具函数", + "type": "application/vnd.ant.code", + "language": "javascript", + "content": "function a() {\n return 1\n}", + "version_uuid": "v1" + } + } + ] + }, + { + "uuid": "m2-abandoned", + "parent_message_uuid": "m1", + "index": 1, + "sender": "assistant", + "created_at": "2026-08-01T10:00:10.000Z", + "content": [{ "type": "text", "text": "这是重新生成前的旧回复，不该出现在导出里" }] + }, + { + "uuid": "m3", + "parent_message_uuid": "m2", + "index": 2, + "sender": "human", + "created_at": "2026-08-01T10:05:00.000Z", + "content": [{ "type": "text", "text": "改成返回 2" }], + "attachments": [ + { + "file_name": "说明.md", + "file_type": "text/markdown", + "file_size": 42, + "extracted_content": "# 附件自带的标题\n附件正文一行。" + } + ], + "files": [ + { + "file_kind": "image", + "file_name": "截图.png", + "file_uuid": "img-1", + "preview_url": "/api/files/img-1/preview", + "size_bytes": 2048 + } + ] + }, + { + "uuid": "m4", + "parent_message_uuid": "m3", + "index": 3, + "sender": "assistant", + "created_at": "2026-08-01T10:05:30.000Z", + "content": [ + { + "type": "tool_use", + "name": "web_search", + "input": { "query": "javascript function" } + }, + { + "type": "tool_result", + "content": "搜索返回了三条结果", + "is_error": false + }, + { + "type": "tool_use", + "name": "artifacts", + "input": { + "command": "update", + "id": "a1", + "old_str": "return 1", + "new_str": "return 2", + "version_uuid": "v2" + } + }, + { + "type": "text", + "text": "改好了，公式 \\(x^2\\) 也顺带演示一下。", + "citations": [ + { "url": "https://example.com/a", "title": "参考 A" }, + { "url": "https://expired.example.com/b", "title": "过期来源", "is_expired": true } + ] + } + ] + } + ] +} diff --git a/vite.config.ts b/vite.config.ts index 5d445af..62a7f66 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,22 +1,34 @@ import { defineConfig } from 'vite' import monkey from 'vite-plugin-monkey' +// 砚台图标（墨滴 + 下载箭头）。内联 data URI，不依赖任何站点的 favicon—— +// 脚本现在同时服务 chatgpt.com 与 claude.ai，借用其中一家的图标会名不副实。 +const ICON = + 'data:image/svg+xml,' + + encodeURIComponent( + ``, + ) + export default defineConfig({ plugins: [ monkey({ entry: 'src/main.ts', userscript: { name: { - '': 'Inkstone — ChatGPT Conversation Exporter', - 'zh-CN': 'Inkstone — ChatGPT 对话导出', + '': 'Inkstone — ChatGPT & Claude Conversation Exporter', + 'zh-CN': 'Inkstone — ChatGPT / Claude 对话导出', }, namespace: 'https://github.com/ZhenHuangLab/inkstone', description: { - '': 'Grind ChatGPT conversations into Obsidian-friendly Markdown — high-fidelity batch export (math / citations / attachments)', - 'zh-CN': '砚 · 把 ChatGPT 对话研磨成 Obsidian 友好的 Markdown，高保真批量导出（公式 / 引用 / 附件）', + '': + 'Grind ChatGPT and Claude conversations into Obsidian-friendly Markdown — ' + + 'high-fidelity export (math / citations / attachments / artifacts)', + 'zh-CN': + '砚 · 把 ChatGPT 与 Claude 对话研磨成 Obsidian 友好的 Markdown，' + + '高保真导出（公式 / 引用 / 附件 / Artifact）', }, - match: ['https://chatgpt.com/*', 'https://chat.openai.com/*'], - icon: 'https://chatgpt.com/favicon.ico', + match: ['https://chatgpt.com/*', 'https://chat.openai.com/*', 'https://claude.ai/*'], + icon: ICON, license: 'GPL-3.0-only', 'run-at': 'document-idle', noframes: true, From f74bf098cf726079f6badf67c1a127f9d680ab15 Mon Sep 17 00:00:00 2001 From: pmwl Date: Fri, 28 Aug 2026 20:18:11 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(claude):=20=E4=BF=AE=E5=A4=8D=E6=B5=AE?= =?UTF-8?q?=E7=AA=97=E5=AE=9A=E4=BD=8D=E4=B8=8E=E7=94=9F=E6=88=90=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + PLAN.md | 13 ++- README.md | 4 +- src/main.ts | 30 ++---- src/output/naming.ts | 42 +++++++++ src/sites/claude/api.ts | 39 +++++++- src/sites/claude/convert.ts | 180 +++++++++++++++++++++++++++++++++++- src/sites/claude/index.ts | 49 ++++++++-- src/sites/claude/types.ts | 20 +++- src/sites/types.ts | 9 +- src/ui-position.ts | 77 +++++++++++++++ src/ui.ts | 135 ++++++++++++++++++--------- test/claude.test.ts | 85 ++++++++++++++++- test/output-naming.test.ts | 41 ++++++++ test/ui-position.test.ts | 40 ++++++++ 15 files changed, 675 insertions(+), 90 deletions(-) create mode 100644 src/output/naming.ts create mode 100644 src/ui-position.ts create mode 100644 test/output-naming.test.ts create mode 100644 test/ui-position.test.ts diff --git a/.gitignore b/.gitignore index b518b92..9e2a6c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ dist/ +/cases/ *.local .DS_Store diff --git a/PLAN.md b/PLAN.md index 71de94d..cdb1abf 100644 --- a/PLAN.md +++ b/PLAN.md @@ -42,7 +42,7 @@ - `thoughts`（推理模型思维链）→ 折叠 callout，默认带、可关 - Canvas textdoc → MVP 整块嵌入，后续做 patch 重放还原终稿 - **未知类型 → 原始 JSON 塞进折叠 callout，永不静默丢内容** -6. **附件管道**：`file-service://` 与 `sediment://` 指针 → files 接口下载 → 笔记同目录下的附件子文件夹（默认 `conversations/attachments/`）→ 链接改写（wikilink `![[...]]` / 标准相对链接，可配；链接相对 .md 严格成立）；下载失败留占位说明并在完成文案里报数。笔记/附件子文件夹名均可设置（可 `a/b` 嵌套、可留空；`sanitizeSubdir` 逐段净化防逃逸），油猴设置面板与 CLI `--notes-dir`/`--attachments-dir` 同规则。⚠️ 直写 vault 时注意 macOS 大小写不敏感：目录名撞上 vault 已有目录（如 `Attachments`）会直接写进去——2026-07-10 排查的"附件丢失"即此叠加 fast-note-sync 云预览删本地所致，非代码 bug。 +6. **附件管道**：`file-service://` 与 `sediment://` 指针 → files 接口下载 → 笔记同目录下的附件子文件夹（默认 `conversations/attachments/`）→ 链接改写（wikilink `![[...]]` / 标准相对链接，可配；标准 Markdown 路径相对当前 `.md`，Obsidian Wikilink 的目录路径相对 vault 根）；下载失败留占位说明并在完成文案里报数。笔记/附件子文件夹名均可设置（可 `a/b` 嵌套、可留空；`sanitizeSubdir` 逐段净化防逃逸），油猴设置面板与 CLI `--notes-dir`/`--attachments-dir` 同规则。⚠️ 直写 vault 时注意 macOS 大小写不敏感：目录名撞上 vault 已有目录（如 `Attachments`）会直接写进去——2026-07-10 排查的"附件丢失"即此叠加 fast-note-sync 云预览删本地所致，非代码 bug。 7. **Frontmatter**：title、chat_id、url（`chatgpt.com/c/`）、created、updated、model（实际生成消息的 model_slug，最后一条为准；多模型另列 models）、tags —— 配 Dataview 即全库索引。 8. **文件名**：`标题-短id.md` 防重名（无空格/波浪线，非法字符与空白归一为 `-`），id 保证增量覆盖稳定。 @@ -141,8 +141,15 @@ inkstone/ 服务端要求的最长等待——未知站点的节奏只能靠实测看清，先让它可见再谈调参。 - **待实测**：`docs/claude-probe.js` 可直接粘进 claude.ai 控制台，打印字段骨架 （不打印对话内容）并做一次 ≤8 请求的保守限流试探。清单见可行性文档第五节： - 分页是否生效、thinking 字段名、citations 挂载形态、附件地址能否直取字节、 - 公式定界符、Projects 字段名、FAB 锚点选择器。 + 分页是否生效、thinking 字段名、citations 挂载形态、普通附件地址能否直取字节、 + 公式定界符、Projects 字段名。FAB 锚点已在 2026-08-28 的真实会话页确认： + 顶栏动作为 `[data-testid="wiggle-controls-actions-group"]`（需锚定整个组的左边界， + 不能只锚定 Share，否则会盖住 Files），输入框为 + `[data-testid="chat-input"][contenteditable="true"]`。同一轮实测还确认 Claude 的 + 图片 `preview_url` 可能把上传 PNG 转码为 WebP，附件落盘扩展名必须服从响应 MIME。 + 生成文件卡片则不在消息的 `files[]`：`present_files` 给出展示路径，实际文件需先从 + `.../conversations/{id}/wiggle/list-files` 取清单，再经 `wiggle/download-file` 下载。 + 默认 Markdown 只链接最终沙箱文件，不再重复嵌入每次 `create_file` 的完整中间版本。 - **未做**：Claude 的行内引用锚定（citations 的字符级定位字段未实测，首版只把 来源汇总进文末 Sources，正文一字不动）；Claude 官方导出 zip 的离线 CLI 通道。 diff --git a/README.md b/README.md index ff023b5..a63df45 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Inkstone runs inside the page and fetches conversations through the same backend | Incremental sync | ✅ | ⏳ not yet enabled | | Rich documents | Canvas patch replay | Artifact fold-up to final version | | Thoughts / tool traces | ✅ opt-in | ✅ opt-in | -| Attachments | images and files downloaded | images downloaded, documents linked, text extractions inlined | +| Attachments | images and files downloaded | images, documents, and generated files downloaded; text extractions inlined | **Why no batch export on Claude yet?** It isn't missing, it's switched off. The pager, watermark, concurrency pool and protective abort are all in place and unit-tested — but @@ -162,4 +162,4 @@ Batch export on Claude once its rate-limit profile has actually been measured, a ## Related Links -[LINUX DO](https://linux.do) \ No newline at end of file +[LINUX DO](https://linux.do) diff --git a/src/main.ts b/src/main.ts index 3875e40..b096d68 100644 --- a/src/main.ts +++ b/src/main.ts @@ -21,6 +21,7 @@ import { type SitePager, } from './sites' import { downloadBlob, makeZip, strToU8, type ZipEntries } from './output/zip' +import { assetFileName, assetReferencePath } from './output/naming' import { acquireVaultDir, forgetVaultDir, @@ -303,10 +304,10 @@ function createProcessor( try { const { bytes, filename, contentType } = await site.fetchAsset(session, a, cancel, cap) const name = assetFileName(a, filename, contentType) - // 链接相对 .md 所在目录，落盘再套上笔记目录前缀 + // 文件落在笔记目录下；标准 Markdown 用相对笔记路径，Wikilink 用 vault 根路径。 const linkPath = `${attachPrefix}${a.fileId.slice(-8)}-${name}` await sink.put(`${notesPrefix}${linkPath}`, bytes, { precompressed: true }) - replacement = assetLink(opts.linkStyle, linkPath, { + replacement = assetLink(opts.linkStyle, assetReferencePath(opts.linkStyle, notesPrefix, linkPath), { embed: a.kind === 'image', label: a.kind === 'image' ? undefined : (a.name ?? name), }) @@ -344,7 +345,10 @@ function createProcessor( await sink.put(path, strToU8(JSON.stringify(raw, null, 2))) return { path } } - const { markdown, title, assets } = renderConversation(site.toIR(raw, item.id), { + const irContext = site.fetchIRContext + ? await site.fetchIRContext(session, item.id, raw, cancel) + : undefined + const { markdown, title, assets } = renderConversation(site.toIR(raw, item.id, irContext), { thoughts: opts.thoughts, toolTraces: opts.toolTraces, headingMode: opts.headingMode, @@ -572,26 +576,6 @@ async function exportItems( ) } -const EXT_BY_MIME: Record = { - 'image/png': '.png', - 'image/webp': '.webp', - 'image/jpeg': '.jpg', - 'image/gif': '.gif', -} - -function assetFileName(a: AssetRef, downloadName: string | null, contentType: string | null): string { - const raw = sanitizeName(downloadName ?? a.name ?? '') - // 截断只砍主名，扩展名要保住 - const ext = /\.[A-Za-z0-9]{1,8}$/.exec(raw)?.[0] ?? '' - const base = (ext ? raw.slice(0, -ext.length) : raw).slice(0, 60).trim() - let name = (base || (a.kind === 'image' ? 'image' : 'file')) + ext - if (!/\.[A-Za-z0-9]{1,8}$/.test(name)) { - const mimeExt = EXT_BY_MIME[(contentType ?? '').split(';')[0]!.trim()] - if (mimeExt) name += mimeExt - } - return name -} - function fmtSize(bytes: number): string { if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)}MB` if (bytes >= 1024) return `${(bytes / 1024).toFixed(0)}KB` diff --git a/src/output/naming.ts b/src/output/naming.ts new file mode 100644 index 0000000..e7a2f0d --- /dev/null +++ b/src/output/naming.ts @@ -0,0 +1,42 @@ +import type { AssetRef } from '../core/ir' +import { sanitizeName } from '../core/render' + +const EXT_BY_MIME: Record = { + 'image/avif': '.avif', + 'image/gif': '.gif', + 'image/jpeg': '.jpg', + 'image/png': '.png', + 'image/svg+xml': '.svg', + 'image/webp': '.webp', +} + +/** + * 下载落盘名必须服从实际响应 MIME。Claude 的图片 preview_url 会把上传的 PNG + * 转码为 WebP；若继续沿用原文件名，Markdown 会得到扩展名与字节格式不一致的附件。 + */ +export function assetFileName( + asset: AssetRef, + downloadName: string | null, + contentType: string | null, +): string { + const raw = sanitizeName(downloadName ?? asset.name ?? '') + const matchedExt = /\.[A-Za-z0-9]{1,8}$/.exec(raw)?.[0] ?? '' + const base = (matchedExt ? raw.slice(0, -matchedExt.length) : raw).slice(0, 60).trim() + const mime = (contentType ?? '').split(';')[0]!.trim().toLowerCase() + const mimeExt = EXT_BY_MIME[mime] + + // 已知图片 MIME 是实际下载字节的权威格式；未知 MIME 才保留来源文件名扩展。 + const ext = mimeExt ?? matchedExt + return (base || (asset.kind === 'image' ? 'image' : 'file')) + ext +} + +/** + * 标准 Markdown 链接相对当前笔记；Obsidian Wikilink 的带目录路径则相对 vault 根。 + */ +export function assetReferencePath( + style: 'wikilink' | 'markdown', + notesPrefix: string, + relativePath: string, +): string { + return style === 'wikilink' ? `${notesPrefix}${relativePath}` : relativePath +} diff --git a/src/sites/claude/api.ts b/src/sites/claude/api.ts index e0c60e8..ecbc152 100644 --- a/src/sites/claude/api.ts +++ b/src/sites/claude/api.ts @@ -18,7 +18,12 @@ import { type Fetcher, type ThrottleConfig, } from '../../core/fetcher' -import type { ClaudeConversation, ClaudeConversationListItem, ClaudeOrganization } from './types' +import type { + ClaudeConversation, + ClaudeConversationListItem, + ClaudeOrganization, + ClaudeSandboxFile, +} from './types' export const CLAUDE_THROTTLE: ThrottleConfig = { spacingBaseMs: 1500, @@ -77,6 +82,38 @@ export async function fetchConversation( return (await res.json()) as ClaudeConversation } +/** + * 会话沙箱清单：用户上传件与 Claude 生成的最终文件都在这里。 + * 注意端点是 conversations，不是主对话接口使用的 chat_conversations。 + */ +export async function listSandboxFiles( + orgId: string, + conversationId: string, + cancel?: CancelToken, +): Promise { + const listUrl = api( + `/api/organizations/${orgId}/conversations/${conversationId}/wiggle/list-files?prefix=`, + ) + const res = await fetcher.request(listUrl, { headers: { Accept: 'application/json' } }, cancel) + const data: unknown = await res.json() + if (!data || typeof data !== 'object' || !Array.isArray((data as { files_metadata?: unknown }).files_metadata)) { + throw new Error('Claude 沙箱文件清单结构已变化') + } + const files = (data as { files_metadata: unknown[] }).files_metadata + + return files.flatMap((item): ClaudeSandboxFile[] => { + if (!item || typeof item !== 'object') return [] + const raw = item as Record + if (typeof raw['path'] !== 'string' || raw['path'] === '') return [] + const path = raw['path'] + const downloadUrl = api( + `/api/organizations/${orgId}/conversations/${conversationId}/wiggle/download-file` + + `?path=${encodeURIComponent(path)}`, + ) + return [{ ...(raw as Omit), path, download_url: downloadUrl }] + }) +} + // ——— 以下是批量导出的地基，当前版本的 UI 不暴露 ——— // 单对话导出只需要上面两个端点。列表接口先按分页写好（形状与 ChatGPT 侧一致， // 便于后续复用同一套编排），但在限流画像实测清楚之前不接进界面。 diff --git a/src/sites/claude/convert.ts b/src/sites/claude/convert.ts index 8465157..e37e23c 100644 --- a/src/sites/claude/convert.ts +++ b/src/sites/claude/convert.ts @@ -7,18 +7,34 @@ import type { AssetRef, IRBlock, IRConversation, IRTurn, SourceLink } from '../../core/ir' import { toIso, yamlQuote } from '../../core/render' import { artifactDocType, blockKey, replayArtifacts, type ArtifactOp } from './artifacts' -import type { ClaudeContentBlock, ClaudeConversation, ClaudeMessage } from './types' +import type { + ClaudeContentBlock, + ClaudeConversation, + ClaudeMessage, + ClaudeSandboxFile, +} from './types' interface Ctx { /** blockKey → 重放成功的 artifact 操作；不在表里的走原始 JSON 兜底 */ artifacts: Map + sandboxByPath: Map + sandboxByBasename: Map + sandboxOutputs: ClaudeSandboxFile[] + presentedPaths: Set + usedPresentFallback: boolean + sandboxUnavailable: boolean } -export function conversationToIR(conv: ClaudeConversation, fallbackId = ''): IRConversation { +export function conversationToIR( + conv: ClaudeConversation, + fallbackId = '', + sandboxFiles: readonly ClaudeSandboxFile[] = [], + sandboxUnavailable = false, +): IRConversation { const convId = String(conv.uuid ?? fallbackId) const title = (conv.name ?? '').trim() || 'Untitled' const messages = linearize(conv) - const ctx: Ctx = { artifacts: replayArtifacts(messages) } + const ctx = buildContext(messages, sandboxFiles, sandboxUnavailable) const turns: IRTurn[] = groupTurns(messages).map((t) => ({ role: t.role, @@ -174,6 +190,30 @@ function toolUseBlocks( const name = str(b.name) const input = b.input ?? {} + if (name === 'present_files') { + let files = sandboxFilesForInput(input, ctx) + // 新旧 present_files 参数名有差异。识别不到路径时，只在第一次调用回退到 + // outputs 全集；这仍比静默丢掉网页上明确可下载的文件卡片更忠实。 + if (files.length === 0 && !ctx.usedPresentFallback) { + ctx.usedPresentFallback = true + files = ctx.sandboxOutputs + } + if (files.length > 0) { + return [{ kind: 'assetList', refs: files.map(sandboxAssetRef) }] + } + if (ctx.sandboxUnavailable) { + return [{ kind: 'note', text: '*(Claude 生成文件清单获取失败，本次未能下载这些文件)*' }] + } + return [ + { + kind: 'tool', + title: '工具调用 → `present_files`', + body: JSON.stringify(input, null, 2), + lang: 'json', + }, + ] + } + if (name === 'artifacts') { const op = ctx.artifacts.get(blockKey(msgUuid, index)) if (!op) { @@ -211,6 +251,19 @@ function toolUseBlocks( if (name === 'create_file' && typeof input['file_text'] === 'string') { const path = str(input['path']) || 'file' + const sandboxFile = sandboxFileForPath(path, ctx) + if (sandboxFile && ctx.presentedPaths.has(sandboxFile.path)) { + // 最终文件会由 present_files 生成可下载链接；默认不再把每次 create_file + // 的完整中间版本重复塞进 Markdown。打开“工具过程”仍可查看调用参数。 + return [ + { + kind: 'tool', + title: `工具调用 → \`create_file\`（${path}）`, + body: JSON.stringify(input, null, 2), + lang: 'json', + }, + ] + } return [ { kind: 'document', @@ -243,6 +296,124 @@ function toolUseBlocks( ] } +function buildContext( + messages: readonly ClaudeMessage[], + sandboxFiles: readonly ClaudeSandboxFile[], + sandboxUnavailable: boolean, +): Ctx { + const sandboxByPath = new Map() + const sandboxByBasename = new Map() + const sandboxOutputs: ClaudeSandboxFile[] = [] + for (const file of sandboxFiles) { + const path = normalizePath(file.path) + sandboxByPath.set(path, file) + if (path.startsWith('/mnt/user-data/outputs/')) sandboxOutputs.push(file) + } + // basename 回退优先 outputs：/home/claude/foo 与 outputs/foo 是常见的同一最终文件； + // 同名 upload 不应抢走 Claude 生成件。 + for (const file of [...sandboxFiles].sort((a, b) => outputRank(a.path) - outputRank(b.path))) { + const base = basename(file.path) + if (base && !sandboxByBasename.has(base)) sandboxByBasename.set(base, file) + } + + const provisional: Ctx = { + artifacts: replayArtifacts(messages), + sandboxByPath, + sandboxByBasename, + sandboxOutputs, + presentedPaths: new Set(), + usedPresentFallback: false, + sandboxUnavailable, + } + let sawUnresolvedPresentFiles = false + for (const msg of messages) { + for (const block of msg.content ?? []) { + if (block.type !== 'tool_use' || block.name !== 'present_files') continue + const matched = sandboxFilesForInput(block.input ?? {}, provisional) + if (matched.length === 0) sawUnresolvedPresentFiles = true + for (const file of matched) { + provisional.presentedPaths.add(file.path) + } + } + } + if (sawUnresolvedPresentFiles) { + for (const file of sandboxOutputs) provisional.presentedPaths.add(file.path) + } + return provisional +} + +function sandboxFilesForInput(input: unknown, ctx: Ctx): ClaudeSandboxFile[] { + const paths = deepPathStrings(input) + const out: ClaudeSandboxFile[] = [] + const seen = new Set() + for (const path of paths) { + const file = sandboxFileForPath(path, ctx) + if (file && !seen.has(file.path)) { + seen.add(file.path) + out.push(file) + } + } + return out +} + +function sandboxFileForPath(path: string, ctx: Ctx): ClaudeSandboxFile | undefined { + const normalized = normalizePath(path) + return ctx.sandboxByPath.get(normalized) ?? ctx.sandboxByBasename.get(basename(normalized)) +} + +function deepPathStrings(value: unknown, out: string[] = []): string[] { + if (typeof value === 'string') { + if (/\/(?:mnt\/user-data\/(?:outputs|uploads)|home\/claude)\//.test(normalizePath(value))) { + out.push(value) + } + return out + } + if (Array.isArray(value)) { + for (const item of value) deepPathStrings(item, out) + } else if (value && typeof value === 'object') { + for (const item of Object.values(value as Record)) deepPathStrings(item, out) + } + return out +} + +function sandboxAssetRef(file: ClaudeSandboxFile): AssetRef { + const name = basename(file.path) || 'file' + const mime = str(file.content_type) + return { + fileId: `wiggle-${hashPath(file.path)}`, + kind: isImageFile(name, mime) ? 'image' : 'file', + name, + url: file.download_url, + sizeBytes: typeof file.size === 'number' ? file.size : undefined, + mime: mime || undefined, + } +} + +function isImageFile(name: string, mime: string): boolean { + return mime.startsWith('image/') || /\.(?:avif|gif|jpe?g|png|svg|webp)$/i.test(name) +} + +function normalizePath(path: string): string { + return path.replace(/\\/g, '/').replace(/\/{2,}/g, '/') +} + +function basename(path: string): string { + return normalizePath(path).split('/').filter(Boolean).pop() ?? '' +} + +function outputRank(path: string): number { + return normalizePath(path).startsWith('/mnt/user-data/outputs/') ? 0 : 1 +} + +function hashPath(path: string): string { + let hash = 0x811c9dc5 + for (const ch of normalizePath(path)) { + hash ^= ch.charCodeAt(0) + hash = Math.imul(hash, 0x01000193) + } + return (hash >>> 0).toString(16).padStart(8, '0') +} + function toolResultBlock(b: ClaudeContentBlock): IRBlock { const isText = typeof b.content === 'string' return { @@ -255,7 +426,8 @@ function toolResultBlock(b: ClaudeContentBlock): IRBlock { } /** - * 附件两处来源，按各自实际提供的东西处理： + * 消息内普通附件的两处来源，按各自实际提供的东西处理。Claude 生成件另由 + * present_files + 会话级 Wiggle 沙箱清单在 toolUseBlocks 中生成链接： * files[] —— 上传的原件。图片有 preview_url（内联嵌入），文档有 * document_asset.url（列为链接），blob 类没有可用地址（留说明） * attachments[] —— 文本抽取件（.md/.docx/…）。没有地址，但正文就在 diff --git a/src/sites/claude/index.ts b/src/sites/claude/index.ts index 05202ad..c0ef8ac 100644 --- a/src/sites/claude/index.ts +++ b/src/sites/claude/index.ts @@ -5,11 +5,12 @@ import { currentConversationId, fetchBinary, fetchConversation, + listSandboxFiles, resolveOrgId, throttleStats, } from './api' import { conversationToIR } from './convert' -import type { ClaudeConversation } from './types' +import type { ClaudeConversation, ClaudeIRContext } from './types' export const claudeAdapter: SiteAdapter = { id: 'claude', @@ -26,7 +27,25 @@ export const claudeAdapter: SiteAdapter = { fetchRaw: (session, id, cancel) => fetchConversation(session, id, cancel), - toIR: (raw, fallbackId) => conversationToIR(raw as ClaudeConversation, fallbackId), + fetchIRContext: async (session, id, raw, cancel): Promise => { + // 普通对话没有 present_files，没必要额外打一遍沙箱接口。 + if (!hasPresentFiles(raw as ClaudeConversation)) return { sandboxFiles: [] } + try { + return { sandboxFiles: await listSandboxFiles(session, id, cancel) } + } catch (error) { + if (cancel?.cancelled) throw error + // 附件发现失败不应吞掉整篇正文；转换层会在文件卡片原位留下说明。 + return { sandboxFiles: [], sandboxUnavailable: true } + } + }, + + toIR: (raw, fallbackId, context) => + conversationToIR( + raw as ClaudeConversation, + fallbackId, + (context as ClaudeIRContext | undefined)?.sandboxFiles ?? [], + (context as ClaudeIRContext | undefined)?.sandboxUnavailable === true, + ), async fetchAsset( _session: string, @@ -34,7 +53,7 @@ export const claudeAdapter: SiteAdapter = { cancel?: CancelToken, maxBytes?: number, ): Promise { - // Claude 的附件地址就在消息里，同源、登录态直接可取，不需要先换签名 URL + // Claude 的普通附件与 Wiggle 下载地址都是同源、登录态直接可取，不需要换签名 URL if (!ref.url) throw new Error(`附件 ${ref.name ?? ref.fileId} 没有可下载地址`) const { bytes, contentType } = await fetchBinary(ref.url, cancel, maxBytes) return { bytes, filename: null, contentType } @@ -43,15 +62,23 @@ export const claudeAdapter: SiteAdapter = { throttleStats, ui: { - // [待测] 以下选择器需要在真实页面上确认。找不到锚点时 FAB 不显示（既有防御）， - // 所以候选写宽是安全的：宁可多试几个，也不要挂在会被本地化的 aria-label 文案上。 + // 2026-08-28 真实会话页实测：Files + Share 外层是 actions-group。锚定整个组的 + // 左边界才不会盖住 Files；旧选择器继续留作回退，兼容 Claude 的灰度发布。 headerAnchor: () => - (document.querySelector('[data-testid="share-button"]') ?? + (document.querySelector('[data-testid="wiggle-controls-actions-group"]') ?? + document.querySelector('[data-testid="wiggle-controls-actions"]') ?? + document.querySelector('[data-testid="wiggle-controls-actions-share"]') ?? + document.querySelector('[data-testid="share-button"]') ?? document.querySelector('[data-testid="chat-menu-trigger"]') ?? - document.querySelector('header button[aria-haspopup="menu"]')) as HTMLElement | null, + document.querySelector('header button[aria-haspopup="menu"]') ?? + document.querySelector('[data-testid="chat-title-split"]')) as HTMLElement | null, composerAnchor: () => - (document.querySelector('fieldset div[contenteditable="true"]')?.closest('fieldset') ?? + ((() => { + const editor = document.querySelector('[data-testid="chat-input"][contenteditable="true"]') + return editor?.closest('.rounded-composer') ?? editor?.closest('fieldset') + })() ?? + document.querySelector('fieldset div[contenteditable="true"]')?.closest('fieldset') ?? document.querySelector('div[contenteditable="true"][role="textbox"]')?.closest('fieldset') ?? document.querySelector('div.ProseMirror[contenteditable="true"]')?.parentElement ?? null) as HTMLElement | null, @@ -81,3 +108,9 @@ export const claudeAdapter: SiteAdapter = { }, }, } + +function hasPresentFiles(conv: ClaudeConversation): boolean { + return (conv.chat_messages ?? []).some((msg) => + (msg.content ?? []).some((block) => block.type === 'tool_use' && block.name === 'present_files'), + ) +} diff --git a/src/sites/claude/types.ts b/src/sites/claude/types.ts index ea4ef63..8f07c36 100644 --- a/src/sites/claude/types.ts +++ b/src/sites/claude/types.ts @@ -37,7 +37,7 @@ export interface ClaudeContentBlock { thinking?: string | null /** thinking 块的分段摘要 [待测] */ summaries?: Array<{ summary?: string | null; [k: string]: unknown }> | null - /** tool_use 的工具名：artifacts / create_file / visualize:show_widget / web_search … */ + /** tool_use 的工具名：artifacts / create_file / present_files / visualize:show_widget / web_search … */ name?: string | null input?: Record | null /** tool_result 的载荷，形态随工具而异 */ @@ -104,3 +104,21 @@ export interface ClaudeConversation { project?: { uuid?: string | null; name?: string | null } | null [k: string]: unknown } + +/** Claude 会话级 Wiggle 沙箱里的文件。 */ +export interface ClaudeSandboxFile { + path: string + size?: number | null + content_type?: string | null + created_at?: string | null + custom_metadata?: Record | null + /** 由 API 层根据 org / conversation / path 生成的同源下载地址。 */ + download_url: string + [k: string]: unknown +} + +export interface ClaudeIRContext { + sandboxFiles: ClaudeSandboxFile[] + /** 清单请求失败时降级导出正文，并在 present_files 位置留明确说明。 */ + sandboxUnavailable?: boolean +} diff --git a/src/sites/types.ts b/src/sites/types.ts index b0d60bf..49fcd0f 100644 --- a/src/sites/types.ts +++ b/src/sites/types.ts @@ -72,7 +72,14 @@ export interface SiteAdapter { /** 原始 JSON（raw 导出与 IR 转换共用同一次抓取） */ fetchRaw(session: string, id: string, cancel?: CancelToken): Promise /** 原始 JSON → IR */ - toIR(raw: unknown, fallbackId: string): IRConversation + toIR(raw: unknown, fallbackId: string, context?: unknown): IRConversation + /** Markdown 转换前需要额外拉取的站点数据（例如 Claude 会话沙箱文件清单）。 */ + fetchIRContext?( + session: string, + id: string, + raw: unknown, + cancel?: CancelToken, + ): Promise /** 取附件字节 */ fetchAsset( session: string, diff --git a/src/ui-position.ts b/src/ui-position.ts new file mode 100644 index 0000000..ad93602 --- /dev/null +++ b/src/ui-position.ts @@ -0,0 +1,77 @@ +export interface AnchorRect { + top: number + right: number + bottom: number + left: number + height: number +} + +export interface FabPlacement { + right: number + bottom: number + panelTop?: number + panelLeft?: number +} + +/** + * 把页面锚点换算成 fixed FAB 的 right / bottom。 + * + * Claude 的 sticky composer 实测会因缩放和子像素布局略微越过 viewport 底边； + * 这里按可见区域裁剪，而不是把整个定位判作失败。 + */ +export function computeFabPlacement( + mode: 'composer' | 'header', + rect: AnchorRect, + viewport: { width: number; height: number }, + size: number, + gap: number, +): FabPlacement | null { + if ( + !Number.isFinite(rect.top) || + !Number.isFinite(rect.right) || + !Number.isFinite(rect.bottom) || + !Number.isFinite(rect.left) || + !Number.isFinite(rect.height) || + rect.height <= 0 || + viewport.width <= 0 || + viewport.height <= 0 + ) { + return null + } + + const visibleTop = Math.max(0, rect.top) + const visibleBottom = Math.min(viewport.height, rect.bottom) + if (visibleBottom <= visibleTop) return null + const visibleHeight = visibleBottom - visibleTop + + if (mode === 'header') { + if (rect.top < 0) return null + return { + right: Math.round(viewport.width - rect.left + gap), + bottom: Math.round(viewport.height - rect.bottom + (rect.height - size) / 2), + panelTop: Math.round(rect.bottom + 10), + } + } + + const beside = Math.round(viewport.width - rect.right - gap - size) + if (beside >= 8) { + const right = beside + return { + right, + bottom: Math.round(Math.max(8, viewport.height - visibleBottom + (visibleHeight - size) / 2)), + panelLeft: composerPanelLeft(viewport.width, right, size), + } + } + const right = 20 + return { + right, + bottom: Math.round(Math.min(viewport.height - 60, viewport.height - visibleTop + gap)), + panelLeft: composerPanelLeft(viewport.width, right, size), + } +} + +/** 面板从按钮向右展开；仅在不足 192px 可用宽度时才向左平移。 */ +function composerPanelLeft(viewportWidth: number, fabRight: number, fabSize: number): number { + const fabLeft = viewportWidth - fabRight - fabSize + return Math.round(Math.max(16, Math.min(fabLeft, viewportWidth - 192 - 16))) +} diff --git a/src/ui.ts b/src/ui.ts index ab884e7..5d5252d 100644 --- a/src/ui.ts +++ b/src/ui.ts @@ -1,5 +1,6 @@ import { sanitizeSubdir } from './core/render' import type { SiteUi } from './sites' +import { computeFabPlacement } from './ui-position' export type ExportFormat = 'markdown' | 'json' export type ExportScope = 'current' | 'all' | 'selection' @@ -170,7 +171,7 @@ const STYLE = ` 底色照抄页面 translucent-surface（透明 + blur(24px) 液态玻璃，无阴影）， 悬浮才出圆角矩形底色；无高光扫过 */ :host([data-pos="header"]) .fab { - width: 36px; height: 36px; border-radius: 8px; + width: var(--header-fab-size, 36px); height: var(--header-fab-size, 36px); border-radius: 8px; background: transparent; border-color: transparent; box-shadow: none; -webkit-backdrop-filter: blur(24px); backdrop-filter: blur(24px); } @@ -196,10 +197,17 @@ const STYLE = ` max-height: min(72vh, calc(100vh - var(--fab-bottom, 88px) - 72px)); max-width: calc(100vw - 32px); transform-origin: 100% 100%; - transition: right .25s var(--ease), bottom .25s var(--ease); + transition: left .25s var(--ease), right .25s var(--ease), bottom .25s var(--ease); } .panel.open { display: block; animation: rise .22s var(--ease); } @keyframes rise { from { opacity: 0; transform: translateY(10px) scale(.97); } } + /* composer 模式优先从按钮向右上展开；右侧空间不足时贴视口右缘，少盖住输入区。 */ + :host([data-pos="composer"]) .panel { + left: var(--panel-left, 16px); right: auto; + width: min(304px, calc(100vw - var(--panel-left, 16px) - 16px)); + transform-origin: 50% 100%; + } + :host([data-pos="composer"]) .adv .row { flex-wrap: wrap; } /* header 模式：面板从按钮下方展开 */ :host([data-pos="header"]) .panel { bottom: auto; top: var(--panel-top, 56px); @@ -553,6 +561,28 @@ export function mountPanel(cb: PanelCallbacks): void { ` root.append(fab, panel) + // Claude 会在 document 上监听交互，并把 shadow DOM 事件的宿主误判成页面本身。 + // 在面板内部冒泡阶段截断可保留控件默认行为与目标监听器，同时避免点击、数字输入、 + // 粘贴或 focusin 被 Claude 接走并落进聊天编辑器。 + for (const type of [ + 'pointerdown', + 'mousedown', + 'mouseup', + 'click', + 'dblclick', + 'focusin', + 'focusout', + 'keydown', + 'keypress', + 'keyup', + 'beforeinput', + 'input', + 'change', + 'paste', + ] as const) { + panel.addEventListener(type, (event) => event.stopPropagation()) + } + // FAB 锚定系统，双模式： // composer（默认）：贴输入框右侧垂直居中，挤不下退到输入框正上方（玻璃圆钮）； // header：贴顶栏 Share 按钮左侧（没有 Share 时贴 header 动作区），面板向下展开（幽灵钮）。 @@ -562,61 +592,59 @@ export function mountPanel(cb: PanelCallbacks): void { // getBoundingClientRect，样式仅在数值变化时写入。 let mode: 'composer' | 'header' = cb.settings.values.fabPos host.dataset['pos'] = mode - const fabSize = () => (mode === 'header' ? 36 : 44) + // Claude 顶栏原生动作实测为 28px；ChatGPT 保持原有 36px。 + const headerFabSize = cb.site.id === 'claude' ? 28 : 36 + host.style.setProperty('--header-fab-size', `${headerFabSize}px`) + const fabSize = () => (mode === 'header' ? headerFabSize : 44) const fabGap = () => (mode === 'header' ? 8 : 12) let curRight = -1 let curBottom = -1 let curPanelTop = -1 + let curPanelLeft = -1 const findAnchor = (): HTMLElement | null => mode === 'header' ? cb.siteUi.headerAnchor() : cb.siteUi.composerAnchor() let anchor: HTMLElement | null = null - const syncPos = (): void => { - if (!anchor?.isConnected) return // 没有锚点：位置保持原样，藏与不藏由 rebindAnchor 决定 + const syncPos = (): boolean => { + if (!anchor?.isConnected) return false // 没有锚点：位置保持原样，藏与不藏由 rebindAnchor 决定 const r = anchor.getBoundingClientRect() - if (r.height <= 0) return - const size = fabSize() - let right: number - let bottom: number - if (mode === 'header') { - if (r.top < 0) return - right = Math.round(window.innerWidth - r.left + fabGap()) - bottom = Math.round(window.innerHeight - r.bottom + (r.height - size) / 2) - const panelTop = Math.round(r.bottom + 10) - if (panelTop !== curPanelTop) { - curPanelTop = panelTop - host.style.setProperty('--panel-top', `${panelTop}px`) - } - } else { - if (r.bottom > window.innerHeight) return - const beside = Math.round(window.innerWidth - r.right - fabGap() - size) - if (beside >= 8) { - right = beside - bottom = Math.round(Math.max(8, window.innerHeight - r.bottom + (r.height - size) / 2)) - } else { - right = 20 - bottom = Math.round(Math.min(window.innerHeight - 60, window.innerHeight - r.top + fabGap())) - } + const placement = computeFabPlacement( + mode, + r, + { width: window.innerWidth, height: window.innerHeight }, + fabSize(), + fabGap(), + ) + if (!placement) return false + if (placement.panelTop != null && placement.panelTop !== curPanelTop) { + curPanelTop = placement.panelTop + host.style.setProperty('--panel-top', `${placement.panelTop}px`) + } + if (placement.panelLeft != null && placement.panelLeft !== curPanelLeft) { + curPanelLeft = placement.panelLeft + host.style.setProperty('--panel-left', `${placement.panelLeft}px`) } - if (right !== curRight) { - curRight = right - host.style.setProperty('--fab-right', `${right}px`) + if (placement.right !== curRight) { + curRight = placement.right + host.style.setProperty('--fab-right', `${placement.right}px`) } - if (bottom !== curBottom) { - curBottom = bottom - host.style.setProperty('--fab-bottom', `${bottom}px`) + if (placement.bottom !== curBottom) { + curBottom = placement.bottom + host.style.setProperty('--fab-bottom', `${placement.bottom}px`) } + return true + } + const closePanel = (): void => { + panel.classList.remove('open') + fab.classList.remove('open') + fab.setAttribute('aria-expanded', 'false') } const hideFab = (): void => { fab.classList.remove('in') - if (panel.classList.contains('open')) { - panel.classList.remove('open') - fab.classList.remove('open') - fab.setAttribute('aria-expanded', 'false') - } + closePanel() } const ro = typeof ResizeObserver === 'undefined' ? null : new ResizeObserver(syncPos) let anchorMissing = 0 - const rebindAnchor = (): void => { + const rebindAnchor = (): boolean => { const c = findAnchor() if (c !== anchor) { ro?.disconnect() @@ -625,10 +653,11 @@ export function mountPanel(cb: PanelCallbacks): void { } if (anchor?.isConnected) { anchorMissing = 0 - syncPos() + return syncPos() } else if (++anchorMissing >= 2) { hideFab() // 连续两轮（约 4s）没有锚点：整个入口隐藏 } + return false } // 找到输入框、且位置连续两拍（250ms）稳定后才现身（.in)——SPA 水合期间 composer // 可能先出现在错误位置（居中/侧栏未挂载），立刻现身会被用户看到「先落错位再闪跳」。 @@ -638,9 +667,9 @@ export function mountPanel(cb: PanelCallbacks): void { let bootKey = '' let bootStable = 0 const boot = (): void => { - rebindAnchor() + const positioned = rebindAnchor() const key = `${curRight},${curBottom}` - bootStable = anchor && key === bootKey ? bootStable + 1 : anchor ? 1 : 0 + bootStable = positioned && key === bootKey ? bootStable + 1 : positioned ? 1 : 0 bootKey = key if (bootStable >= 2) { bootDone = true @@ -658,8 +687,8 @@ export function mountPanel(cb: PanelCallbacks): void { let tick = 0 setInterval(() => { if (++tick % 4 === 0) { - rebindAnchor() - if (bootDone && anchor?.isConnected && curRight >= 0 && !fab.classList.contains('in')) { + const positioned = rebindAnchor() + if (bootDone && positioned && !fab.classList.contains('in')) { detectAccent(true) fab.classList.add('in') } @@ -710,9 +739,23 @@ export function mountPanel(cb: PanelCallbacks): void { fabPosEl.value = mode fabPosEl.addEventListener('change', () => { mode = fabPosEl.value === 'header' ? 'header' : 'composer' + // select 位于展开面板内：换位时先正常收起，否则面板可能移出视口、按钮却残留 + // open/蓝底/向下箭头状态。位置缓存也必须清空，确保双向切换都立即写入新坐标。 + closePanel() + fab.classList.remove('in') + ro?.disconnect() + anchor = null + curRight = -1 + curBottom = -1 + curPanelTop = -1 + curPanelLeft = -1 + host.style.removeProperty('--fab-right') + host.style.removeProperty('--fab-bottom') + host.style.removeProperty('--panel-top') + host.style.removeProperty('--panel-left') host.dataset['pos'] = mode cb.settings.onSettingsChange({ fabPos: mode }) - rebindAnchor() + if (rebindAnchor()) fab.classList.add('in') }) const linkStyleEl = panel.querySelector('select[data-opt="linkStyle"]')! diff --git a/test/claude.test.ts b/test/claude.test.ts index ac29d14..dc31857 100644 --- a/test/claude.test.ts +++ b/test/claude.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test' import { renderConversation } from '../src/core/render' import { replayArtifacts } from '../src/sites/claude/artifacts' import { conversationToIR, linearize } from '../src/sites/claude/convert' -import type { ClaudeConversation, ClaudeMessage } from '../src/sites/claude/types' +import type { ClaudeConversation, ClaudeMessage, ClaudeSandboxFile } from '../src/sites/claude/types' import fixtureJson from './fixtures/claude-basic.json' const fixture = fixtureJson as unknown as ClaudeConversation @@ -238,6 +238,89 @@ describe('附件', () => { const out = md(conv) expect(out).toContain('*(附件：录音.m4a · blob — 无可下载地址)*') }) + + test('present_files 输出沙箱文件链接，并隐藏重复的 create_file 中间版本', () => { + const conv = withBlocks([ + { + type: 'tool_use', + name: 'create_file', + input: { path: '/home/claude/certificate_scatter.py', file_text: 'FIRST VERSION' }, + }, + { + type: 'tool_use', + name: 'create_file', + input: { path: '/home/claude/certificate_scatter.py', file_text: 'FINAL VERSION' }, + }, + { + type: 'tool_use', + name: 'present_files', + input: { + files: [ + { file_path: '/mnt/user-data/outputs/certificate_scatter.py' }, + { file_path: '/mnt/user-data/outputs/demo.png' }, + ], + }, + }, + ]) + const sandbox: ClaudeSandboxFile[] = [ + { + path: '/mnt/user-data/outputs/certificate_scatter.py', + size: 1200, + content_type: 'text/plain', + download_url: '/api/wiggle/download?path=certificate_scatter.py', + }, + { + path: '/mnt/user-data/outputs/demo.png', + size: 2400, + content_type: 'image/png', + download_url: '/api/wiggle/download?path=demo.png', + }, + ] + + const rendered = renderConversation(conversationToIR(conv, '', sandbox)) + expect(rendered.markdown).not.toContain('FIRST VERSION') + expect(rendered.markdown).not.toContain('FINAL VERSION') + expect(rendered.assets.map((a) => a.name)).toEqual(['certificate_scatter.py', 'demo.png']) + expect(rendered.assets.find((a) => a.name === 'demo.png')?.kind).toBe('image') + expect(rendered.assets.find((a) => a.name === 'certificate_scatter.py')?.url).toContain('/api/wiggle/download') + }) + + test('present_files 参数无法识别时只回退一次 outputs 全集', () => { + const conv = withBlocks([ + { type: 'tool_use', name: 'present_files', input: { future_shape: true } }, + { type: 'tool_use', name: 'present_files', input: { future_shape: true } }, + ]) + const sandbox: ClaudeSandboxFile[] = [ + { + path: '/mnt/user-data/outputs/result.csv', + size: 10, + content_type: 'text/plain', + download_url: '/api/wiggle/download?path=result.csv', + }, + { + path: '/mnt/user-data/uploads/input.csv', + size: 10, + content_type: 'text/csv', + download_url: '/api/wiggle/download?path=input.csv', + }, + ] + const rendered = renderConversation(conversationToIR(conv, '', sandbox)) + expect(rendered.assets.map((a) => a.name)).toEqual(['result.csv']) + }) + + test('沙箱清单请求失败时正文仍可导出，并在文件卡片原位说明', () => { + const conv = withBlocks([ + { type: 'text', text: '正文保留' }, + { + type: 'tool_use', + name: 'present_files', + input: { filepaths: ['/mnt/user-data/outputs/result.csv'] }, + }, + ]) + const out = renderConversation(conversationToIR(conv, '', [], true)).markdown + expect(out).toContain('正文保留') + expect(out).toContain('Claude 生成文件清单获取失败，本次未能下载这些文件') + }) }) describe('引用与 frontmatter', () => { diff --git a/test/output-naming.test.ts b/test/output-naming.test.ts new file mode 100644 index 0000000..e2e3a58 --- /dev/null +++ b/test/output-naming.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from 'bun:test' +import type { AssetRef } from '../src/core/ir' +import { assetFileName, assetReferencePath } from '../src/output/naming' + +const image: AssetRef = { + fileId: 'image-1', + kind: 'image', + name: 'capture.png', +} + +describe('assetFileName', () => { + test('实际响应为 WebP 时纠正来源 PNG 扩展名', () => { + expect(assetFileName(image, null, 'image/webp')).toBe('capture.webp') + }) + + test('Content-Type 参数不影响 MIME 识别', () => { + expect(assetFileName(image, null, 'image/jpeg; charset=binary')).toBe('capture.jpg') + }) + + test('未知 MIME 保留来源扩展名', () => { + expect(assetFileName(image, null, 'application/octet-stream')).toBe('capture.png') + }) + + test('没有来源扩展时按已知 MIME 补齐', () => { + expect(assetFileName({ ...image, name: undefined }, null, 'image/png')).toBe('image.png') + }) +}) + +describe('assetReferencePath', () => { + test('Wikilink 使用 vault 根路径，包含笔记子目录', () => { + expect(assetReferencePath('wikilink', 'conversations/', 'attachments/a.png')).toBe( + 'conversations/attachments/a.png', + ) + }) + + test('标准 Markdown 链接仍相对当前笔记', () => { + expect(assetReferencePath('markdown', 'conversations/', 'attachments/a.png')).toBe( + 'attachments/a.png', + ) + }) +}) diff --git a/test/ui-position.test.ts b/test/ui-position.test.ts new file mode 100644 index 0000000..5d4e97a --- /dev/null +++ b/test/ui-position.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from 'bun:test' +import { computeFabPlacement } from '../src/ui-position' + +describe('computeFabPlacement', () => { + test('Claude composer 贴在输入框表面右侧并垂直居中，面板尽量向右展开', () => { + expect( + computeFabPlacement( + 'composer', + { top: 893, right: 1274, bottom: 945, left: 506, height: 52 }, + { width: 1494, height: 983 }, + 44, + 12, + ), + ).toEqual({ right: 164, bottom: 42, panelLeft: 1286 }) + }) + + test('composer 子像素越过 viewport 底边时按可见部分定位', () => { + expect( + computeFabPlacement( + 'composer', + { top: 893.2, right: 1274, bottom: 983.4, left: 506, height: 90.2 }, + { width: 1494, height: 983 }, + 44, + 12, + ), + ).toEqual({ right: 164, bottom: 23, panelLeft: 1286 }) + }) + + test('Claude header 贴在 Files + Share 动作组左侧', () => { + expect( + computeFabPlacement( + 'header', + { top: 10, right: 1482, bottom: 38, left: 1392, height: 28 }, + { width: 1494, height: 983 }, + 28, + 8, + ), + ).toEqual({ right: 110, bottom: 945, panelTop: 48 }) + }) +}) From 73158627a4e5626afeaa5fa9b4d046505721e590 Mon Sep 17 00:00:00 2001 From: pmwl Date: Sat, 29 Aug 2026 15:24:20 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat(claude):=20=E5=BC=80=E6=94=BE=E5=8F=97?= =?UTF-8?q?=E4=BF=9D=E6=8A=A4=E7=9A=84=E6=89=B9=E9=87=8F=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PLAN.md | 21 +-- README.md | 22 +-- README.zh-CN.md | 21 +-- docs/claude-adapter-feasibility.md | 7 +- docs/claude-home-layout-probe.js | 263 +++++++++++++++++++++++++++++ src/api.ts | 249 ++++++++++++++++++++------- src/convert/markdown.ts | 10 +- src/core/batch-safety.ts | 66 ++++++++ src/core/fetcher.ts | 26 ++- src/main.ts | 100 +++++++++-- src/sites/chatgpt/convert.ts | 11 +- src/sites/chatgpt/index.ts | 41 ++++- src/sites/claude/api.ts | 50 +++++- src/sites/claude/index.ts | 62 +++++-- src/sites/types.ts | 12 +- src/ui.ts | 64 +++++-- test/batch-safety.test.ts | 73 ++++++++ test/claude-pager.test.ts | 64 ++++++- test/claude-ui.test.ts | 12 ++ test/fetcher.test.ts | 36 ++++ test/markdown.test.ts | 9 + test/ui-position.test.ts | 13 ++ 22 files changed, 1067 insertions(+), 165 deletions(-) create mode 100644 docs/claude-home-layout-probe.js create mode 100644 src/core/batch-safety.ts create mode 100644 test/batch-safety.test.ts create mode 100644 test/claude-ui.test.ts create mode 100644 test/fetcher.test.ts diff --git a/PLAN.md b/PLAN.md index cdb1abf..b99ec9f 100644 --- a/PLAN.md +++ b/PLAN.md @@ -7,7 +7,7 @@ ## 目标 - 在 chatgpt.com / claude.ai 页内一键导出对话为 Obsidian 等笔记软件友好的 Markdown - （ChatGPT 支持批量与增量；Claude 首版只做当前对话，理由见 P5） + （两站均支持当前、选择、批量与增量导出；Claude 使用更保守的独立风控） - 高保真：公式、引用链接、代码、图片/附件、思维链、Canvas 不丢不乱 - 增量同步：重跑只导出有变化的对话 - 全程本地处理，不经任何第三方服务 @@ -77,6 +77,7 @@ inkstone/ ir.ts # 中间表示：IRConversation / IRTurn / IRBlock render.ts # IR → Markdown（轮次标题、callout、围栏、frontmatter） fetcher.ts # 限速 / 退避 / 并发池 / 取消 / 限流观测（每站点一个实例） + batch-safety.ts # 站点级并发/重试策略 + 429/请求预算/失败率熔断 sites/ types.ts # SiteAdapter 契约（取数 + 转换 + 界面锚点 + 批量能力） index.ts # 按 location.host 分派 @@ -84,8 +85,8 @@ inkstone/ index.ts # adapter 实装 convert.ts # backend-api JSON → IR（content_type 分发、canmore 语义） claude/ - index.ts # adapter 实装（supportsBatch: false） - api.ts # 内部 API 客户端 + 保守限流参数 + 分页器（未接界面） + index.ts # adapter 实装 + Claude 专属批量风控策略 + api.ts # 内部 API 客户端 + 保守限流参数 + 有界分页器 types.ts # 从宽的字段类型，[待测] 处已标注 convert.ts # 内部 API JSON → IR（块级分发、主线回溯、附件两处来源） artifacts.ts # artifact create/update/rewrite 折叠成终稿 @@ -103,7 +104,7 @@ inkstone/ fsaccess.ts # File System Access 直写 vault（句柄存 IndexedDB） test/ fixtures/*.json # 对话 JSON（ChatGPT 真实脱敏 / Claude 合成） - *.test.ts # bun test（132 个） + *.test.ts # bun test 回归套件 ``` ## 阶段 @@ -130,13 +131,13 @@ inkstone/ - **Claude 侧的脏活更少**：Canvas 的正则 patch 重放（150 行）与私有区 Unicode 引用 还原（178 行）在 Claude 都不需要——artifact 的 update 是字面量 `old_str`→`new_str`， 引用是结构化数组。artifact 折叠约 40 行。 - - **⚠️ 首版刻意只做「导出当前对话」**：批量的地基（分页器、水位线、并发池、 - 保护性中止）全部就位且已单测，但 `supportsBatch: false` 关着。理由是限流画像 - 未知——ChatGPT 侧的参数是 344 + 432 对话实测调出来的，Claude 侧一条实测数据 - 都没有。调研过的三个开源 claude.ai 导出器**没有一个实现了 429 退避** - （最激进的是 3 并发 + 固定 200ms 间隔且不看 429），所以没有可借鉴的安全参数。 + - **批量分阶段开放**：首版因限流画像未知而只开放当前对话；2026-08-29 恢复选择、 + 全部与增量导出，但不照搬 ChatGPT 参数。Claude 固定单并发、不做失败项整批二次重试； + 一次带 `Retry-After` 的 429、累计 3 次 429、单批 1000 次 HTTP 尝试，或至少 5 条失败 + 且失败率超过 25%，都会保护性中止。成功条目才推进水位线，未尝试部分由下次增量补齐。 + 列表分页另设 250 请求 / 10000 条硬上限，并检测重复页与缺失 uuid，防接口漂移后空转。 - **Claude 限流起步参数**（保守，待实测调整）：间距 1500ms（ChatGPT 侧的两倍慢）、 - 上限 8000ms、每 40 请求歇 30s、最多重试 6 次。每个站点持有独立的 fetcher 实例， + 上限 8000ms、每 40 请求歇 30s、首次失败后最多重试 1 次。每个站点持有独立的 fetcher 实例， 一边的限流不拖累另一边。吃到 429 时导出完成文案会报出次数、被推大的间距与 服务端要求的最长等待——未知站点的节奏只能靠实测看清，先让它可见再谈调参。 - **待实测**：`docs/claude-probe.js` 可直接粘进 claude.ai 控制台，打印字段骨架 diff --git a/README.md b/README.md index a63df45..11264a7 100644 --- a/README.md +++ b/README.md @@ -36,18 +36,18 @@ Inkstone runs inside the page and fetches conversations through the same backend | | ChatGPT | Claude | | --- | --- | --- | | Export current conversation | ✅ | ✅ | -| Batch / export-all | ✅ | ⏳ not yet enabled | -| Incremental sync | ✅ | ⏳ not yet enabled | +| Batch / export-all | ✅ | ✅ conservative safeguards | +| Incremental sync | ✅ | ✅ | | Rich documents | Canvas patch replay | Artifact fold-up to final version | | Thoughts / tool traces | ✅ opt-in | ✅ opt-in | | Attachments | images and files downloaded | images, documents, and generated files downloaded; text extractions inlined | -**Why no batch export on Claude yet?** It isn't missing, it's switched off. The pager, -watermark, concurrency pool and protective abort are all in place and unit-tested — but -there is no measured rate-limit profile for Claude yet. The ChatGPT numbers only became -trustworthy after 344 + 432 real conversations. Until comparable evidence exists, the cost -of a wrong guess lands on your account, and that isn't a call a default-on switch should -make. See [`docs/claude-adapter-feasibility.md`](./docs/claude-adapter-feasibility.md). +**Claude batch export uses a separate, deliberately conservative policy.** Its rate-limit +profile is still not backed by a large real-world sample, so ChatGPT's settings are not +reused: one worker, spacing from 1500 ms, a 30-second rest every 40 requests, and no second +pass over failed items. One global `Retry-After` signal, three 429s, 1000 HTTP attempts, or +an abnormal failure ratio stops the batch. Successful conversations are still written; +unfinished ones do not advance the watermark and are picked up by the next incremental run. ## Screenshots @@ -117,10 +117,10 @@ bun run build # → dist/inkstone.user.js, drag it into Tampermonkey Open chatgpt.com or claude.ai (logged in) → click the **⤓ button** in the top bar → pick **Markdown zip** or **raw JSON zip** → unzip into your Obsidian vault. -- On Claude only **current conversation** is offered; the batch options are hidden, not disabled - The button position is switchable (panel → advanced settings): next to Share in the top bar, or a glass button beside the input box -- The UI follows the host page's appearance automatically (light/dark + accent color) +- The UI follows the host page's light/dark appearance; ChatGPT follows its selected accent, while Claude uses its brand orange - Exports are cancelable; a single failed conversation never aborts the run — failures are summarized in `_failures.json` +- Claude batches run with one worker; after a protective abort, wait as instructed instead of restarting immediately ## Offline CLI @@ -154,7 +154,7 @@ Adding a site means adding an adapter, not touching the orchestration. See `PLAN ## Roadmap -Batch export on Claude once its rate-limit profile has actually been measured, an MV3 browser extension (no Tampermonkey, store release), and Gemini support. Already done: the multi-site adapter architecture, Claude single-conversation export, incremental sync, direct-write to an Obsidian vault, settings panel, Canvas patch replay, Artifact fold-up, and the offline CLI. Details in [PLAN.md](./PLAN.md) (Chinese). +An MV3 browser extension (no Tampermonkey, store release) and Gemini support. Already done: the multi-site adapter architecture, Claude batch and incremental export, direct-write to an Obsidian vault, settings panel, Canvas patch replay, Artifact fold-up, and the offline CLI. Details in [PLAN.md](./PLAN.md) (Chinese). ## License diff --git a/README.zh-CN.md b/README.zh-CN.md index ff5bda5..1e36444 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -36,17 +36,17 @@ Inkstone 直接运行在页内，通过应用自己使用的 backend API 抓取 | | ChatGPT | Claude | | --- | --- | --- | | 导出当前对话 | ✅ | ✅ | -| 批量 / 全部导出 | ✅ | ⏳ 暂不开放 | -| 增量同步 | ✅ | ⏳ 暂不开放 | +| 批量 / 全部导出 | ✅ | ✅ 保守风控 | +| 增量同步 | ✅ | ✅ | | 富文档还原 | Canvas patch 重放 | Artifact 折叠还原终稿 | | 思维链 / 工具痕迹 | ✅ 可开关 | ✅ 可开关 | | 附件 | 图片与文件下载 | 图片下载、文档链接、文本抽取件内联 | -**Claude 端为什么先不做批量？** 不是没写，是没开。批量所需的分页器、水位线、并发池、 -保护性中止都已就位并通过单测，但 Claude 侧的限流画像还没有任何实测数据—— -ChatGPT 端那套参数是 344 + 432 条对话跑出来才敢用的。在拿到同等的实测证据之前， -批量抓取整个历史的风险由用户账号承担，这个代价不该由一个默认开启的开关来决定。 -细节与实测计划见 [`docs/claude-adapter-feasibility.md`](./docs/claude-adapter-feasibility.md)。 +**Claude 批量导出采用更保守的独立风控。** Claude 的限流画像仍没有大样本实测， +因此不照搬 ChatGPT 参数：固定单并发、请求间隔从 1500ms 起、每 40 次请求休息 30 秒， +不对整批失败项做第二轮重试。一次带 `Retry-After` 的全局限流信号、累计 3 次 429、 +1000 次 HTTP 尝试或异常失败率过高都会保护性中止；已经成功的对话照常落盘，未完成项 +不推进水位线，下次增量导出会继续补齐。 ## 截图 @@ -116,8 +116,9 @@ bun run build # 产物 dist/inkstone.user.js，拖进 Tampermonkey 即可 打开 chatgpt.com 或 claude.ai（已登录）→ 点**顶栏 Share 左侧的 ⤓ 按钮** → 选 **Markdown zip** 或**原始 JSON zip** → 解压到 Obsidian vault。 - 按钮位置可换（面板 → 高级设置）：顶栏 Share 旁，或输入框旁的玻璃圆钮 -- UI 主题色自动跟随 ChatGPT 的外观设置（明暗 + accent color） +- UI 明暗自动跟随所在站点；ChatGPT 跟随用户选择的重点色，Claude 使用品牌橙色 - 可随时取消；单条对话失败不中断整体导出，失败汇总进 `_failures.json` +- Claude 批量默认单并发慢速执行；触发保护性中止后不要立刻重跑，先按界面提示等待 ## 离线 CLI @@ -151,7 +152,7 @@ claude.ai 的 CSP 可能拦掉 dev server 的脚本，Claude 端的改动请用 ## 路线图 -Claude 端的批量导出（等限流画像实测清楚再开）、MV3 浏览器扩展（脱离 Tampermonkey、上架商店）、Gemini 适配。已完成：多站点适配器架构、Claude 单对话导出、增量同步、直写 vault、设置面板、Canvas patch 重放、Artifact 折叠还原、离线 CLI。详见 [PLAN.md](./PLAN.md)。 +MV3 浏览器扩展（脱离 Tampermonkey、上架商店）、Gemini 适配。已完成：多站点适配器架构、Claude 批量与增量导出、直写 vault、设置面板、Canvas patch 重放、Artifact 折叠还原、离线 CLI。详见 [PLAN.md](./PLAN.md)。 ## 许可证 @@ -159,4 +160,4 @@ Claude 端的批量导出（等限流画像实测清楚再开）、MV3 浏览器 ## 友情链接 -[LINUX DO](https://linux.do) \ No newline at end of file +[LINUX DO](https://linux.do) diff --git a/docs/claude-adapter-feasibility.md b/docs/claude-adapter-feasibility.md index 50fff44..f320bb4 100644 --- a/docs/claude-adapter-feasibility.md +++ b/docs/claude-adapter-feasibility.md @@ -1,8 +1,9 @@ # Inkstone → Claude 对话导出：可行性分析 -> **实施状态（2026-08-28）**：本文第四节的架构改造与第六节的 P1–P3 已完成， -> Claude 单对话导出可用；批量（P4）按第五节的风险判断**刻意未开放**。 -> 落地记录见 `PLAN.md` § P5，待实测清单见本文第五节，探针脚本见 `docs/claude-probe.js`。 +> **实施状态（2026-08-29）**：本文第四节的架构改造与第六节的 P1–P4 已完成。 +> Claude 的当前、选择、全部与增量导出均已接入；因限流画像仍缺少大样本实测，P4 +> 使用单并发、独立慢速 Fetcher、429/Retry-After 熔断、1000 请求预算与低失败率阈值， +> 不做整批二次重试。落地记录见 `PLAN.md` § P5，探针脚本见 `docs/claude-probe.js`。 > 本文其余部分保持评估当时的原貌，不随实施回填——它是决策依据的快照。 > 评估日期：2026-08-28 · 基准代码：`2121e9f`（v0.2.3，与上游 ZhenHuangLab/inkstone 同步） diff --git a/docs/claude-home-layout-probe.js b/docs/claude-home-layout-probe.js new file mode 100644 index 0000000..441b28e --- /dev/null +++ b/docs/claude-home-layout-probe.js @@ -0,0 +1,263 @@ +// Claude 首页布局探针（只读、零网络请求） +// +// 用法： +// 1. 登录 claude.ai，停留在「首页 / 新对话页」； +// 2. F12 → Console； +// 3. 粘贴本文件全部内容并回车； +// 4. 控制台会输出一段 JSON，并尝试复制到剪贴板；把 JSON 发给开发者。 +// +// 隐私边界： +// - 不读取 textContent、输入框 value、contenteditable 内容、cookie、localStorage； +// - 不调用任何接口； +// - URL 中的 UUID、邮箱、长 token 会脱敏； +// - aria-label/title/placeholder 可能帮助识别控件，但同样会经过脱敏。 + +;(() => { + const PROBE = 'inkstone-claude-home-layout-v1' + const MAX_NODES = 80 + + const redact = (value) => { + if (value == null) return null + return String(value) + .replace(/[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}/g, '') + .replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi, '') + .replace(/\b(?:sk-|sess-|org-|user-)?[A-Za-z0-9_-]{32,}\b/g, '') + .replace(/\b\d{7,}\b/g, '') + .slice(0, 160) + } + + const cleanUrl = () => { + const u = new URL(location.href) + u.search = '' + u.hash = '' + u.pathname = redact(u.pathname) + return u.toString() + } + + const round = (n) => Math.round(n * 10) / 10 + const rectOf = (el) => { + const r = el.getBoundingClientRect() + return { + x: round(r.x), + y: round(r.y), + width: round(r.width), + height: round(r.height), + right: round(r.right), + bottom: round(r.bottom), + } + } + + const isVisible = (el) => { + if (!(el instanceof HTMLElement)) return false + if (el.closest('[data-inkstone]')) return false + const r = el.getBoundingClientRect() + if (r.width < 1 || r.height < 1) return false + const s = getComputedStyle(el) + return s.display !== 'none' && s.visibility !== 'hidden' && Number(s.opacity) !== 0 + } + + const simpleSelector = (el) => { + const parts = [el.tagName.toLowerCase()] + const testId = el.getAttribute('data-testid') + const role = el.getAttribute('role') + const slot = el.getAttribute('data-slot') + if (el.id) parts.push(`#${redact(el.id)}`) + if (testId) parts.push(`[data-testid="${redact(testId)}"]`) + if (slot) parts.push(`[data-slot="${redact(slot)}"]`) + if (role) parts.push(`[role="${redact(role)}"]`) + const stableClasses = [...el.classList] + .filter((name) => name.length <= 48 && !/^css-|^_[A-Za-z0-9]{6,}/.test(name)) + .slice(0, 4) + if (!testId && stableClasses.length) parts.push(`.${stableClasses.join('.')}`) + return parts.join('') + } + + const domPath = (el) => { + const out = [] + let cur = el + while (cur instanceof HTMLElement && cur !== document.body && out.length < 7) { + out.unshift(simpleSelector(cur)) + cur = cur.parentElement + } + out.unshift('body') + return out.join(' > ') + } + + const safeAttributes = (el) => { + const names = [ + 'id', + 'role', + 'type', + 'data-testid', + 'data-slot', + 'data-state', + 'data-side', + 'aria-label', + 'aria-haspopup', + 'aria-expanded', + 'title', + 'placeholder', + ] + const out = {} + for (const name of names) { + const value = el.getAttribute(name) + if (value != null && value !== '') out[name] = redact(value) + } + return out + } + + const describe = (el) => { + const style = getComputedStyle(el) + return { + selector: simpleSelector(el), + path: domPath(el), + attributes: safeAttributes(el), + rect: rectOf(el), + style: { + position: style.position, + display: style.display, + zIndex: style.zIndex, + overflow: style.overflow, + }, + directChildren: [...el.children].slice(0, 12).map(simpleSelector), + } + } + + const ancestors = (el) => { + const out = [] + let cur = el + while (cur instanceof HTMLElement && cur !== document.body && out.length < 8) { + out.push(describe(cur)) + cur = cur.parentElement + } + return out + } + + const queryVisible = (selector) => + [...document.querySelectorAll(selector)].filter(isVisible).slice(0, MAX_NODES) + + const selectorChecks = [ + '[data-testid="wiggle-controls-actions-group"]', + '[data-testid="wiggle-controls-actions"]', + '[data-testid="wiggle-controls-actions-share"]', + '[data-testid="share-button"]', + '[data-testid="chat-menu-trigger"]', + '[data-testid="chat-title-split"]', + '#prompt-textarea', + '[data-testid="chat-input"][contenteditable="true"]', + 'div[contenteditable="true"][role="textbox"]', + 'div.ProseMirror[contenteditable="true"]', + ] + + const knownSelectors = Object.fromEntries( + selectorChecks.map((selector) => { + const nodes = queryVisible(selector) + return [selector, { count: nodes.length, matches: nodes.slice(0, 4).map(describe) }] + }), + ) + + const interactiveSelector = [ + 'button', + 'a[href]', + '[role="button"]', + '[role="menuitem"]', + '[data-testid]', + 'input', + 'textarea', + 'select', + '[contenteditable="true"]', + ].join(',') + + const visibleInteractive = queryVisible(interactiveSelector) + const topLimit = Math.max(180, innerHeight * 0.24) + const rightLimit = innerWidth - Math.min(560, innerWidth * 0.5) + const topRight = visibleInteractive + .filter((el) => { + const r = el.getBoundingClientRect() + return r.top <= topLimit && r.right >= rightLimit + }) + .sort((a, b) => { + const ar = a.getBoundingClientRect() + const br = b.getBoundingClientRect() + return ar.top - br.top || br.right - ar.right + }) + .slice(0, 40) + + const composer = visibleInteractive + .filter((el) => { + const r = el.getBoundingClientRect() + const inputLike = + el.matches('textarea,input,[contenteditable="true"],[role="textbox"]') || + el.querySelector('textarea,input,[contenteditable="true"],[role="textbox"]') + return Boolean(inputLike) && r.width >= 160 && r.height >= 24 + }) + .sort((a, b) => b.getBoundingClientRect().width - a.getBoundingClientRect().width) + .slice(0, 12) + + const landmarks = queryVisible('header,nav,main,aside,[role="banner"],[role="navigation"],[role="main"]') + .map(describe) + .slice(0, 30) + + const testIdInventory = {} + for (const el of queryVisible('[data-testid]')) { + const key = redact(el.getAttribute('data-testid')) + if (!key) continue + if (!testIdInventory[key]) testIdInventory[key] = [] + if (testIdInventory[key].length < 3) testIdInventory[key].push(rectOf(el)) + } + + const topRightAncestorChains = topRight.slice(0, 12).map((el) => ({ + target: describe(el), + ancestors: ancestors(el), + })) + + const report = { + probe: PROBE, + capturedAt: new Date().toISOString(), + page: { + url: cleanUrl(), + titleLength: document.title.length, + viewport: { width: innerWidth, height: innerHeight, devicePixelRatio }, + body: { + attributes: safeAttributes(document.body), + directChildren: [...document.body.children] + .filter((el) => !el.matches('[data-inkstone]')) + .slice(0, 30) + .map(describe), + }, + }, + knownSelectors, + landmarks, + topRightCandidates: topRight.map(describe), + topRightAncestorChains, + composerCandidates: composer.map((el) => ({ target: describe(el), ancestors: ancestors(el) })), + visibleDataTestIds: testIdInventory, + notes: [ + 'No network requests were made.', + 'No textContent, input value, cookie, localStorage, or conversation content was read.', + 'The Inkstone injected shadow host was excluded from the scan.', + ], + } + + const json = JSON.stringify(report, null, 2) + console.log(`%c[inkstone] ${PROBE}`, 'color:#1e6b72;font-weight:bold') + console.log('INKSTONE_CLAUDE_HOME_PROBE_BEGIN') + console.log(json) + console.log('INKSTONE_CLAUDE_HOME_PROBE_END') + + const copyResult = async () => { + try { + if (typeof copy === 'function') { + copy(json) + return '已通过 DevTools copy() 复制到剪贴板' + } + await navigator.clipboard.writeText(json) + return '已通过 Clipboard API 复制到剪贴板' + } catch (error) { + return `自动复制失败，请手动复制 BEGIN/END 之间的 JSON：${String(error)}` + } + } + + void copyResult().then((message) => console.log(`%c[inkstone] ${message}`, 'color:#1e6b72')) + return report +})() diff --git a/src/api.ts b/src/api.ts index e99525f..a0abc60 100644 --- a/src/api.ts +++ b/src/api.ts @@ -14,7 +14,15 @@ import { type Fetcher, type ThrottleConfig, } from './core/fetcher' -import type { ConversationDetail, ConversationListItem, ConversationListPage, SessionResponse } from './types' +import type { + ConversationDetail, + ConversationListItem, + ConversationListPage, + GizmoConversationsPage, + GizmoSidebarPage, + ProjectInfo, + SessionResponse, +} from './types' export { ApiError, @@ -63,97 +71,208 @@ export async function listConversationsPage( return (await res.json()) as ConversationListPage } -export async function listAllConversations( - token: string, - onProgress?: (fetched: number) => void, - cancel?: CancelToken, -): Promise { - const all: ConversationListItem[] = [] - let offset = 0 - let limit = 100 - let emptyRetries = 0 - // 注意：接口的 total 字段不可靠（实测翻页途中返回 offset+len+1）， - // 终止条件只认「空页」或「不足一页」。 +// ===== Projects（gizmo）===== +// 主列表接口只返回侧栏「Chats」那份平铺列表，project 里的会话必须按 project +// 逐个走 gizmos 接口拿。两个坐标系完全不同：主列表是 offset，gizmos 是字符串游标。 +const PROJECT_PAGE_LIMIT = 50 + +const projectNames = new Map() + +/** 已知的 project 名（需先 listProjects 拉过）；不在 project 侧栏里的 gizmo 返回 undefined。 */ +export const projectNameOf = (gizmoId: string | null | undefined): string | undefined => + gizmoId ? projectNames.get(gizmoId) : undefined + +// 面板来源下拉与归并分页器会几乎同时取列表；只合并正在进行的请求，避免长期缓存陈数据。 +let projectsInFlight: Promise | null = null + +export function listProjects(token: string, cancel?: CancelToken): Promise { + projectsInFlight ??= fetchProjects(token, cancel).finally(() => { + projectsInFlight = null + }) + return projectsInFlight +} + +async function fetchProjects(token: string, cancel?: CancelToken): Promise { + const out: ProjectInfo[] = [] + let cursor: number | null = null for (;;) { ensureAlive(cancel) - let page: ConversationListPage - try { - page = await listConversationsPage(token, offset, limit, cancel) - } catch (e) { - // limit 上限历史上收紧过；非限流的 4xx 先降到 50 重试一次 - if (e instanceof ApiError && e.status >= 400 && e.status < 500 && e.status !== 429 && limit > 50) { - limit = 50 - continue - } - throw e + const url = + `${location.origin}/backend-api/gizmos/snorlax/sidebar?conversations_per_gizmo=0` + + (cursor == null ? '' : `&cursor=${encodeURIComponent(String(cursor))}`) + const res = await fetcher.request(url, { headers: auth(token) }, cancel) + const page = (await res.json()) as GizmoSidebarPage + for (const entry of page.items ?? []) { + const g = entry.gizmo?.gizmo + if (!g?.id) continue + const name = (g.display?.name ?? '').trim() || '未命名项目' + projectNames.set(g.id, name) + out.push({ id: g.id, name }) } - const items = page.items ?? [] - all.push(...items) - onProgress?.(all.length) - // 服务端可能按自己的上限截页（返回数 < 请求 limit 不代表到底），只认空页； - // 而且列表索引实测会瞬时降级、提前返回空页/短列表（对话本身还在）， - // 所以空页也不轻信，隔几秒重试确认，连续空 3 次才算到底。 - if (items.length === 0) { - if (all.length === 0 || emptyRetries >= 2) break - emptyRetries++ - await sleep(4000 * emptyRetries) - continue - } - emptyRetries = 0 - offset += items.length + cursor = page.cursor ?? null + if (cursor == null) return out } - return all +} + +async function listProjectConversationsPage( + token: string, + gizmoId: string, + cursor: string, + cancel?: CancelToken, +): Promise { + const url = + `${location.origin}/backend-api/gizmos/${encodeURIComponent(gizmoId)}/conversations` + + `?cursor=${encodeURIComponent(cursor)}&limit=${PROJECT_PAGE_LIMIT}` + const res = await fetcher.request(url, { headers: auth(token) }, cancel) + return (await res.json()) as GizmoConversationsPage } export interface ConversationPager { - /** 拉下一页；done=true 表示已确认到底（此后再调直接返回空页 + done） */ + /** 拉下一页；done=true 表示所有来源都到底（此后再调直接返回空页 + done） */ next(): Promise<{ items: ConversationListItem[]; done: boolean }> } -/** - * 惰性分页器：把 listAllConversations 的翻页与防御逻辑逐页化，供「选择对话」 - * 的懒加载使用（切到「选择」不再一次性翻完全部页）。终止条件与全量版一致： - * 只认空页，且空页要隔几秒重试确认，连续空 3 次才算到底。 - */ -export function createConversationPager(token: string, cancel?: CancelToken): ConversationPager { +const SOURCE_ALL = 'all' +const SOURCE_MAIN = 'main' + +function timeOf(i: ConversationListItem): number { + const t = i.update_time ?? i.create_time + if (typeof t === 'number') return t + if (typeof t === 'string') { + const ms = Date.parse(t) + return Number.isNaN(ms) ? -Infinity : ms / 1000 + } + return -Infinity +} + +interface SourceStream { + done: boolean + peek(): ConversationListItem | undefined + take(): ConversationListItem + fill(): Promise +} + +function makeStream(nextPage: () => Promise): SourceStream { + const buf: ConversationListItem[] = [] + const stream: SourceStream = { + done: false, + peek: () => buf[0], + take: () => buf.shift()!, + async fill() { + const page = await nextPage() + if (page == null) stream.done = true + else buf.push(...page) + }, + } + return stream +} + +/** 主列表与各 project 的多源惰性分页器，按更新时间倒序归并并按 id 去重。 */ +export function createConversationPager( + token: string, + cancel?: CancelToken, + source: string = SOURCE_ALL, +): ConversationPager { + const onlyProject = source === SOURCE_ALL || source === SOURCE_MAIN ? null : source + const seen = new Set() let offset = 0 let limit = 100 let emptyRetries = 0 + let streams: SourceStream[] | null = null let done = false + + async function mainPage(): Promise { + for (;;) { + ensureAlive(cancel) + let page: ConversationListPage + try { + page = await listConversationsPage(token, offset, limit, cancel) + } catch (e) { + if (e instanceof ApiError && e.status >= 400 && e.status < 500 && e.status !== 429 && limit > 50) { + limit = 50 + continue + } + throw e + } + const items = page.items ?? [] + if (items.length === 0) { + if (offset === 0 || emptyRetries >= 2) return null + emptyRetries++ + await sleep(4000 * emptyRetries) + continue + } + emptyRetries = 0 + offset += items.length + return items + } + } + + function projectPages(gizmoId: string): () => Promise { + let cursor: string | null = '0' + return async () => { + if (cursor == null) return null + ensureAlive(cancel) + const page = await listProjectConversationsPage(token, gizmoId, cursor, cancel) + cursor = page.cursor ?? null + return (page.items ?? []).map((i) => ({ ...i, gizmo_id: i.gizmo_id ?? gizmoId })) + } + } + + async function buildStreams(): Promise { + if (onlyProject != null) return [makeStream(projectPages(onlyProject))] + if (source === SOURCE_MAIN) return [makeStream(mainPage)] + const projects = await listProjects(token, cancel) + return [makeStream(mainPage), ...projects.map((p) => makeStream(projectPages(p.id)))] + } + return { async next() { if (done) return { items: [], done: true } + streams ??= await buildStreams() for (;;) { - ensureAlive(cancel) - let page: ConversationListPage - try { - page = await listConversationsPage(token, offset, limit, cancel) - } catch (e) { - if (e instanceof ApiError && e.status >= 400 && e.status < 500 && e.status !== 429 && limit > 50) { - limit = 50 - continue - } - throw e + for (const s of streams) { + while (!s.done && s.peek() === undefined) await s.fill() } - const items = page.items ?? [] - if (items.length === 0) { - // 首页即空 = 账号真没对话；否则可能是列表索引瞬时降级，隔几秒重试确认 - if (offset === 0 || emptyRetries >= 2) { + const out: ConversationListItem[] = [] + for (;;) { + let best: SourceStream | undefined + for (const s of streams) { + const head = s.peek() + if (head === undefined) continue + if (best === undefined || timeOf(head) > timeOf(best.peek()!)) best = s + } + if (best === undefined) { done = true - return { items: [], done: true } + break } - emptyRetries++ - await sleep(4000 * emptyRetries) - continue + const item = best.take() + if (!seen.has(item.id)) { + seen.add(item.id) + out.push(item) + } + if (best.peek() === undefined && !best.done) break } - emptyRetries = 0 - offset += items.length - return { items, done: false } + if (out.length > 0 || done) return { items: out, done } } }, } } +export async function listAllConversations( + token: string, + onProgress?: (fetched: number) => void, + cancel?: CancelToken, +): Promise { + const pager = createConversationPager(token, cancel) + const all: ConversationListItem[] = [] + for (;;) { + const { items, done } = await pager.next() + all.push(...items) + if (items.length > 0) onProgress?.(all.length) + if (done) return all + } +} + export async function fetchConversation( token: string, id: string, diff --git a/src/convert/markdown.ts b/src/convert/markdown.ts index 4440309..4630a9f 100644 --- a/src/convert/markdown.ts +++ b/src/convert/markdown.ts @@ -7,7 +7,7 @@ import { conversationToIR } from '../sites/chatgpt/convert' import { renderConversation, - type ConvertOptions, + type ConvertOptions as CoreConvertOptions, type ConvertResult, type LinkStyle, } from '../core/render' @@ -19,16 +19,20 @@ export { filenameFor, sanitizeName, sanitizeSubdir, - type ConvertOptions, type ConvertResult, type LinkStyle, } from '../core/render' export type { AssetRef } from '../core/ir' +/** ChatGPT 兼容入口额外接受 project 名；通用渲染器仍保持站点无关。 */ +export interface ConvertOptions extends CoreConvertOptions { + projectName?: string +} + export function conversationToMarkdown( conv: ConversationDetail, fallbackId = '', copts: ConvertOptions = {}, ): ConvertResult { - return renderConversation(conversationToIR(conv, fallbackId), copts) + return renderConversation(conversationToIR(conv, fallbackId, copts.projectName), copts) } diff --git a/src/core/batch-safety.ts b/src/core/batch-safety.ts new file mode 100644 index 0000000..ac78dba --- /dev/null +++ b/src/core/batch-safety.ts @@ -0,0 +1,66 @@ +import type { FetchStats } from './fetcher' + +/** 站点级批量策略：显式写出并发、失败处理与限流熔断，避免未知站点沿用激进默认值。 */ +export interface BatchPolicy { + concurrency: number + retryFailed: boolean + retryDelayMs: number + failureAbortMin: number + failureAbortRatio: number + /** 本批次新增 429 达到此数即中止；缺省表示不额外熔断。 */ + max429Hits?: number + /** 带 Retry-After 的 429 是全局限流信号，达到此数即中止。 */ + maxRetryAfterHits?: number + /** 本批次所有 HTTP 尝试的总上限（含内部重试与附件请求）。 */ + maxRequests?: number +} + +export class BatchSafetyError extends Error { + constructor(message: string) { + super(message) + this.name = 'BatchSafetyError' + } +} + +/** + * Fetcher 统计是页面生命周期累计值；熔断只看当前批次相对创建时的增量， + * 不能让用户此前一次单对话请求留下的 429 永久锁死后续批量导出。 + */ +export function createBatchSafetyGuard( + policy: BatchPolicy, + getStats: () => FetchStats, +): () => void { + const baseline = getStats() + return () => { + const current = getStats() + const requests = Math.max(0, current.requests - baseline.requests) + if (policy.maxRequests != null && requests >= policy.maxRequests) { + throw new BatchSafetyError(`本批次已发出 ${requests} 次 HTTP 请求，达到安全上限并停止`) + } + + const retryAfterHits = Math.max(0, current.retryAfterHits - baseline.retryAfterHits) + if (policy.maxRetryAfterHits != null && retryAfterHits >= policy.maxRetryAfterHits) { + throw new BatchSafetyError( + `服务端已返回 Retry-After 全局限流信号，本批次安全中止（最长等待 ${current.maxRetryAfterSec}s）`, + ) + } + + const hits429 = Math.max(0, current.hits429 - baseline.hits429) + if (policy.max429Hits != null && hits429 >= policy.max429Hits) { + throw new BatchSafetyError(`本批次已遇到 ${hits429} 次 HTTP 429，为保护账号安全停止后续请求`) + } + } +} + +/** 小样本偶发失败不误杀；达到最小失败数后才看失败比例。 */ +export function failureLimitReached( + policy: BatchPolicy, + failed: number, + attempted: number, +): boolean { + return ( + failed >= policy.failureAbortMin && + attempted > 0 && + failed / attempted > policy.failureAbortRatio + ) +} diff --git a/src/core/fetcher.ts b/src/core/fetcher.ts index 62a9b09..4f0078c 100644 --- a/src/core/fetcher.ts +++ b/src/core/fetcher.ts @@ -42,6 +42,15 @@ export interface CancelToken { export const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) export const jitter = (base: number, spread = base): number => base + Math.random() * spread +/** Retry-After 允许秒数或 HTTP-date；两种都归一成剩余秒数。 */ +export function parseRetryAfterSeconds(raw: string | null, now = Date.now()): number { + if (raw == null || raw.trim() === '') return 0 + const seconds = Number(raw) + if (Number.isFinite(seconds) && seconds > 0) return seconds + const at = Date.parse(raw) + return Number.isFinite(at) && at > now ? Math.ceil((at - now) / 1000) : 0 +} + export function ensureAlive(cancel?: CancelToken): void { if (cancel?.cancelled) throw new CancelledError() } @@ -134,22 +143,27 @@ export function createFetcher(cfg: ThrottleConfig): Fetcher { return res } const retryable = res.status === 429 || res.status >= 500 - if (!retryable || attempt >= cfg.maxAttempts) { - throw new ApiError(res.status, `HTTP ${res.status}: ${url}`) - } + let retryAfterMs = 0 if (res.status === 429) { + // 即使本次已经没有重试额度，也要先记录；上层批量熔断依赖完整统计。 hits429++ global429Streak++ slowDown() - const retryAfterSec = Number(res.headers.get('retry-after')) - const retryAfterMs = retryAfterSec * 1000 + const retryAfterSec = parseRetryAfterSeconds(res.headers.get('retry-after')) + retryAfterMs = retryAfterSec * 1000 if (retryAfterMs > 0) { retryAfterHits++ maxRetryAfterSec = Math.max(maxRetryAfterSec, retryAfterSec) cooldownUntil = Math.max(cooldownUntil, Date.now() + retryAfterMs) } else if (global429Streak >= 5) { cooldownUntil = Math.max(cooldownUntil, Date.now() + 15_000) - } else { + } + } + if (!retryable || attempt >= cfg.maxAttempts) { + throw new ApiError(res.status, `HTTP ${res.status}: ${url}`) + } + if (res.status === 429) { + if (retryAfterMs <= 0 && global429Streak < 5) { headerless429s++ if (headerless429s > 1) throw new ApiError(429, `HTTP 429（条目级，快速放弃）: ${url}`) await sleep(jitter(delay)) diff --git a/src/main.ts b/src/main.ts index b096d68..b975dfb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,6 +6,11 @@ import { sleep, type CancelToken, } from './core/fetcher' +import { + BatchSafetyError, + createBatchSafetyGuard, + failureLimitReached, +} from './core/batch-safety' import type { AssetRef } from './core/ir' import { assetLink, @@ -64,13 +69,18 @@ if (detected) { function mount(): void { mountPanel({ - site: { id: site.id, label: site.label, supportsBatch: site.supportsBatch }, + site: { + id: site.id, + label: site.label, + supportsBatch: site.supportsBatch, + supportsSources: site.batch?.listSources != null, + }, siteUi: site.ui, onExport(scope, format, ids, panel, opts) { void dispatchExport(scope, format, ids, panel, opts) }, - onPickList(panel) { - void loadPickList(panel) + onPickList(panel, source) { + void loadPickList(panel, source) }, onPickMore(panel) { void loadNextPage(panel) @@ -131,7 +141,7 @@ async function dispatchExport( * 重置分页并拉第一页。注意这里**不碰** activeCancel / panel.finish()—— * 懒加载不占用「运行中」状态，取消按钮只属于导出流程。 */ -async function loadPickList(panel: PanelHandle): Promise { +async function loadPickList(panel: PanelHandle, source: string): Promise { const gen = ++pagerGen pager = null pickedList = [] @@ -140,7 +150,16 @@ async function loadPickList(panel: PanelHandle): Promise { panel.setStatus('获取登录态…') const session = await site.prepare() if (gen !== pagerGen) return - pager = site.batch!.createPager(session) + pager = site.batch!.createPager(session, undefined, source) + // 来源选项后台补齐，不阻塞第一页；不支持来源筛选的站点保持固定选项。 + if (site.batch!.listSources) { + void site.batch! + .listSources(session) + .then((sources) => { + if (gen === pagerGen) panel.setPickerProjects(sources) + }) + .catch(() => {}) + } panel.setStatus('拉取对话列表…') await loadNextPage(panel, gen) } catch (e) { @@ -165,6 +184,7 @@ async function loadNextPage(panel: PanelHandle, gen: number = pagerGen): Promise id: i.id, title: i.title, updated: shortDate(i.update_time), + project: i.project, })) panel.appendPicker(picked, done) panel.setStatus( @@ -196,8 +216,10 @@ async function exportSelection( return } panel.setStatus('获取登录态…') + const checkBatchSafety = createBatchSafetyGuard(site.batch!.policy, site.throttleStats) const session = await site.prepare(cancel) - await exportItems(format, items, 0, session, cancel, panel, opts, sink) + checkBatchSafety() + await exportItems(format, items, 0, session, cancel, panel, opts, sink, checkBatchSafety) } catch (e) { panel.setStatus(e instanceof CancelledError ? '已取消' : `出错：${String(e)}`) } finally { @@ -281,6 +303,7 @@ function createProcessor( panel: PanelHandle, opts: ExportOptions, sink: OutputSink, + checkBatchSafety?: () => void, ) { // fileId → 正文替换文本；同一附件跨对话只下载一次 const assetCache = new Map() @@ -302,7 +325,9 @@ function createProcessor( replacement = skippedNote(a, a.sizeBytes!, cap) } else { try { + checkBatchSafety?.() const { bytes, filename, contentType } = await site.fetchAsset(session, a, cancel, cap) + checkBatchSafety?.() const name = assetFileName(a, filename, contentType) // 文件落在笔记目录下；标准 Markdown 用相对笔记路径，Wikilink 用 vault 根路径。 const linkPath = `${attachPrefix}${a.fileId.slice(-8)}-${name}` @@ -313,6 +338,7 @@ function createProcessor( }) } catch (e) { if (e instanceof CancelledError) throw e + if (e instanceof BatchSafetyError) throw e if (e instanceof SizeLimitError) { assetsSkipped++ replacement = skippedNote(a, e.actualBytes, cap) @@ -339,7 +365,9 @@ function createProcessor( } async function processConversation(item: SiteConversationItem): Promise<{ path: string }> { + checkBatchSafety?.() const raw = await site.fetchRaw(session, item.id, cancel) + checkBatchSafety?.() if (kind === 'json') { const path = `raw/${item.id}.json` await sink.put(path, strToU8(JSON.stringify(raw, null, 2))) @@ -348,6 +376,7 @@ function createProcessor( const irContext = site.fetchIRContext ? await site.fetchIRContext(session, item.id, raw, cancel) : undefined + checkBatchSafety?.() const { markdown, title, assets } = renderConversation(site.toIR(raw, item.id, irContext), { thoughts: opts.thoughts, toolTraces: opts.toolTraces, @@ -439,15 +468,23 @@ async function startExport( const cancel: CancelToken = { cancelled: false } activeCancel = cancel try { + // prepare 也是本批次的网络请求，必须在它之前建立统计基线。 + const checkBatchSafety = createBatchSafetyGuard(site.batch!.policy, site.throttleStats) panel.setStatus('获取登录态…') const session = await site.prepare(cancel) + checkBatchSafety() + // 全量列表本身也会连续请求：从翻第一页前就开始观测，不能等列表拉完才熔断。 panel.setStatus('拉取对话列表…') const fullList = await site.batch!.listAll( session, - (n) => panel.setStatus(`拉取对话列表… 已 ${n} 条`), + (n) => { + checkBatchSafety() + panel.setStatus(`拉取对话列表… 已 ${n} 条`) + }, cancel, ) + checkBatchSafety() if (fullList.length === 0) { panel.setStatus('没有可导出的对话') return @@ -462,7 +499,7 @@ async function startExport( } if (skipped > 0) panel.setStatus(`跳过未变化 ${skipped} 条，导出 ${list.length} 条…`) - await exportItems(kind, list, skipped, session, cancel, panel, opts, sink) + await exportItems(kind, list, skipped, session, cancel, panel, opts, sink, checkBatchSafety) } catch (e) { panel.setStatus(e instanceof CancelledError ? '已取消' : `出错：${String(e)}`) } finally { @@ -481,11 +518,15 @@ async function exportItems( panel: PanelHandle, opts: ExportOptions, sinkIn: OutputSink | null, + checkBatchSafetyIn?: () => void, ): Promise { const sink = sinkIn ?? zipSink() + const policy = site.batch!.policy + const checkBatchSafety = checkBatchSafetyIn ?? createBatchSafetyGuard(policy, site.throttleStats) // 水位线合并推进：导出成功的对话记下 update_time，其余保持原状 const wmDraft: Watermark = { ...loadWatermark(wmKey(kind)) } - const proc = createProcessor(kind, session, cancel, panel, opts, sink) + const proc = createProcessor(kind, session, cancel, panel, opts, sink, checkBatchSafety) + let safetyReason: string | null = null // 单条失败不中断，收集后统一重试；失败过多则保护性中止（防止触发/加重账号级反滥用）， // 已抓取的内容照常落地 @@ -517,7 +558,27 @@ async function exportItems( } catch (e) { if (e instanceof CancelledError) throw e failed.push(item) - if (failed.length >= 25 && failed.length > done / 2) aborted = true + if (e instanceof BatchSafetyError) { + safetyReason = e.message + aborted = true + } else { + try { + // 请求本身抛出 429 时，处理器来不及在返回后检查；失败分支补查一次。 + checkBatchSafety() + } catch (risk) { + if (risk instanceof BatchSafetyError) { + safetyReason = risk.message + aborted = true + } else { + throw risk + } + } + const attempted = done + 1 + if (failureLimitReached(policy, failed.length, attempted)) { + safetyReason = `失败率过高（${failed.length}/${attempted}），已停止后续请求` + aborted = true + } + } } done++ panel.setProgress(done, items.length) @@ -528,17 +589,20 @@ async function exportItems( return { failed, untried, aborted } } - const pass1 = await runPass(list, 2, '抓取对话') + const pass1 = await runPass(list, policy.concurrency, '抓取对话') let failedItems = pass1.failed let untriedItems = pass1.untried let safetyAborted = pass1.aborted - if (failedItems.length > 0 && !safetyAborted) { - // 大概率是限流长尾：歇口气再用单并发慢速补一遍 - for (let s = 20; s > 0; s--) { + if (failedItems.length > 0 && !safetyAborted && policy.retryFailed) { + // 只有经过站点实测、明确允许的适配器才做第二遍；Claude 默认不重试整批失败项。 + let remainingMs = policy.retryDelayMs + while (remainingMs > 0) { ensureAlive(cancel) - panel.setStatus(`${failedItems.length} 条失败，${s}s 后低速重试…`) - await sleep(1000) + panel.setStatus(`${failedItems.length} 条失败，${Math.ceil(remainingMs / 1000)}s 后低速重试…`) + const waitMs = Math.min(remainingMs, 1000) + await sleep(waitMs) + remainingMs -= waitMs } const pass2 = await runPass(failedItems, 1, '重试失败条目') failedItems = pass2.failed @@ -555,7 +619,7 @@ async function exportItems( ...untriedItems.map((i) => ({ id: i.id, title: i.title, - error: '保护性中止，本次未尝试（下次增量导出会自动补上）', + error: `保护性中止，本次未尝试（${safetyReason ?? '失败过多'}；下次增量导出会自动补上）`, })), ] if (failures.length > 0) { @@ -567,7 +631,7 @@ async function exportItems( // 水位线只在产物真正落地后推进：取消/崩溃的运行不记，避免下次增量漏数据 saveWatermark(wmKey(kind), wmDraft) panel.setStatus( - `${safetyAborted ? '保护性中止（失败过多，防止触发服务端限制）。' : '完成：'}` + + `${safetyAborted ? `保护性中止（${safetyReason ?? '失败过多'}）。` : '完成：'}` + `${list.length - failures.length} 个对话，${doneDesc}` + (skipped > 0 ? `（另跳过未变化 ${skipped} 条）` : '') + (failures.length ? `，${failures.length} 个失败（见 _failures.json）` : '') + diff --git a/src/sites/chatgpt/convert.ts b/src/sites/chatgpt/convert.ts index 29e36be..9c81e04 100644 --- a/src/sites/chatgpt/convert.ts +++ b/src/sites/chatgpt/convert.ts @@ -22,7 +22,11 @@ interface Ctx { canvas: Map } -export function conversationToIR(conv: ConversationDetail, fallbackId = ''): IRConversation { +export function conversationToIR( + conv: ConversationDetail, + fallbackId = '', + projectName?: string, +): IRConversation { const convId = String(conv.conversation_id ?? conv.id ?? fallbackId) const title = (conv.title ?? '').trim() || 'Untitled' const messages = linearize(conv) @@ -56,12 +60,15 @@ export function conversationToIR(conv: ConversationDetail, fallbackId = ''): IRC `https://chatgpt.com/c/${branchMeta.branching_from_conversation_id}`, ]) } + if (projectName) extra.unshift(['project', yamlQuote(projectName)]) + + const gizmoId = conv.gizmo_id ?? null return { source: 'chatgpt', id: convId, title, - url: `https://chatgpt.com/c/${convId}`, + url: `https://chatgpt.com${gizmoId ? `/g/${gizmoId}` : ''}/c/${convId}`, created: toIso(conv.create_time), updated: toIso(conv.update_time), model: model || undefined, diff --git a/src/sites/chatgpt/index.ts b/src/sites/chatgpt/index.ts index 8e785fc..5cf3ebf 100644 --- a/src/sites/chatgpt/index.ts +++ b/src/sites/chatgpt/index.ts @@ -4,6 +4,8 @@ import { fetchConversation, getAccessToken, listAllConversations, + listProjects, + projectNameOf, resolveFileDownload, throttleStats, } from '../../api' @@ -17,8 +19,13 @@ const toItem = (i: ConversationListItem): SiteConversationItem => ({ id: i.id, title: i.title ?? '', update_time: i.update_time ?? null, + project: projectNameOf(i.gizmo_id), }) +interface ChatGPTIRContext { + projectName?: string +} + export const chatgptAdapter: SiteAdapter = { id: 'chatgpt', label: 'ChatGPT', @@ -35,7 +42,26 @@ export const chatgptAdapter: SiteAdapter = { fetchRaw: (session, id, cancel) => fetchConversation(session, id, cancel), - toIR: (raw, fallbackId) => conversationToIR(raw as ConversationDetail, fallbackId), + fetchIRContext: async (session, _id, raw, cancel): Promise => { + const gizmoId = (raw as ConversationDetail).gizmo_id + if (!gizmoId) return {} + const known = projectNameOf(gizmoId) + if (known) return { projectName: known } + try { + await listProjects(session, cancel) + } catch (error) { + if (cancel?.cancelled) throw error + return {} + } + return { projectName: projectNameOf(gizmoId) } + }, + + toIR: (raw, fallbackId, context) => + conversationToIR( + raw as ConversationDetail, + fallbackId, + (context as ChatGPTIRContext | undefined)?.projectName, + ), async fetchAsset( session: string, @@ -52,11 +78,20 @@ export const chatgptAdapter: SiteAdapter = { throttleStats, batch: { + // 保持 344 + 432 对话实测得到的既有节奏与失败重试策略。 + policy: { + concurrency: 2, + retryFailed: true, + retryDelayMs: 20_000, + failureAbortMin: 25, + failureAbortRatio: 0.5, + }, async listAll(session, onProgress, cancel) { return (await listAllConversations(session, onProgress, cancel)).map(toItem) }, - createPager(session, cancel) { - const pager = createConversationPager(session, cancel) + listSources: (session, cancel) => listProjects(session, cancel), + createPager(session, cancel, source) { + const pager = createConversationPager(session, cancel, source) return { async next() { const { items, done } = await pager.next() diff --git a/src/sites/claude/api.ts b/src/sites/claude/api.ts index ecbc152..7496177 100644 --- a/src/sites/claude/api.ts +++ b/src/sites/claude/api.ts @@ -30,7 +30,8 @@ export const CLAUDE_THROTTLE: ThrottleConfig = { spacingMaxMs: 8000, restEveryN: 40, restDurationMs: 30_000, - maxAttempts: 6, + // 首次失败后最多再试一次；批量层随后依据完整 429 统计决定是否熔断。 + maxAttempts: 1, } const fetcher: Fetcher = createFetcher(CLAUDE_THROTTLE) @@ -54,6 +55,7 @@ export async function resolveOrgId(cancel?: CancelToken): Promise { const uuid = list.find((o) => typeof o?.uuid === 'string')?.uuid if (uuid) return uuid } catch { + ensureAlive(cancel) /* 落到 cookie 兜底 */ } const fromCookie = /(?:^|;\s*)lastActiveOrg=([^;]+)/.exec(document.cookie)?.[1] @@ -114,9 +116,7 @@ export async function listSandboxFiles( }) } -// ——— 以下是批量导出的地基，当前版本的 UI 不暴露 ——— -// 单对话导出只需要上面两个端点。列表接口先按分页写好（形状与 ChatGPT 侧一致， -// 便于后续复用同一套编排），但在限流画像实测清楚之前不接进界面。 +// ——— 批量导出列表 —— 与 ChatGPT 共用编排，但使用 Claude 专属的保守节奏与熔断。——— export async function listConversationsPage( orgId: string, @@ -129,7 +129,12 @@ export async function listConversationsPage( const data: unknown = await res.json() // [待测] 分页参数是否被服务端认。若不认，这里会一次性拿回全部——调用方靠 // 「返回数 < limit」判断到底会误判，所以终止条件同样只认空页。 - return Array.isArray(data) ? (data as ClaudeConversationListItem[]) : [] + if (Array.isArray(data)) return data as ClaudeConversationListItem[] + if (data && typeof data === 'object') { + const nested = (data as { chat_conversations?: unknown }).chat_conversations + if (Array.isArray(nested)) return nested as ClaudeConversationListItem[] + } + throw new Error('Claude 对话列表结构已变化：预期数组') } export interface ConversationPager { @@ -148,6 +153,10 @@ export interface PagerOptions { fetchPage?: FetchPage /** 空页重试的等待基数（第 n 次等 n 倍）；设 0 即不等待 */ emptyRetryBaseMs?: number + /** 单个分页器最多发出的列表请求数，防接口忽略 offset 或结构漂移后空转。 */ + maxRequests?: number + /** 单个分页器最多接收的去重对话数。 */ + maxItems?: number } export function createConversationPager( @@ -157,10 +166,13 @@ export function createConversationPager( ): ConversationPager { const fetchPage = opts.fetchPage ?? listConversationsPage const emptyRetryBaseMs = opts.emptyRetryBaseMs ?? 4000 + const maxRequests = opts.maxRequests ?? 250 + const maxItems = opts.maxItems ?? 10_000 let offset = 0 let limit = 50 let emptyRetries = 0 let done = false + let requests = 0 const seen = new Set() return { @@ -168,6 +180,10 @@ export function createConversationPager( if (done) return { items: [], done: true } for (;;) { ensureAlive(cancel) + if (requests >= maxRequests) { + throw new Error(`Claude 列表分页请求已达安全上限（${maxRequests} 次），已停止以防空转`) + } + requests++ let items: ClaudeConversationListItem[] try { items = await fetchPage(orgId, offset, limit, cancel) @@ -193,11 +209,16 @@ export function createConversationPager( // 有返回、但全是见过的：要么服务端忽略了分页参数每次给同一批，要么已到底。 // 两种都不该重试——再问一次只会拿到同样的东西。 - const fresh = items.filter((i) => typeof i?.uuid === 'string' && !seen.has(i.uuid)) + const valid = items.filter((i) => typeof i?.uuid === 'string' && i.uuid !== '') + if (valid.length === 0) throw new Error('Claude 对话列表结构已变化：返回条目缺少 uuid') + const fresh = valid.filter((i) => !seen.has(i.uuid)) if (fresh.length === 0) { done = true return { items: [], done: true } } + if (seen.size + fresh.length > maxItems) { + throw new Error(`Claude 对话数已超过安全上限（${maxItems} 条），已停止本次列表拉取`) + } for (const i of fresh) seen.add(i.uuid) emptyRetries = 0 @@ -208,6 +229,23 @@ export function createConversationPager( } } +/** 全量列表与选择器共用同一套去重、空页确认和安全上限。 */ +export async function listAllConversations( + orgId: string, + onProgress?: (fetched: number) => void, + cancel?: CancelToken, + opts: PagerOptions = {}, +): Promise { + const pager = createConversationPager(orgId, cancel, opts) + const all: ClaudeConversationListItem[] = [] + for (;;) { + const { items, done } = await pager.next() + all.push(...items) + if (items.length > 0) onProgress?.(all.length) + if (done) return all + } +} + /** Claude 的附件地址是同源相对路径，登录态直接可取，不需要先换签名 URL。 */ export function fetchBinary( url: string, diff --git a/src/sites/claude/index.ts b/src/sites/claude/index.ts index c0ef8ac..4d52acc 100644 --- a/src/sites/claude/index.ts +++ b/src/sites/claude/index.ts @@ -1,23 +1,29 @@ import type { AssetRef } from '../../core/ir' import type { CancelToken } from '../../core/fetcher' -import type { AssetPayload, Rgb, SiteAdapter } from '../types' +import type { AssetPayload, Rgb, SiteAdapter, SiteConversationItem } from '../types' import { + createConversationPager, currentConversationId, fetchBinary, fetchConversation, + listAllConversations, listSandboxFiles, resolveOrgId, throttleStats, } from './api' import { conversationToIR } from './convert' -import type { ClaudeConversation, ClaudeIRContext } from './types' +import type { ClaudeConversation, ClaudeConversationListItem, ClaudeIRContext } from './types' + +const toItem = (item: ClaudeConversationListItem): SiteConversationItem => ({ + id: item.uuid, + title: item.name ?? '', + update_time: item.updated_at ?? null, +}) export const claudeAdapter: SiteAdapter = { id: 'claude', label: 'Claude', - // 首版只做「导出当前对话」。批量的地基（分页器、水位线、并发池）都在， - // 但在 Claude 的限流画像实测清楚之前不开——见 sites/claude/api.ts 的说明。 - supportsBatch: false, + supportsBatch: true, matches: () => /(^|\.)claude\.ai$/.test(location.hostname), @@ -61,13 +67,43 @@ export const claudeAdapter: SiteAdapter = { throttleStats, + batch: { + // Claude 限流画像仍未知：单并发、不做整批二次重试；一旦出现明确的全局 + // Retry-After 或累计 3 次 429，立刻保护性中止，未完成条目留给下次增量补齐。 + policy: { + concurrency: 1, + retryFailed: false, + retryDelayMs: 0, + failureAbortMin: 5, + failureAbortRatio: 0.25, + max429Hits: 3, + maxRetryAfterHits: 1, + maxRequests: 1000, + }, + async listAll(session, onProgress, cancel) { + return (await listAllConversations(session, onProgress, cancel)).map(toItem) + }, + createPager(session, cancel) { + const pager = createConversationPager(session, cancel) + return { + async next() { + const { items, done } = await pager.next() + return { items: items.map(toItem), done } + }, + } + }, + }, + ui: { // 2026-08-28 真实会话页实测：Files + Share 外层是 actions-group。锚定整个组的 - // 左边界才不会盖住 Files；旧选择器继续留作回退，兼容 Claude 的灰度发布。 + // 左边界才不会盖住 Files。2026-08-29 首页 /new 没有 wiggle 控件，但有稳定的 + // dframe-header-actions-slot（当前承载隐身模式按钮），同样锚定整组左边界。 + // 旧选择器继续留作回退，兼容 Claude 的灰度发布。 headerAnchor: () => (document.querySelector('[data-testid="wiggle-controls-actions-group"]') ?? document.querySelector('[data-testid="wiggle-controls-actions"]') ?? document.querySelector('[data-testid="wiggle-controls-actions-share"]') ?? + document.querySelector('#dframe-header-actions-slot') ?? document.querySelector('[data-testid="share-button"]') ?? document.querySelector('[data-testid="chat-menu-trigger"]') ?? document.querySelector('header button[aria-haspopup="menu"]') ?? @@ -96,15 +132,11 @@ export const claudeAdapter: SiteAdapter = { themeAttributes: ['class', 'data-mode', 'data-theme'], - // [待测] Claude 的主色变量名未确认。先试几个常见命名，命中不了就返回 null， - // 交给界面层的通用兜底（扫描含 accent 的自定义属性，取最饱和的那个）。 - accent(parse: (raw: string) => Rgb | null) { - const cs = getComputedStyle(document.documentElement) - for (const name of ['--accent-main-000', '--accent-main-100', '--accent-brand', '--brand']) { - const bg = parse(cs.getPropertyValue(name)) - if (bg) return { bg, fg: null, ring: null } - } - return null + // 与 Claude 星形图标接近的品牌珊瑚橙。Claude 页面没有稳定公开的 accent 变量， + // 固定色比扫描任意同名 CSS 变量更可预测；前景色仍交给通用层按对比度选择。 + accent(_parse: (raw: string) => Rgb | null) { + const brandOrange: Rgb = [217, 119, 87] // #D97757 + return { bg: brandOrange, fg: null, ring: brandOrange } }, }, } diff --git a/src/sites/types.ts b/src/sites/types.ts index 49fcd0f..f8f014d 100644 --- a/src/sites/types.ts +++ b/src/sites/types.ts @@ -3,6 +3,7 @@ import type { AssetRef, IRConversation } from '../core/ir' import type { CancelToken, FetchStats } from '../core/fetcher' +import type { BatchPolicy } from '../core/batch-safety' export type SiteId = 'chatgpt' | 'claude' @@ -37,6 +38,8 @@ export interface SiteConversationItem { id: string title: string update_time: string | number | null + /** 列表里的来源标签；ChatGPT 用 project 名，其他站点可缺省。 */ + project?: string } export interface SitePager { @@ -46,12 +49,17 @@ export interface SitePager { /** 批量导出能力。supportsBatch 为 true 时必须提供。 */ export interface SiteBatch { + /** 站点专属风控；编排层不得自行猜测未知站点可承受的并发与重试。 */ + policy: BatchPolicy listAll( session: string, onProgress?: (fetched: number) => void, cancel?: CancelToken, ): Promise - createPager(session: string, cancel?: CancelToken): SitePager + /** 可选的来源列表；缺省时 UI 只展示站点的统一列表。 */ + listSources?(session: string, cancel?: CancelToken): Promise> + /** source 缺省或为 all 时覆盖所有来源，其余值由站点自行解释。 */ + createPager(session: string, cancel?: CancelToken, source?: string): SitePager } export interface SiteAdapter { @@ -60,7 +68,7 @@ export interface SiteAdapter { label: string /** * 是否开放批量 / 全量导出。 - * Claude 首版为 false：限流画像尚无实测数据，先只做当前对话。 + * 开放前必须同时提供 batch 实现和站点级风控策略。 */ supportsBatch: boolean /** 当前页面是否属于这个站点 */ diff --git a/src/ui.ts b/src/ui.ts index 5d5252d..15436cc 100644 --- a/src/ui.ts +++ b/src/ui.ts @@ -44,6 +44,8 @@ export interface PickerItem { id: string title: string updated: string + /** 所属 project 名；缺省 = 主列表会话。 */ + project?: string } export interface PanelHandle { @@ -54,13 +56,15 @@ export interface PanelHandle { appendPicker(items: PickerItem[], done: boolean): void /** 清空多选列表（重新拉取前调用） */ clearPicker(): void + /** 填充来源下拉中的 project 选项，并保留仍然存在的当前选项。 */ + setPickerProjects(projects: { id: string; name: string }[]): void /** 某一页拉取失败：解除加载中状态，允许再次触发 */ pickerLoadFailed(): void } export interface PanelCallbacks { /** 当前站点：决定标题文案与批量入口是否出现 */ - site: { id: string; label: string; supportsBatch: boolean } + site: { id: string; label: string; supportsBatch: boolean; supportsSources: boolean } /** 站点专属的锚点与配色探测 */ siteUi: SiteUi /** ids 仅在 scope === 'selection' 时有意义 */ @@ -71,8 +75,8 @@ export interface PanelCallbacks { panel: PanelHandle, opts: ExportOptions, ): void - /** 首次切到「选择」或点重新拉取：回调负责重置分页并拉第一页 */ - onPickList(panel: PanelHandle): void + /** 首次进入、重新拉取或切换来源；source 为 all/main/project id。 */ + onPickList(panel: PanelHandle, source: string): void /** 列表滚到底部：回调负责拉下一页并调用 panel.appendPicker */ onPickMore(panel: PanelHandle): void onCancel(): void @@ -238,9 +242,17 @@ const STYLE = ` .picker { display: none; margin-top: 8px; } .picker.open { display: block; } + .picker .srcrow { display: flex; gap: 4px; margin-bottom: 6px; } + .picker select.src { + flex-shrink: 0; max-width: 108px; padding: 6px 7px; border: 1px solid var(--border); + border-radius: 8px; font-size: 12px; background: transparent; color: var(--fg); + outline: none; cursor: pointer; transition: border-color .15s var(--ease); + } + .picker select.src:focus { border-color: var(--accent); } + .picker select.src option { background: Canvas; color: CanvasText; } .picker input[type="search"] { - width: 100%; padding: 6px 9px; border: 1px solid var(--border); border-radius: 8px; - font-size: 12px; margin-bottom: 6px; background: transparent; color: var(--fg); outline: none; + flex: 1; min-width: 0; padding: 6px 9px; border: 1px solid var(--border); border-radius: 8px; + font-size: 12px; background: transparent; color: var(--fg); outline: none; transition: border-color .15s var(--ease); } .picker input[type="search"]::placeholder { color: var(--muted); } @@ -258,6 +270,10 @@ const STYLE = ` .picker .row:hover { background: var(--hover); } .picker .row.hidden { display: none; } .picker .row .t { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .picker .row .p { + flex-shrink: 0; max-width: 84px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; + color: var(--muted); font-size: 10px; padding: 1px 5px; border-radius: 999px; border: 1px solid var(--border); + } .picker .row .d { color: var(--muted); font-size: 10px; flex-shrink: 0; font-variant-numeric: tabular-nums; } .picker .sentinel { padding: 7px 0; text-align: center; color: var(--muted); font-size: 11px; } .picker .sentinel:empty { padding: 0; } @@ -495,6 +511,7 @@ export function mountPanel(cb: PanelCallbacks): void { // 批量能力未开放的站点直接不出现「全部 / 选择…」——按钮存在但点不动， // 比它根本不出现更让人困惑 const batchAttr = cb.site.supportsBatch ? '' : ' hidden' + const sourceAttr = cb.site.supportsSources ? '' : ' hidden' panel.innerHTML = `
导出 ${cb.site.label} 对话
@@ -505,7 +522,13 @@ export function mountPanel(cb: PanelCallbacks): void {
- +
+ + +
@@ -709,6 +732,7 @@ export function mountPanel(cb: PanelCallbacks): void { const pickerList = pickerEl.querySelector('.list')! const sentinel = pickerList.querySelector('.sentinel')! const pickerSearch = pickerEl.querySelector('input[type="search"]')! + const pickerSrc = pickerEl.querySelector('select.src')! const pickerEmpty = pickerEl.querySelector('.empty')! const pickerCount = pickerEl.querySelector('.count')! const segButtons = [...panel.querySelectorAll('.seg button')] @@ -880,7 +904,7 @@ export function mountPanel(cb: PanelCallbacks): void { for (const item of items) { const row = document.createElement('label') row.className = 'row' - row.title = item.title + row.title = item.project ? `${item.title}（${item.project}）` : item.title const box = document.createElement('input') box.type = 'checkbox' box.dataset['id'] = item.id @@ -890,7 +914,14 @@ export function mountPanel(cb: PanelCallbacks): void { const d = document.createElement('span') d.className = 'd' d.textContent = item.updated - row.append(box, t, d) + if (item.project) { + const p = document.createElement('span') + p.className = 'p' + p.textContent = item.project + row.append(box, t, p, d) + } else { + row.append(box, t, d) + } // 始终插在哨兵之前，哨兵保持在列表末尾 sentinel.before(row) } @@ -904,6 +935,17 @@ export function mountPanel(cb: PanelCallbacks): void { // 需要主动续拉，否则懒加载会停在第一页。 if (!done) queueMicrotask(maybeAutoFill) }, + setPickerProjects: (projects) => { + const keep = pickerSrc.value + while (pickerSrc.options.length > 2) pickerSrc.remove(2) + for (const project of projects) { + const option = document.createElement('option') + option.value = project.id + option.textContent = project.name + pickerSrc.add(option) + } + pickerSrc.value = [...pickerSrc.options].some((option) => option.value === keep) ? keep : 'all' + }, clearPicker: () => { for (const r of rows()) r.remove() listLoaded = false @@ -948,15 +990,17 @@ export function mountPanel(cb: PanelCallbacks): void { sentinel.addEventListener('click', requestMore) - /** 重置并拉第一页（首次进入「选择」/ 点重新拉取按钮） */ + /** 重置并拉第一页（首次进入「选择」/ 点重新拉取 / 切换来源） */ function loadList(): void { handle.clearPicker() pickerSearch.value = '' listLoading = true sentinel.textContent = '加载中…' - cb.onPickList(handle) + cb.onPickList(handle, pickerSrc.value) } + pickerSrc.addEventListener('change', loadList) + for (const btn of segButtons) { btn.addEventListener('click', () => { const group = btn.parentElement!.dataset['seg']! diff --git a/test/batch-safety.test.ts b/test/batch-safety.test.ts new file mode 100644 index 0000000..7c9ef5a --- /dev/null +++ b/test/batch-safety.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from 'bun:test' +import { + BatchSafetyError, + createBatchSafetyGuard, + failureLimitReached, + type BatchPolicy, +} from '../src/core/batch-safety' +import type { FetchStats } from '../src/core/fetcher' + +const policy: BatchPolicy = { + concurrency: 1, + retryFailed: false, + retryDelayMs: 0, + failureAbortMin: 5, + failureAbortRatio: 0.25, + max429Hits: 3, + maxRetryAfterHits: 1, + maxRequests: 1000, +} + +const stats = (patch: Partial = {}): FetchStats => ({ + spacingMs: 1500, + requests: 0, + hits429: 0, + retryAfterHits: 0, + cooldownMs: 0, + maxRetryAfterSec: 0, + ...patch, +}) + +describe('批量导出限流熔断', () => { + test('只计算本批次新增的 429，不受此前统计污染', () => { + let current = stats({ hits429: 7, retryAfterHits: 2 }) + const check = createBatchSafetyGuard(policy, () => current) + expect(() => check()).not.toThrow() + + current = stats({ hits429: 9, retryAfterHits: 2 }) + expect(() => check()).not.toThrow() + }) + + test('无 Retry-After 的 429 累计达到 3 次就停止', () => { + let current = stats() + const check = createBatchSafetyGuard(policy, () => current) + current = stats({ hits429: 3 }) + expect(() => check()).toThrow(BatchSafetyError) + expect(() => check()).toThrow('本批次已遇到 3 次 HTTP 429') + }) + + test('一次带 Retry-After 的全局限流信号就停止', () => { + let current = stats() + const check = createBatchSafetyGuard(policy, () => current) + current = stats({ hits429: 1, retryAfterHits: 1, maxRetryAfterSec: 60 }) + expect(() => check()).toThrow('Retry-After') + }) + + test('总请求数包含内部重试，达到批次上限就停止', () => { + let current = stats({ requests: 40 }) + const check = createBatchSafetyGuard({ ...policy, maxRequests: 3 }, () => current) + current = stats({ requests: 43 }) + expect(() => check()).toThrow('达到安全上限') + }) +}) + +describe('批量导出失败率护栏', () => { + test('未达到最小失败数时不误杀小样本', () => { + expect(failureLimitReached(policy, 4, 4)).toBe(false) + }) + + test('同时达到最小失败数并超过失败率才停止', () => { + expect(failureLimitReached(policy, 5, 20)).toBe(false) + expect(failureLimitReached(policy, 6, 20)).toBe(true) + }) +}) diff --git a/test/claude-pager.test.ts b/test/claude-pager.test.ts index f5c41b9..553cb82 100644 --- a/test/claude-pager.test.ts +++ b/test/claude-pager.test.ts @@ -1,10 +1,16 @@ import { describe, expect, test } from 'bun:test' import { ApiError } from '../src/core/fetcher' -import { createConversationPager, type FetchPage } from '../src/sites/claude/api' +import { + createConversationPager, + listAllConversations, + resolveOrgId, + type FetchPage, +} from '../src/sites/claude/api' import type { ClaudeConversationListItem } from '../src/sites/claude/types' // api.ts 拼 URL 要用 location.origin，bun 环境里补一个 ;(globalThis as unknown as { location: { origin: string } }).location = { origin: 'https://claude.ai' } +;(globalThis as unknown as { document: { cookie: string } }).document = { cookie: 'lastActiveOrg=org-from-cookie' } const items = (n: number, from = 0): ClaudeConversationListItem[] => Array.from({ length: n }, (_, i) => ({ uuid: `c${from + i}` })) @@ -97,4 +103,60 @@ describe('claude 分页器', () => { expect(again).toEqual({ items: [], done: true }) expect(calls.length).toBe(1) }) + + test('listAll 复用同一分页器并逐页报告进度', async () => { + const { fetchPage } = pages(({ offset }) => (offset < 60 ? items(30, offset) : [])) + const progress: number[] = [] + const all = await listAllConversations('org', (n) => progress.push(n), undefined, { + fetchPage, + emptyRetryBaseMs: 0, + }) + expect(all).toHaveLength(60) + expect(progress).toEqual([30, 60]) + }) + + test('进度回调触发风控异常时立即停止继续翻页', async () => { + const { calls, fetchPage } = pages(({ offset }) => items(5, offset)) + await expect( + listAllConversations( + 'org', + () => { + throw new Error('stop-by-risk-guard') + }, + undefined, + { fetchPage, emptyRetryBaseMs: 0 }, + ), + ).rejects.toThrow('stop-by-risk-guard') + expect(calls).toHaveLength(1) + }) + + test('分页请求超过安全上限时停止，不继续空转', async () => { + const { calls, fetchPage } = pages(({ offset }) => items(5, offset)) + const pager = createConversationPager('org', undefined, { + fetchPage, + emptyRetryBaseMs: 0, + maxRequests: 2, + }) + await pager.next() + await pager.next() + await expect(pager.next()).rejects.toThrow('分页请求已达安全上限') + expect(calls).toHaveLength(2) + }) + + test('累计条目超过安全上限时停止', async () => { + const { fetchPage } = pages(({ offset }) => items(5, offset)) + const pager = createConversationPager('org', undefined, { + fetchPage, + emptyRetryBaseMs: 0, + maxItems: 8, + }) + await pager.next() + await expect(pager.next()).rejects.toThrow('对话数已超过安全上限') + }) +}) + +describe('Claude 会话准备', () => { + test('已取消时不能吞掉取消异常并回退 cookie 继续执行', async () => { + await expect(resolveOrgId({ cancelled: true })).rejects.toThrow('已取消') + }) }) diff --git a/test/claude-ui.test.ts b/test/claude-ui.test.ts new file mode 100644 index 0000000..45c736f --- /dev/null +++ b/test/claude-ui.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, test } from 'bun:test' +import { claudeAdapter } from '../src/sites/claude' + +describe('Claude 界面主题', () => { + test('重点色固定为接近 Claude 图标的珊瑚橙', () => { + expect(claudeAdapter.ui.accent(() => null)).toEqual({ + bg: [217, 119, 87], + fg: null, + ring: [217, 119, 87], + }) + }) +}) diff --git a/test/fetcher.test.ts b/test/fetcher.test.ts new file mode 100644 index 0000000..401668f --- /dev/null +++ b/test/fetcher.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from 'bun:test' +import { createFetcher, parseRetryAfterSeconds } from '../src/core/fetcher' + +describe('Fetcher 限流统计', () => { + test('Retry-After 同时支持秒数和 HTTP-date', () => { + const now = Date.parse('2026-08-29T00:00:00Z') + expect(parseRetryAfterSeconds('60', now)).toBe(60) + expect(parseRetryAfterSeconds('Sat, 29 Aug 2026 00:01:00 GMT', now)).toBe(60) + }) + + test('重试耗尽的最后一次 429 也必须计入熔断统计', async () => { + const originalFetch = globalThis.fetch + globalThis.fetch = (() => + Promise.resolve( + new Response('', { status: 429, headers: { 'Retry-After': '60' } }), + )) as unknown as typeof fetch + try { + const fetcher = createFetcher({ + spacingBaseMs: 0, + spacingMaxMs: 0, + restEveryN: 0, + restDurationMs: 0, + maxAttempts: 0, + }) + await expect(fetcher.request('https://example.test/rate-limited')).rejects.toThrow('HTTP 429') + expect(fetcher.stats()).toMatchObject({ + requests: 1, + hits429: 1, + retryAfterHits: 1, + maxRetryAfterSec: 60, + }) + } finally { + globalThis.fetch = originalFetch + } + }) +}) diff --git a/test/markdown.test.ts b/test/markdown.test.ts index 4b2af57..4d625a1 100644 --- a/test/markdown.test.ts +++ b/test/markdown.test.ts @@ -18,6 +18,15 @@ describe('conversationToMarkdown', () => { expect(markdown).toContain('tags:\n - chatgpt') }) + test('project 会话使用 gizmo 地址并写入 project 名', () => { + const conv = { ...fixture, gizmo_id: 'g-p-project' } + const { markdown: md } = conversationToMarkdown(conv, '', { projectName: '研究计划' }) + expect(md).toContain( + 'url: https://chatgpt.com/g/g-p-project/c/abc12345-6789-4def-8012-3456789abcde', + ) + expect(md).toContain('project: "研究计划"') + }) + test('User / ChatGPT 作为最高级标题', () => { expect(markdown).toContain('\n# User\n') expect(markdown).toContain('\n# ChatGPT\n') diff --git a/test/ui-position.test.ts b/test/ui-position.test.ts index 5d4e97a..30a3cd4 100644 --- a/test/ui-position.test.ts +++ b/test/ui-position.test.ts @@ -37,4 +37,17 @@ describe('computeFabPlacement', () => { ), ).toEqual({ right: 110, bottom: 945, panelTop: 48 }) }) + + test('Claude 首页 header 贴在隐身模式动作槽左侧', () => { + // /new 实测：#dframe-header-actions-slot = x 578–610、y 8–40，viewport 630×898。 + expect( + computeFabPlacement( + 'header', + { top: 8, right: 610, bottom: 40, left: 578, height: 32 }, + { width: 630, height: 898 }, + 28, + 8, + ), + ).toEqual({ right: 60, bottom: 860, panelTop: 50 }) + }) })