From 9effdaf6d269bb8c93ca09b84ecbfb35c1be0589 Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Wed, 14 Oct 2026 16:34:56 +0800 Subject: [PATCH 1/4] test(queuefs): support script-aware cache mocks Co-authored-by: TRAE CLI --- crates/ragfs/src/cache_runtime/memory.rs | 64 +++++++++++++++++-- crates/ragfs/src/core/mountable.rs | 7 +- .../src/plugins/queuefs/cache_backend.rs | 30 ++++++++- crates/ragfs/src/plugins/queuefs/mod.rs | 9 ++- 4 files changed, 102 insertions(+), 8 deletions(-) diff --git a/crates/ragfs/src/cache_runtime/memory.rs b/crates/ragfs/src/cache_runtime/memory.rs index dbe9f4eb5c..49e864e1ec 100644 --- a/crates/ragfs/src/cache_runtime/memory.rs +++ b/crates/ragfs/src/cache_runtime/memory.rs @@ -3,16 +3,18 @@ use super::provider::{CacheOperation, CacheProvider}; use super::{ CacheError, CacheResult, Expiration, ListDirection, ListInsertPosition, ListInsertRequest, - ListMoveRequest, SetCondition, SetOptions, SetResult, + ListMoveRequest, ScriptRequest, ScriptResult, SetCondition, SetOptions, SetResult, }; use async_trait::async_trait; use bytes::Bytes; use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::sync::RwLock; +type ScriptHandler = dyn Fn(ScriptRequest) -> CacheResult + Send + Sync + 'static; + /// Controllable in-memory provider for tests and smoke validation. pub struct MemoryMockProvider { values: RwLock>, @@ -33,6 +35,7 @@ pub struct MemoryMockProvider { active_sets: AtomicU64, max_active_sets: AtomicU64, next_set_failure: Mutex>, + script_handler: Option>, } struct ActiveSetGuard<'a> { @@ -67,9 +70,19 @@ impl MemoryMockProvider { active_sets: AtomicU64::new(0), max_active_sets: AtomicU64::new(0), next_set_failure: Mutex::new(None), + script_handler: None, } } + /// Install a deterministic script handler for tests that need to advertise script support. + pub fn with_script_handler(mut self, handler: F) -> Self + where + F: Fn(ScriptRequest) -> CacheResult + Send + Sync + 'static, + { + self.script_handler = Some(Arc::new(handler)); + self + } + /// Delay individual get calls to exercise inflight and concurrency behavior. pub fn with_get_delay(mut self, delay: Duration) -> Self { self.get_delay = delay; @@ -231,9 +244,9 @@ impl Default for MemoryMockProvider { #[async_trait] impl CacheProvider for MemoryMockProvider { - /// Rejects script execution because the in-memory provider has no Lua engine. + /// Reject script-dependent consumers unless a test explicitly installs a handler. fn validate_operations(&self, operations: &[CacheOperation]) -> CacheResult<()> { - if operations.contains(&CacheOperation::ExecuteScript) { + if operations.contains(&CacheOperation::ExecuteScript) && self.script_handler.is_none() { return Err(CacheError::UnsupportedOperation( CacheOperation::ExecuteScript.name().to_string(), )); @@ -618,6 +631,15 @@ impl CacheProvider for MemoryMockProvider { Ok(Some(value)) } + async fn execute_script(&self, request: ScriptRequest) -> CacheResult { + self.ensure_open()?; + let handler = self + .script_handler + .as_ref() + .ok_or_else(|| CacheError::UnsupportedScript(request.script_id.clone()))?; + handler(request) + } + async fn close(&self) -> CacheResult<()> { self.closed.store(true, Ordering::Release); self.values.write().await.clear(); @@ -677,3 +699,37 @@ fn list_range(list: &VecDeque, start: i64, stop: i64) -> Vec { .cloned() .collect() } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[tokio::test] + async fn configured_script_handler_is_advertised_and_invoked() { + let calls = Arc::new(AtomicUsize::new(0)); + let observed_calls = Arc::clone(&calls); + let provider = MemoryMockProvider::new().with_script_handler(move |request| { + assert_eq!(request.script_id, "test.script.v1"); + observed_calls.fetch_add(1, Ordering::Relaxed); + Ok(ScriptResult { + payload: Bytes::from_static(b"handled"), + }) + }); + + provider + .validate_operations(&[CacheOperation::ExecuteScript]) + .unwrap(); + let result = provider + .execute_script(ScriptRequest { + script_id: "test.script.v1".to_string(), + keys: Vec::new(), + args: Vec::new(), + }) + .await + .unwrap(); + + assert_eq!(result.payload, Bytes::from_static(b"handled")); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } +} diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs index 97565c0cfa..e1fb696e14 100644 --- a/crates/ragfs/src/core/mountable.rs +++ b/crates/ragfs/src/core/mountable.rs @@ -1755,7 +1755,12 @@ mod tests { use crate::cache_runtime::{CacheRuntime, MemoryMockProvider}; use crate::plugins::QueueFSPlugin; - let provider = Arc::new(MemoryMockProvider::new()); + let provider = Arc::new(MemoryMockProvider::new().with_script_handler(|request| { + panic!( + "script execution is outside this shutdown test's scope: {}", + request.script_id + ) + })); let runtime = CacheRuntime::memory_with_provider(provider.clone()); let mfs = MountableFS::new(); mfs.register_plugin(QueueFSPlugin::with_cache_runtime(runtime)) diff --git a/crates/ragfs/src/plugins/queuefs/cache_backend.rs b/crates/ragfs/src/plugins/queuefs/cache_backend.rs index f96759afd4..26d0005264 100644 --- a/crates/ragfs/src/plugins/queuefs/cache_backend.rs +++ b/crates/ragfs/src/plugins/queuefs/cache_backend.rs @@ -670,6 +670,32 @@ fn cache_error(operation: &str, error: CacheError) -> Error { mod tests { use super::*; + fn lifecycle_test_runtime() -> Arc { + let provider = Arc::new( + crate::cache_runtime::MemoryMockProvider::new().with_script_handler(|request| { + panic!( + "script execution is outside this lifecycle test's scope: {}", + request.script_id + ) + }), + ); + CacheRuntime::memory_with_provider(provider) + } + + #[tokio::test] + async fn open_rejects_provider_without_script_support() { + let error = CacheQueueStorage::open(CacheRuntime::memory(), "unsupported".to_string()) + .await + .err() + .expect("memory provider must be rejected"); + + assert!(matches!( + error, + Error::Internal(message) + if message.contains("unsupported cache operation: execute_script") + )); + } + #[test] fn startup_recovery_uses_original_zero_thirty_sixty_second_offsets() { assert_eq!( @@ -688,7 +714,7 @@ mod tests { #[test] fn drop_outside_tokio_runtime_removes_heartbeat() { - let runtime = CacheRuntime::memory(); + let runtime = lifecycle_test_runtime(); let tokio_runtime = tokio::runtime::Runtime::new().unwrap(); let storage = tokio_runtime .block_on(CacheQueueStorage::open( @@ -706,7 +732,7 @@ mod tests { #[tokio::test] async fn shutdown_removes_heartbeat_before_returning() { - let runtime = CacheRuntime::memory(); + let runtime = lifecycle_test_runtime(); let storage = CacheQueueStorage::open(Arc::clone(&runtime), "async-shutdown-test".to_string()) .await diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rs index cf074939e4..5cfb5244cb 100644 --- a/crates/ragfs/src/plugins/queuefs/mod.rs +++ b/crates/ragfs/src/plugins/queuefs/mod.rs @@ -1330,7 +1330,14 @@ mod tests { #[cfg(feature = "cache")] #[tokio::test] async fn cache_runtime_failure_is_propagated_by_stat_and_read_dir() { - let provider = std::sync::Arc::new(crate::cache_runtime::MemoryMockProvider::new()); + let provider = std::sync::Arc::new( + crate::cache_runtime::MemoryMockProvider::new().with_script_handler(|request| { + panic!( + "script execution is outside this failure propagation test's scope: {}", + request.script_id + ) + }), + ); let runtime = crate::cache_runtime::CacheRuntime::memory_with_provider(provider.clone()); let fs = QueueFileSystem::with_cache_runtime(runtime, "queuefs-runtime-errors".to_string()) .await From fdae61e85956a1fd4cae6af2f5332a5e1459fbdf Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Wed, 14 Oct 2026 17:52:49 +0800 Subject: [PATCH 2/4] docs: add 23-data-organize.md guide Co-authored-by: TRAE CLI --- .../skills/openviking-memory/SKILL.md | 8 +++ docs/.vitepress/theme/docs-sections.ts | 1 + docs/en/api/02-resources.md | 2 +- docs/en/guides/06-mcp-integration.md | 2 +- docs/en/guides/23-data-organize.md | 69 +++++++++++++++++++ docs/zh/api/02-resources.md | 2 +- docs/zh/guides/06-mcp-integration.md | 2 +- docs/zh/guides/23-data-organize.md | 69 +++++++++++++++++++ openviking/server/mcp_endpoint.py | 19 +++-- openviking/server/resource_ingest.py | 1 + tests/server/test_mcp_endpoint.py | 10 ++- tests/server/test_mcp_parse_mode.py | 1 + 12 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 docs/en/guides/23-data-organize.md create mode 100644 docs/zh/guides/23-data-organize.md diff --git a/agent-plugins/skills/openviking-memory/SKILL.md b/agent-plugins/skills/openviking-memory/SKILL.md index 8c7dfc3cf3..e3fcddd697 100644 --- a/agent-plugins/skills/openviking-memory/SKILL.md +++ b/agent-plugins/skills/openviking-memory/SKILL.md @@ -80,6 +80,14 @@ decisions with their rationale, and reusable procedures or fixes. What not to persist: secrets and credentials, transient state, speculation, or bulk transcript dumps — store conclusions, not scrollback. +### File resources deliberately + +Resource directory rules do not apply to Memory; use `remember` or session +commit and let the extraction system organize it. Before `add_resource`, keep +the existing Resource layout and account policy, choose shared versus +identity-private scope, and avoid duplicate source roots. Use `parent` for a new +source and exact `to` for refresh. + ## Example User asks to fix a failing deployment: diff --git a/docs/.vitepress/theme/docs-sections.ts b/docs/.vitepress/theme/docs-sections.ts index 86884dc165..5ea57b1aab 100644 --- a/docs/.vitepress/theme/docs-sections.ts +++ b/docs/.vitepress/theme/docs-sections.ts @@ -129,6 +129,7 @@ export const sections: DocSection[] = [ p('workflows/01-overview') ]), g('Manage context', '组织与维护上下文', [ + p('guides/23-data-organize'), p('guides/18-openviking-assets'), p('guides/09-ovpack'), p('guides/15-snapshot'), diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md index 183a075b70..74ad605f11 100644 --- a/docs/en/api/02-resources.md +++ b/docs/en/api/02-resources.md @@ -1,6 +1,6 @@ # Resource Management -Resources are external knowledge that agents can reference. This page covers resource import, scheduled updates, and temporary uploads. For package export and import, see [OVPack](14-ovpack.md). +Resources are external knowledge that agents can reference. This page covers resource import, scheduled updates, and temporary uploads. See the [data organization guide](../guides/23-data-organize.md) for directory conventions and [OVPack](14-ovpack.md) for package export and import. ## Core Concepts diff --git a/docs/en/guides/06-mcp-integration.md b/docs/en/guides/06-mcp-integration.md index 8505ad5441..53d677fd8b 100644 --- a/docs/en/guides/06-mcp-integration.md +++ b/docs/en/guides/06-mcp-integration.md @@ -134,7 +134,7 @@ The built-in MCP tools are listed below. Use the connected server's `tools/list` | `remember` | Submit messages for long-term memory extraction; returns the background extraction `task_id` right away | `messages` (list of `{role, content}`) | | `write` | Write text to a `viking://` file (create/overwrite/append). Parent directories are created automatically; use `read` first to see current content before overwriting, and prefer `edit` for changing part of an existing file. Skill packages are not maintained this way: the caller's own `skills/` subtree is refused, and a write under `viking://agent/skills` produces a plain file that skips skill installation — use `add_skill` | `uri`, `content`, `mode` (optional: `replace` default — overwrites or creates if missing; `append` — appends or creates if missing; `create` — compatibility alias for `replace`), `wait` (optional, block until re-indexed), `acl` (optional) | | `edit` | Replace an exact string with new text in an existing `viking://` file — for targeted changes instead of a full rewrite. The file is left unchanged if `old_string` is not found, or matches multiple times while `replace_all` is false. Editing a file inside a skill package does not re-run skill installation — use `add_skill` | `uri`, `old_string`, `new_string`, `replace_all` (optional), `wait` (optional, block until re-indexed) | -| `add_resource` | Add a local file or URL as a resource (local files trigger a progressive upload flow) | `path`, `temp_file_id` (optional), `description` (optional), `watch_interval` (optional, minutes — auto-refresh cadence for remote URLs), `processing_mode` (optional: `semantic_and_vectors` default, or `vectors_only` to skip VLM semantic understanding and only vectorize current files), `to` (optional, target `viking://resources/...` URI; if omitted when `watch_interval > 0`, the watch auto-binds to the resource's created URI), `args` (optional parser-specific options, including `{"parse_mode":"no_split"}` to parse each source document into one Markdown body, `{"feishu_access_token":"u-..."}` for one-time Feishu user-token imports, or access/refresh tokens plus an optional `feishu_app_id` / `feishu_app_secret` pair for Feishu user-token watches), `acl` (optional) | +| `add_resource` | Add a local file or URL as a resource (local files trigger a progressive upload flow) | `path`, `temp_file_id` (optional), `description` (optional), `watch_interval` (optional, minutes — auto-refresh cadence for remote URLs), `processing_mode` (optional: `semantic_and_vectors` default, or `vectors_only` to skip VLM semantic understanding and only vectorize current files), `to` (optional, target `viking://resources/...` URI; if omitted when `watch_interval > 0`, the watch auto-binds to the resource's created URI), `parent` (optional destination directory, created automatically), `tags` / `tag_mode` (optional retrieval tags), `args` (optional parser-specific options, including `{"parse_mode":"no_split"}` to parse each source document into one Markdown body, `{"feishu_access_token":"u-..."}` for one-time Feishu user-token imports, or access/refresh tokens plus an optional `feishu_app_id` / `feishu_app_secret` pair for Feishu user-token watches), `acl` (optional) | | `add_skill` | Create, install, or replace an agent skill. New skills pass the full SKILL.md text; Git and GitHub tree URLs install every skill in the source unless `skills` names some; a local SKILL.md, directory, or zip returns a signed upload URL like `add_resource` | `data` (SKILL.md text) or `path` (Git URL or local path), `skills` (optional), `target_uri` (optional; `viking://agent/skills` shares with the account), `list_only` (optional) | | `list_watches` | List watch tasks (auto-refresh subscriptions) visible to the current agent. Each entry shows target URI, refresh interval (minutes), active/paused status, and next scheduled execution time | none | | `cancel_watch` | Cancel (delete) a watch task by its target URI. To change the cadence or pause temporarily, cancel and re-add with a new `watch_interval` | `to_uri` (must match the watch task's `to` value, e.g. `viking://resources/...`) | diff --git a/docs/en/guides/23-data-organize.md b/docs/en/guides/23-data-organize.md new file mode 100644 index 0000000000..f6b5c2a988 --- /dev/null +++ b/docs/en/guides/23-data-organize.md @@ -0,0 +1,69 @@ +# Data organization guide + +This guide defines how to organize Resource data in OpenViking. Directories establish visibility, retrieval scope, and collaboration boundaries. + +## Context types + +| Type | Content | Write and organization model | +| --- | --- | --- | +| Resource | Documents, sites, repositories, datasets, and compiled knowledge bases | Write with `add_resource`, `ov add-resource`, or `ov compile`; organize with this guide | +| Memory | Preferences, facts, events, and experience extracted from interactions | Write through session commit and other memory-extraction flows; let the memory schema and extraction system organize it | +| Skill | Repeatable methods, constraints, scripts, and supporting material | Install with `add_skill` or `ov add-skill` | + +The directory conventions in this guide apply only to Resource data. Do not mirror them under `memories/`. + +## Visibility + +| Location | Visibility | Use | +| --- | --- | --- | +| `viking://resources/` | Shared within the current account | Material used by multiple identities; ACLs provide finer access control | +| `viking://~/resources/` | Private to the current authenticated identity | Material used independently by the current identity | +| `viking://~/peers//resources/` | One peer under the current identity | Material associated with a stable interaction peer | + +An OpenViking `user` is an authenticated context owner and may represent a person, agent, application, or service identity. `viking://~` always refers to the current request identity's own user space. + +The `account_id` already forms the team-sharing boundary, so another `teams//` layer is unnecessary. Do not upload passwords, tokens, private keys, or other secrets as ordinary Resources. + +## Directory conventions + +`projects//` is an optional collaboration container. Use it only when the project name is stable and multiple identities need to share, retrieve, and maintain the same context. A project directory does not replace account or ACL visibility controls. + +The other category directories may sit directly below a Resource root or inside one project directory. Create them only as needed; a space does not need the full set. These names are conventions that prevent different agents from creating synonymous directories such as `doc`, `docs`, `documents`, and even `pdfs`. The table defines the common names; their full URIs follow the same pattern as `viking://resources/repos`. + +| Directory | Content | Boundary | +| --- | --- | --- | +| `projects//` | Project collaboration context | Continue using the categories in this table below it | +| `repos/` | Complete code repositories | Preserve the repository structure; do not split by programming language | +| `docs/` | Manuals, specifications, product documentation, Office files, papers, preprints, and other document snapshots | Put reproduction code and independent datasets in `repos/` and `datasets/` | +| `sites/` | Content sourced from a URL, domain, or feed and imported or refreshed as a web tree | Put a discrete offline document in `docs/` | +| `wiki/` | Knowledge bases that have been organized, deduplicated, restructured, and linked or indexed | Use as a stable output for LLM Wiki and similar `ov compile` tasks; keep raw sources in their source directories | +| `datasets/` | Tables, corpora, evaluation sets, structured archives, observations, and training or analysis data | Point clouds and GIS observations are datasets | +| `media/` | Finished images, audio, and video intended for direct viewing or listening | Put editable sources and engineering outputs in `artifacts/` | +| `artifacts/` | Non-document outputs used by editing, manufacturing, rendering, or runtime workflows | Includes drawings, CAD, 3D models, scenes, visual source files, templates, and brand material | +| `items/` | Entity collection organized by commodity, product, SKU, or another business object | Put bulk tables and training data in `datasets/` | +| `people/` | Entity collection organized by person, customer, contact, or another identity | User space, account boundaries, and ACLs still control visibility and sensitive information | +| `inbox/` | Resources not yet classified by the current identity | Recommended only under `viking://~/resources/`; not a durable account-shared category | + +`items/` and `people/` are examples of entity collection directories, not reserved names. An agent may use `companies/`, `positions/`, or another clear plural noun for the domain. These directories are ordinary Resource directories today and align with a future Item retrieval capability; the current release provides no dedicated Item retrieval semantics. + +A directory name does not imply format support. Check [supported resource formats](../api/02-resources.md#resource-types) before import. Store unsupported artifact files through an enabled object-storage Connector, or wait for raw-file storage and parser support. + +## Add and update parameter notes + +1. Keep one stable root per logical source; before adding, you can use `list` (`ls`) or `stat` to check whether the target already exists. +2. For a new source, prefer `parent` to specify its parent directory. Use an exact `to` to synchronize a fixed source, or when you want to name the target directory directly; in general, `parent` is recommended. +3. When the parent directory is expected to be missing, use `--parent-auto-create` in the CLI or `create_parent=true` in the SDK and API; MCP's `parent` creates the directory automatically. +4. Use the watch mechanism for remote sources that change and can be read again, to avoid manually importing duplicate copies. +5. `reason` should state the intent and purpose of adding the resource; add `tags` only when the request or system policy explicitly requires them. +6. The import returns the final URI, visibility, and task state; retrieve the resource only after processing completes. + +Account administrators can set a default add directory with `user_config_defaults.add_targets.resource_uri`; an identity-level `add_targets.resource_uri` can override it. Explicit `to` or `parent` values always take precedence. + +## Agent filing rules + +1. Distinguish Resource, Memory, and Skill first. +2. Preserve the target space's existing layout; follow `RESOURCE_POLICY.md` when present. +3. Use `projects//` for a clear collaboration project; otherwise file by source or form. +4. When visibility is unclear, use the current identity's `viking://~/resources/inbox/` and report the final location. +5. Name entity collection directories for the domain; do not create unsupported categories. +6. Ask before writing when the target already contains the same logical source or the shared scope is unclear. diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md index 6755b83335..9e4686cb56 100644 --- a/docs/zh/api/02-resources.md +++ b/docs/zh/api/02-resources.md @@ -1,6 +1,6 @@ # 资源管理 -资源是智能体可以引用的外部知识。本页介绍资源导入、定时更新和临时文件上传。数据包导出与导入见 [OVPack](14-ovpack.md)。 +资源是智能体可以引用的外部知识。本页介绍资源导入、定时更新和临时文件上传。目录规划见[数据组织指引](../guides/23-data-organize.md)，数据包导出与导入见 [OVPack](14-ovpack.md)。 ## 核心概念 diff --git a/docs/zh/guides/06-mcp-integration.md b/docs/zh/guides/06-mcp-integration.md index b8acb303d4..6999bd0360 100644 --- a/docs/zh/guides/06-mcp-integration.md +++ b/docs/zh/guides/06-mcp-integration.md @@ -130,7 +130,7 @@ claude mcp add --transport http openviking \ | `remember` | 提交消息做长期记忆提取；立即返回后台任务的 `task_id`，提取结果可能什么都不保留 | `messages`（`{role, content}` 列表） | | `write` | 向 `viking://` 文件写入文本（创建/覆盖/追加）。自动创建缺失的父目录；覆盖前请先用 `read` 查看当前内容；只改文件局部时优先用 `edit`。skill 包不要用它维护：调用方自己的 `skills/` 子树会被拒绝，写 `viking://agent/skills` 则生成绕过安装流程的普通文件，请改用 `add_skill` | `uri`, `content`, `mode`(可选:默认 `replace` — 覆盖或在缺失时创建,`append` — 追加或在缺失时创建,`create` — `replace` 的兼容别名), `wait`(可选,阻塞直到重建索引完成), `acl`(可选) | | `edit` | 在已有 `viking://` 文件中把精确字符串替换为新文本——用于局部修改，避免整文件重写。若 `old_string` 找不到、或匹配多处且 `replace_all` 为 false，则编辑失败且文件保持不变。编辑 skill 包内的文件不会重新触发 skill 安装流程，请改用 `add_skill` | `uri`, `old_string`, `new_string`, `replace_all`(可选), `wait`(可选,阻塞直到重建索引完成) | -| `add_resource` | 添加本地文件或 URL 作为资源(本地文件触发渐进式上传流) | `path`, `temp_file_id`(可选), `description`(可选), `watch_interval`(可选,分钟数 — 远程 URL 的自动刷新周期), `processing_mode`(可选：默认 `semantic_and_vectors`；传 `vectors_only` 时跳过 VLM 语义理解，只向量化当前文件), `to`(可选,目标 `viking://resources/...` URI；`watch_interval > 0` 时若省略 `to`,watch 将自动绑定到本次 add 创建的资源 URI), `args`(可选,特定 parser 参数，包括 `{"parse_mode":"no_split"}` 用于正常解析但每个源文档只生成一个 Markdown 正文、飞书一次性用户 token 导入使用 `{"feishu_access_token":"u-..."}`，或飞书用户 token watch 使用 access/refresh token，并可选传入 `feishu_app_id` / `feishu_app_secret`), `acl`(可选) | +| `add_resource` | 添加本地文件或 URL 作为资源(本地文件触发渐进式上传流) | `path`, `temp_file_id`(可选), `description`(可选), `watch_interval`(可选,分钟数 — 远程 URL 的自动刷新周期), `processing_mode`(可选：默认 `semantic_and_vectors`；传 `vectors_only` 时跳过 VLM 语义理解，只向量化当前文件), `to`(可选,目标 `viking://resources/...` URI；`watch_interval > 0` 时若省略 `to`,watch 将自动绑定到本次 add 创建的资源 URI), `parent`(可选,目标父目录，缺失时自动创建), `tags` / `tag_mode`(可选,检索标签), `args`(可选,特定 parser 参数，包括 `{"parse_mode":"no_split"}` 用于正常解析但每个源文档只生成一个 Markdown 正文、飞书一次性用户 token 导入使用 `{"feishu_access_token":"u-..."}`，或飞书用户 token watch 使用 access/refresh token，并可选传入 `feishu_app_id` / `feishu_app_secret`), `acl`(可选) | | `add_skill` | 新建、安装或替换 agent skill。新 skill 直接传完整 SKILL.md 文本；Git 与 GitHub tree URL 默认安装源里的全部 skill，可用 `skills` 挑选；本地 SKILL.md、目录或 zip 会和 `add_resource` 一样返回签名上传 URL | `data`（SKILL.md 文本）或 `path`（Git URL 或本地路径）, `skills`(可选), `target_uri`(可选；`viking://agent/skills` 表示账户共享), `list_only`(可选) | | `list_watches` | 列出当前 Agent 可见的 watch 任务（自动刷新订阅），每行显示目标 URI、刷新间隔（分钟）、active/paused 状态以及下一次调度时间 | 无 | | `cancel_watch` | 按目标 URI 取消（删除）watch 任务。若需调整刷新周期或临时暂停，请取消后使用新的 `watch_interval` 重新添加 | `to_uri`（必须匹配 watch 任务的 `to` 值，例如 `viking://resources/...`） | diff --git a/docs/zh/guides/23-data-organize.md b/docs/zh/guides/23-data-organize.md new file mode 100644 index 0000000000..219901ec76 --- /dev/null +++ b/docs/zh/guides/23-data-organize.md @@ -0,0 +1,69 @@ +# 数据组织指引 + +本指引规定 OpenViking 中 Resource 的目录组织方式。目录用于确定可见范围、检索范围和协作边界。 + +## 上下文类型 + +| 类型 | 内容 | 写入与组织方式 | +| --- | --- | --- | +| Resource | 文档、网站、代码仓库、数据集及编译后的知识库 | 通过 `add_resource`、`ov add-resource` 或 `ov compile` 写入，按本指引组织 | +| Memory | 从交互中提取的偏好、事实、事件和经验 | 通过 session commit 等记忆抽取流程写入，由 memory schema 和抽取系统组织 | +| Skill | 可重复执行的方法、约束、脚本和配套资料 | 通过 `add_skill` 或 `ov add-skill` 安装 | + +本指引的目录约定仅适用于 Resource。不要在 `memories/` 下套用 Resource 目录结构。 + +## 可见范围 + +| 位置 | 可见范围 | 用途 | +| --- | --- | --- | +| `viking://resources/` | 当前 account 共享 | 多个身份共同使用的资料；细粒度权限由 ACL 控制 | +| `viking://~/resources/` | 当前认证身份私有 | 当前身份独立使用的资料 | +| `viking://~/peers//resources/` | 当前身份下的指定 peer | 与稳定交互对象关联的资料 | + +OpenViking `user` 是经过认证的上下文所有者，可以代表人、Agent、应用或服务身份。`viking://~` 始终指向当前请求身份自己的 user 空间。 + +`account_id` 已构成团队共享边界，无需再增加 `teams//`。密码、令牌、私钥等秘密不得作为普通 Resource 上传。 + +## 目录约定 + +`projects//` 是可选的协作容器。仅当项目名称稳定，且多个身份需要共享、检索和维护同一批上下文时使用。项目目录不替代 account 或 ACL 的可见性控制。 + +其余分类目录可以直接位于 Resource 根目录，也可以位于一个项目目录下并按需创建，不要求每个空间具备完整分类，只作为一种约定，避免不同的 Agent 由于理解不一致创建出诸如 `doc`, `docs`, `documents` 甚至 `pdfs` 这样的重复目录导致信息沟通不畅。我们在下表约定一些常用的目录名命名，其完整路径如 `viking://resources/repos`。 + +| 目录 | 内容 | 边界 | +| --- | --- | --- | +| `projects//` | 项目协作上下文 | 在其下继续使用本表中的分类目录 | +| `repos/` | 完整代码仓库 | 保留仓库内部结构；不要按编程语言拆分 | +| `docs/` | 手册、规范、产品文档、Office 文档、论文、预印本及其他文档快照 | 复现代码和独立数据集分别放入 `repos/` 与 `datasets/` | +| `sites/` | 以 URL、域名或 Feed 为来源，并按网页树导入或更新的内容 | 单个离线文档放入 `docs/` | +| `wiki/` | 经归纳、去重、重组并建立导航或链接的知识库 | 用作 LLM Wiki 等 `ov compile` 任务的稳定输出；原始来源保留在来源目录 | +| `datasets/` | 表格、语料、评测集、结构化归档、观测数据及训练或分析数据 | 点云、GIS 等观测数据属于数据集 | +| `media/` | 供直接观看或收听的图片、音频和视频成品 | 可编辑源文件和工程产物放入 `artifacts/` | +| `artifacts/` | 进入编辑、制造、渲染或运行流程的非文档产物 | 包括图纸、CAD、3D 模型、场景、视觉源文件、模板和品牌素材 | +| `items/` | 以商品、产品、SKU 或其他业务对象为粒度的实体集合 | 批量表格和训练数据放入 `datasets/` | +| `people/` | 以人物、客户、联系人或其他身份为粒度的实体集合 | 可见范围和敏感信息仍由 user 空间、account 与 ACL 控制 | +| `inbox/` | 当前身份尚未归类的 Resource | 仅建议位于 `viking://~/resources/`，不作为 account 共享空间的长期分类 | + +`items/` 和 `people/` 是实体集合目录的示例，不是保留名称。Agent 可以根据领域使用 `companies/`、`positions/` 或其他明确的复数名词。此类目录当前仍是普通 Resource 目录，用于与后续 Item 检索能力对齐；当前版本不提供专门的 Item 检索语义。 + +目录名称不代表格式支持情况。导入前应检查[资源管理支持的格式](../api/02-resources.md#资源类型)。`artifacts/` 中尚未支持的文件应通过已启用的对象存储 Connector 保存，或等待相应的原始文件存储与解析能力。 + +## 添加和更新参数说明 + +1. 一个逻辑来源只保留一个稳定根目录，添加前可使用 `list(ls)` 或 `stat` 检查目标是否存在。 +2. 新增来源使用建议使用 `parent` 指定其父目录位置，同步固定来源或希望直接指定目标目录则使用精确的 `to`，一般建议用 `parent`。 +3. 预期父目录不存在时，CLI 使用 `--parent-auto-create`，SDK 和 API 使用 `create_parent=true`；MCP 的 `parent` 会自动创建目录。 +4. 可重复读取且会变化的远程来源使用 watch 机制，避免手动重复导入副本。 +5. `reason` 应说明资源添加意图和用途，`tags` 仅在请求或系统规范明确要求时添加。 +6. 导入完成后返回最终 URI、可见范围和任务状态；处理完成后再检索。 + +Account 管理者可以通过 `user_config_defaults.add_targets.resource_uri` 设置默认添加目录；身份级 `add_targets.resource_uri` 可以覆盖该默认值。显式传入的 `to` 或 `parent` 始终优先。 + +## Agent 归档规则 + +1. 先区分 Resource、Memory 和 Skill。 +2. 沿用目标空间已有的目录规范；存在 `RESOURCE_POLICY.md` 时优先遵循。 +3. 有明确协作项目时使用 `projects//`，否则按来源或形态归档。 +4. 无法确定可见范围时，写入当前身份的 `viking://~/resources/inbox/`，并返回最终位置。 +5. 实体集合目录按领域命名，不创建无依据的实体分类。 +6. 目标已有同一逻辑来源，或共享范围无法确定时，先确认再写入。 diff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.py index 769da2964e..4153385c97 100644 --- a/openviking/server/mcp_endpoint.py +++ b/openviking/server/mcp_endpoint.py @@ -1422,11 +1422,18 @@ async def add_resource( resource and never becomes an installed skill. Copying a source's text into a file with write instead skips parsing and stores a plain file. + Preserve the existing Resource layout and account policy. Resource directory rules do not + apply to Memory, which is organized by the memory extraction system. Keep one stable root + per source; use ``parent`` for a new source and exact ``to`` for an in-place refresh. See + ``docs/*/guides/23-data-organize.md`` for directory conventions. When writing to shared + resources, inspect the existing layout when practical and choose ``parent`` carefully. + Where it goes: ``viking://resources/`` is shared with the whole account and is the default when ``to`` and ``parent`` are empty (unless a default add target is configured). ``viking://~/resources/`` is the caller's own resources, visible only to - them; pass ``parent="viking://~/resources"`` to store it there. If the user has not - said, ask whether the resource is personal or shared with the account. + them. If visibility cannot be inferred, prefer the current identity's + ``parent="viking://~/resources/inbox"`` and report that choice; never silently put + identity-private material in the shared root. Remote URL: pass ``path`` as an http(s)://, git@, ssh://, or git:// URL. A sitemap / RSS / Atom URL ingests the WHOLE site as one resource tree; pass ``args={"site": true}`` @@ -1453,12 +1460,13 @@ async def add_resource( "vectors_only" to skip semantic understanding and only build vector indexes. to: Exact final URI including the leaf name (e.g. "viking://resources/volcengine/OpenViking", or - "viking://~/resources/OpenViking" for a personal copy). Written verbatim; an existing + "viking://~/resources/OpenViking" for an identity-private copy). Written verbatim; an existing target is synced to match the new source, so visible entries it does not contain are deleted. Required when ``add_type`` is set. - parent: Existing directory to store the resource under, for remote or + parent: Directory to store the resource under, for remote or local-file imports (e.g. "viking://~/resources" or "viking://resources"); - the leaf name comes from the source. Never overwrites + missing directories are created automatically. The leaf name comes from the source. + Never overwrites — a collision reserves the next free name ("name_1", "name_2", ...) and returns a warning. Mutually exclusive with ``to``; not supported when ``add_type`` is set. Leaving both empty derives the directory and the name @@ -1579,6 +1587,7 @@ async def add_resource( add_type=add_type or None, to=to or None, parent=parent or None, + create_parent=bool(parent), reason=description, wait=False, watch_interval=watch_interval, diff --git a/openviking/server/resource_ingest.py b/openviking/server/resource_ingest.py index d284783a4d..fff217d09f 100644 --- a/openviking/server/resource_ingest.py +++ b/openviking/server/resource_ingest.py @@ -59,6 +59,7 @@ async def ingest_temp_upload( ctx=ctx, to=to or None, parent=parent or None, + create_parent=bool(parent), reason=reason, source_name=resolved.original_filename, wait=False, diff --git a/tests/server/test_mcp_endpoint.py b/tests/server/test_mcp_endpoint.py index 7ac8555296..0f44ddbf82 100644 --- a/tests/server/test_mcp_endpoint.py +++ b/tests/server/test_mcp_endpoint.py @@ -1697,6 +1697,7 @@ async def fake_add_resource(*, path, ctx, **kwargs): ) assert captured["parent"] == "viking://resources/imports" + assert captured["create_parent"] is True assert captured["to"] is None @@ -1784,15 +1785,20 @@ async def test_add_resource_temp_file_id_branch_resolves_and_ingests( async def fake_add_resource(*, path, ctx, **kwargs): captured["path"] = path - captured["allow_local_path_resolution"] = kwargs.get("allow_local_path_resolution") + captured.update(kwargs) return {"root_uri": "viking://resources/from_tfid"} monkeypatch.setattr(service.resources, "add_resource", fake_add_resource) - result = await add_resource(temp_file_id=tfid) + result = await add_resource( + temp_file_id=tfid, + parent="viking://resources/imports", + ) assert "Resource added" in result assert captured["path"] == str(target.resolve()) assert captured["allow_local_path_resolution"] is True + assert captured["parent"] == "viking://resources/imports" + assert captured["create_parent"] is True upload_token_store.clear() diff --git a/tests/server/test_mcp_parse_mode.py b/tests/server/test_mcp_parse_mode.py index 97b14de5d6..83cd00f5ee 100644 --- a/tests/server/test_mcp_parse_mode.py +++ b/tests/server/test_mcp_parse_mode.py @@ -42,6 +42,7 @@ async def test_mcp_add_resource_schema_exposes_parse_mode_only_through_args(mcp_ assert "parse_mode" not in schema["properties"] assert "args" in schema["properties"] + assert "create_parent" not in schema["properties"] @pytest.mark.asyncio From c6b66d4e129e169cbf1e9408c69636bf6fc7c98b Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Wed, 14 Oct 2026 20:13:23 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix(mcp):=20=E5=86=85=E8=81=94=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E7=9B=AE=E5=BD=95=E7=BA=A6=E5=AE=9A=EF=BC=8C=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=E8=BF=90=E8=A1=8C=E6=97=B6=E4=B8=8D=E5=8F=AF=E8=AF=BB?= =?UTF-8?q?=E7=9A=84=E6=96=87=E6=A1=A3=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: TRAE CLI --- openviking/server/mcp_endpoint.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.py index 4153385c97..9da7e4db3d 100644 --- a/openviking/server/mcp_endpoint.py +++ b/openviking/server/mcp_endpoint.py @@ -1424,9 +1424,13 @@ async def add_resource( Preserve the existing Resource layout and account policy. Resource directory rules do not apply to Memory, which is organized by the memory extraction system. Keep one stable root - per source; use ``parent`` for a new source and exact ``to`` for an in-place refresh. See - ``docs/*/guides/23-data-organize.md`` for directory conventions. When writing to shared - resources, inspect the existing layout when practical and choose ``parent`` carefully. + per source; use ``parent`` for a new source and exact ``to`` for an in-place refresh. + Inspect the target space with ``list``/``tree`` before adding when practical and reuse its + existing directories; never create a synonymous category (e.g. ``documents/``, ``pdfs/``) + when one already fits. For a new category prefer the conventional names ``repos/``, + ``docs/``, ``sites/``, ``datasets/``, ``media/``, ``artifacts/``, + ``projects//`` for a multi-category project, or another plural domain noun for entity + collections (``items/``, ``people/``). Where it goes: ``viking://resources/`` is shared with the whole account and is the default when ``to`` and ``parent`` are empty (unless a default add target is From a5f9ee2a9a8734b1cf7060382ef9e693bf60112d Mon Sep 17 00:00:00 2001 From: Maojia Sheng Date: Wed, 14 Oct 2026 20:49:53 +0800 Subject: [PATCH 4/4] =?UTF-8?q?docs(data-organize):=20=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E6=97=A5=E5=8E=86=E8=B7=AF=E5=BE=84=E5=8F=98=E9=87=8F=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=EF=BC=8Cprojects=20=E8=A1=8C=E7=A7=BB=E8=87=B3?= =?UTF-8?q?=E8=A1=A8=E6=9C=AB=E5=B9=B6=E5=BC=BA=E8=B0=83=E5=8F=AF=E9=80=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: TRAE CLI --- docs/en/guides/23-data-organize.md | 4 +++- docs/zh/guides/23-data-organize.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/en/guides/23-data-organize.md b/docs/en/guides/23-data-organize.md index f6b5c2a988..1646c9cc9a 100644 --- a/docs/en/guides/23-data-organize.md +++ b/docs/en/guides/23-data-organize.md @@ -32,7 +32,6 @@ The other category directories may sit directly below a Resource root or inside | Directory | Content | Boundary | | --- | --- | --- | -| `projects//` | Project collaboration context | Continue using the categories in this table below it | | `repos/` | Complete code repositories | Preserve the repository structure; do not split by programming language | | `docs/` | Manuals, specifications, product documentation, Office files, papers, preprints, and other document snapshots | Put reproduction code and independent datasets in `repos/` and `datasets/` | | `sites/` | Content sourced from a URL, domain, or feed and imported or refreshed as a web tree | Put a discrete offline document in `docs/` | @@ -43,9 +42,12 @@ The other category directories may sit directly below a Resource root or inside | `items/` | Entity collection organized by commodity, product, SKU, or another business object | Put bulk tables and training data in `datasets/` | | `people/` | Entity collection organized by person, customer, contact, or another identity | User space, account boundaries, and ACLs still control visibility and sensitive information | | `inbox/` | Resources not yet classified by the current identity | Recommended only under `viking://~/resources/`; not a durable account-shared category | +| `projects//` | Project collaboration context | Optional container; when a project genuinely needs one, continue using the categories in this table below it | `items/` and `people/` are examples of entity collection directories, not reserved names. An agent may use `companies/`, `positions/`, or another clear plural noun for the domain. These directories are ordinary Resource directories today and align with a future Item retrieval capability; the current release provides no dedicated Item retrieval semantics. +For time-series material (daily reports, email archives, periodic reports), layer it by date inside the category rather than creating a category per date. Path variables can be used directly in the URI: `viking://resources/docs/{calendar:ym}/` renders to `docs/2026/05/`. Common variables include `{calendar:today}`, `{calendar:ym}`, `{calendar:yq}`, and `{calendar:yw}`; the CLI, SDK, and MCP pass the template as-is and the server resolves it. See [Viking URI path variables](../concepts/04-viking-uri.md#path-variables). + A directory name does not imply format support. Check [supported resource formats](../api/02-resources.md#resource-types) before import. Store unsupported artifact files through an enabled object-storage Connector, or wait for raw-file storage and parser support. ## Add and update parameter notes diff --git a/docs/zh/guides/23-data-organize.md b/docs/zh/guides/23-data-organize.md index 219901ec76..7754073f53 100644 --- a/docs/zh/guides/23-data-organize.md +++ b/docs/zh/guides/23-data-organize.md @@ -32,7 +32,6 @@ OpenViking `user` 是经过认证的上下文所有者，可以代表人、Agent | 目录 | 内容 | 边界 | | --- | --- | --- | -| `projects//` | 项目协作上下文 | 在其下继续使用本表中的分类目录 | | `repos/` | 完整代码仓库 | 保留仓库内部结构；不要按编程语言拆分 | | `docs/` | 手册、规范、产品文档、Office 文档、论文、预印本及其他文档快照 | 复现代码和独立数据集分别放入 `repos/` 与 `datasets/` | | `sites/` | 以 URL、域名或 Feed 为来源，并按网页树导入或更新的内容 | 单个离线文档放入 `docs/` | @@ -43,9 +42,12 @@ OpenViking `user` 是经过认证的上下文所有者，可以代表人、Agent | `items/` | 以商品、产品、SKU 或其他业务对象为粒度的实体集合 | 批量表格和训练数据放入 `datasets/` | | `people/` | 以人物、客户、联系人或其他身份为粒度的实体集合 | 可见范围和敏感信息仍由 user 空间、account 与 ACL 控制 | | `inbox/` | 当前身份尚未归类的 Resource | 仅建议位于 `viking://~/resources/`，不作为 account 共享空间的长期分类 | +| `projects//` | 项目协作上下文 | 可选容器；确有项目需要时，在其下继续使用本表中的分类目录 | `items/` 和 `people/` 是实体集合目录的示例，不是保留名称。Agent 可以根据领域使用 `companies/`、`positions/` 或其他明确的复数名词。此类目录当前仍是普通 Resource 目录，用于与后续 Item 检索能力对齐；当前版本不提供专门的 Item 检索语义。 +时间序列资料（日报、邮件归档、定期报告等）在分类目录下按日期分层即可，不要再为日期新建分类。URI 中可以直接使用服务端路径变量，例如 `viking://resources/docs/{calendar:ym}/` 会渲染为 `docs/2026/05/`；常用变量包括 `{calendar:today}`、`{calendar:ym}`、`{calendar:yq}`、`{calendar:yw}`，CLI、SDK 和 MCP 原样传递模板，由服务端解析。详见 [Viking URI 路径变量](../concepts/04-viking-uri.md#路径变量)。 + 目录名称不代表格式支持情况。导入前应检查[资源管理支持的格式](../api/02-resources.md#资源类型)。`artifacts/` 中尚未支持的文件应通过已启用的对象存储 Connector 保存，或等待相应的原始文件存储与解析能力。 ## 添加和更新参数说明