diff --git a/.env.example b/.env.example index 3bb75859b..5275a9f54 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,13 @@ UTOPIA_OPEN_REGISTRATION=true # 与 Utopia 共用一个 client 的就写 Utopia 自己的 client id。留空 = 不开放这个端点。 # 只认本人绑定过的身份,不建号、不自动绑定;每个 ID token 只能换一次。 # UTOPIA_OIDC_EXCHANGE_AUDIENCES= +# Google Drive 来源(0068):一个人用自己的 Google 账号连接,按他选的文件夹同步。 +# 在 Google Cloud 建一个「Web 应用」类型的 OAuth 客户端,回调地址填下面这个,并启用 Drive API。 +# 三项都填才开放;与上面 SSO 的客户端是两回事(权限范围不同)。drive.readonly 是受限范围: +# Workspace 内部应用不需要 Google 审核,对外发布的需要。 +# UTOPIA_GOOGLE_DRIVE_CLIENT_ID= +# UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET= +# UTOPIA_GOOGLE_DRIVE_REDIRECT_URI=https://your-deployment/api/v1/google-drive/callback # 数据库连接池上限。缺省 32,与 worker 并发的缺省对齐。 # 池子小于并发时症状是请求变慢、而不是任何一处说"池子不够"——调高 worker # 并发时一并考虑这里,也别超过 Postgres 自己的 max_connections(默认 100)。 diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 3d700e9d2..7a9ba5011 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current` // 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 100; +const CURRENT_SCHEMA_VERSION: u32 = 101; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 360bef0c5..0224f2a43 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -233,6 +233,8 @@ pub const SOURCE_SECRET_KEYS: &[&str] = &[ "secret_access_key", "account_key", "service_account_key", + // Google Drive 的 OAuth 刷新令牌(0068):能换出那个人整个 Drive 的只读权限 + "refresh_token", ]; impl Source { @@ -277,6 +279,8 @@ pub enum SourceKind { Gcs, Webdav, Notion, + /// 一个人授权、按他选的文件夹同步(0068) + GoogleDrive, Api, /// 推送的不是文档而是陈述本身(0054):请求体就是开放抽取契约,抽取不问模型 Statements, diff --git a/crates/utopia-server/src/api/google_drive_routes.rs b/crates/utopia-server/src/api/google_drive_routes.rs new file mode 100644 index 000000000..7e8d28da3 --- /dev/null +++ b/crates/utopia-server/src/api/google_drive_routes.rs @@ -0,0 +1,384 @@ +//! Google Drive 来源的连接与选文件夹(0068)。 +//! +//! **连接由来源的编辑者本人完成。** 他在来源设置里点「连接」,我们记下一次流程 +//! (state、PKCE verifier、是谁、哪个来源),把他送到 Google 的同意页;回调时这个 +//! 浏览器里的会话必须还是他,而且他对那个库还得是 Editor。不核对的话,有人可以 +//! 自己发起一次流程,再诱使别人点完同意——别人的 Drive 就接进了他的来源。 +//! +//! 刷新令牌只进不出:存进来源配置的 `refresh_token`(封印),文件夹浏览器的请求 +//! 在服务器上拿它换访问令牌,浏览器从头到尾拿不到任何一枚 Google 令牌。 + +use axum::extract::{Path, Query, State}; +use axum::response::Html; +use axum::Json; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use utopia_core::models::{Role, Source, SourceKind}; +use utopia_core::AppError; +use uuid::Uuid; + +use super::graph_routes::require_kb; +use crate::auth::{self, AuthUser}; +use crate::error::{ApiErr, ApiResult}; +use crate::google_drive::{self, Drive, FolderEntry, OAuthConfig}; +use crate::state::AppState; + +/// 一次连接流程能活多久:同意页上停十分钟还没点完,就当放弃了 +const FLOW_TTL_SECS: u64 = 600; +/// 同时在途的连接流程上限,理由同 SSO:每一次 `connect` 都落一行 +const MAX_PENDING_FLOWS: i64 = 1000; +/// 一个来源最多选多少个文件夹。选得更多的,选它们共同的上级 +const MAX_FOLDERS: usize = 50; + +fn configured() -> Result { + google_drive::oauth_config().ok_or_else(|| { + AppError::invalid( + "google_drive_not_configured", + "Google Drive is not configured on this server", + ) + }) +} + +/// 取一条 Google Drive 来源,确认它属于路径上的这个库 +async fn drive_source(state: &AppState, kb_id: Uuid, source_id: Uuid) -> Result { + let source = utopia_store::sources::get(&state.pool, source_id).await?; + if source.kb_id != kb_id || SourceKind::parse(&source.kind) != Some(SourceKind::GoogleDrive) { + return Err(AppError::NotFound); + } + Ok(source) +} + +/// 界面据此决定要不要提供这种来源的连接按钮 +pub async fn status(AuthUser(_): AuthUser) -> Json { + Json(json!({ "enabled": google_drive::oauth_config().is_some() })) +} + +/// 开始一次连接:回给界面一个 Google 同意页的地址,界面在弹窗里打开它 +pub async fn connect( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + drive_source(&state, kb_id, source_id).await?; + let cfg = configured()?; + + sqlx::query("DELETE FROM google_drive_flows WHERE expires_at < now()") + .execute(&state.pool) + .await?; + let pending: i64 = sqlx::query_scalar("SELECT count(*) FROM google_drive_flows") + .fetch_one(&state.pool) + .await?; + if pending >= MAX_PENDING_FLOWS { + return Err(AppError::invalid( + "google_drive_busy", + "Too many Google Drive connections are in progress; try again in a few minutes", + ) + .into()); + } + + let flow_state = auth::generate_jwt_secret(); + let verifier = auth::generate_jwt_secret(); + let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())); + sqlx::query( + "INSERT INTO google_drive_flows (state, verifier, user_id, source_id, expires_at) + VALUES ($1, $2, $3, $4, now() + make_interval(secs => $5))", + ) + .bind(&flow_state) + .bind(&verifier) + .bind(user.id) + .bind(source_id) + .bind(FLOW_TTL_SECS as f64) + .execute(&state.pool) + .await?; + + let mut url = + reqwest::Url::parse(google_drive::AUTHORIZE_URL).map_err(|e| AppError::Other(e.into()))?; + url.query_pairs_mut().extend_pairs([ + ("response_type", "code"), + ("client_id", cfg.client_id.as_str()), + ("redirect_uri", cfg.redirect.as_str()), + ("scope", google_drive::SCOPES), + ("state", flow_state.as_str()), + ("code_challenge", challenge.as_str()), + ("code_challenge_method", "S256"), + // 刷新令牌只在 offline + 每次都请同意时才保证发:重连一个撤过授权的来源也一样 + ("access_type", "offline"), + ("prompt", "consent"), + ("include_granted_scopes", "false"), + ]); + Ok(Json(json!({ "url": url.as_str() }))) +} + +#[derive(Deserialize)] +pub struct Callback { + code: Option, + state: Option, + /// 人在同意页点了取消 + error: Option, +} + +/// 回调落在弹窗里:成功与失败都回一张不带脚本的小页面。界面那头在轮询来源, +/// 看见 `account_email` 出现就关掉弹窗 +pub async fn callback( + State(state): State, + user: Result, + Query(q): Query, +) -> Html { + match finish(&state, user.ok().map(|AuthUser(u)| u), q).await { + Ok(()) => page( + "Google Drive connected", + "You can close this window and choose the folders to sync.", + ), + Err((code, reason)) => { + tracing::warn!(code, reason = %reason, "Google Drive 连接被拒"); + page( + "Google Drive was not connected", + match code { + "cancelled" => "The consent screen was cancelled. Close this window and try again.", + "session" => "This window is not signed in to Utopia as the person who started the connection. Close it and start again from the source settings.", + "expired" => "This connection attempt has expired. Close this window and start again.", + "scope" => "Google did not grant read access to Drive. Close this window, connect again and allow it.", + _ => "Google could not be reached or refused the request. Close this window and try again.", + }, + ) + } + } +} + +async fn finish( + state: &AppState, + user: Option, + q: Callback, +) -> Result<(), (&'static str, String)> { + let db = |e: sqlx::Error| ("unavailable", format!("database: {e}")); + let (Some(code), Some(flow_state)) = (q.code, q.state) else { + return Err(match q.error { + Some(e) => ("cancelled", format!("google returned {e}")), + None => ("invalid", "callback without code or state".into()), + }); + }; + // 一次性:先删后用,同一个 state 第二次到就什么都取不出来 + let flow: Option<(String, Uuid, Uuid)> = sqlx::query_as( + "DELETE FROM google_drive_flows WHERE state = $1 AND expires_at > now() + RETURNING verifier, user_id, source_id", + ) + .bind(&flow_state) + .fetch_optional(&state.pool) + .await + .map_err(db)?; + let (verifier, flow_user, source_id) = + flow.ok_or(("expired", "unknown or expired state".to_string()))?; + if let Some(e) = q.error { + return Err(("cancelled", format!("google returned {e}"))); + } + let user = user.filter(|u| u.id == flow_user).ok_or(( + "session", + "callback session is not the user who started it".to_string(), + ))?; + let source = utopia_store::sources::get(&state.pool, source_id) + .await + .map_err(|e| ("invalid", format!("source: {e}")))?; + // 发起之后的这几分钟里权限可能被收回了 + require_kb(state, &user, source.kb_id, Role::Editor) + .await + .map_err(|_| { + ( + "session", + "user is no longer an editor of the knowledge base".to_string(), + ) + })?; + + let cfg = google_drive::oauth_config().ok_or(("unavailable", "not configured".to_string()))?; + let grant = google_drive::exchange_code(&cfg, &code, &verifier) + .await + .map_err(|e| { + let msg = format!("{e:#}"); + ( + if msg.contains("read access to Drive") { + "scope" + } else { + "unavailable" + }, + msg, + ) + })?; + + let mut config = source.config.clone(); + if !config.is_object() { + config = json!({}); + } + config["refresh_token"] = json!(grant.refresh_token); + config["account_email"] = json!(grant.account_email); + // 列表里看不见令牌,界面凭这一项知道「连上了」:没验证过的邮箱不显示,不能拿它当标志 + config["connected_at"] = json!(chrono::Utc::now().to_rfc3339()); + utopia_store::sources::update(&state.pool, source_id, None, Some(&config), None, None) + .await + .map_err(|e| ("unavailable", format!("saving the grant: {e}")))?; + state.emit_source(source.kb_id); + // 审计不落令牌,只记连上的是哪个 Google 账号 + let _ = utopia_store::audit::record( + &state.pool, + Some(source.kb_id), + user.id, + "source.google_drive_connected", + "source", + Some(source_id), + json!({ "account_email": grant.account_email }), + ) + .await; + Ok(()) +} + +/// 断开:删掉刷新令牌,并请 Google 把它作废。作废失败不拦着——库里已经没有它了 +pub async fn disconnect( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + let mut config = source.config.clone(); + let token = config["refresh_token"].as_str().map(String::from); + if let Some(obj) = config.as_object_mut() { + obj.remove("refresh_token"); + obj.remove("account_email"); + obj.remove("connected_at"); + } + utopia_store::sources::update(&state.pool, source_id, None, Some(&config), None, None).await?; + if let Some(token) = token { + let revoked = reqwest::Client::new() + .post("https://oauth2.googleapis.com/revoke") + .form(&[("token", token.as_str())]) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + if let Err(e) = revoked { + tracing::warn!(error = %e, "Google 刷新令牌作废失败,库里已删除"); + } + } + state.emit_source(kb_id); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "source.google_drive_disconnected", + "source", + Some(source_id), + json!({}), + ) + .await; + Ok(Json(json!({ "ok": true }))) +} + +#[derive(Deserialize)] +pub struct FoldersQuery { + /// 不给 = 第一层(我的云端硬盘、与我共享、各个共享云端硬盘) + parent: Option, +} + +/// 文件夹浏览器:以连上的那个人的身份列子文件夹 +pub async fn folders( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, + Query(q): Query, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + let cfg = configured()?; + let refresh = source.config["refresh_token"] + .as_str() + .filter(|t| !t.is_empty()) + .ok_or_else(|| { + AppError::invalid( + "google_drive_not_connected", + "Connect a Google account first", + ) + })?; + let unavailable = |e: anyhow::Error| { + AppError::invalid_detail( + "google_drive_unavailable", + "Google Drive could not be read", + format!("{e:#}"), + ) + }; + let drive = Drive::connect(&cfg, refresh).await.map_err(unavailable)?; + let folders: Vec = match q.parent.as_deref().filter(|p| !p.is_empty()) { + None => drive.roots().await, + Some(parent) => drive.child_folders(parent).await, + } + .map_err(unavailable)?; + Ok(Json(json!({ "folders": folders }))) +} + +#[derive(Deserialize)] +pub struct SelectBody { + folders: Vec, +} + +/// 存下选中的文件夹,并立即同步一次。换了文件夹,上一批里不在新范围内的文档 +/// 会在这一轮被标成「不在来源中」,删不删由人决定,与别的来源一样 +pub async fn select( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, + Json(body): Json, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + if body.folders.len() > MAX_FOLDERS { + return Err(AppError::invalid( + "google_drive_too_many_folders", + format!("Choose at most {MAX_FOLDERS} folders"), + ) + .into()); + } + if body.folders.iter().any(|f| !google_drive::valid_id(&f.id)) { + return Err(AppError::invalid("google_drive_invalid_folder", "Invalid folder id").into()); + } + let mut config = source.config.clone(); + if !config.is_object() { + config = json!({}); + } + config["folders"] = json!(body.folders); + let updated = + utopia_store::sources::update(&state.pool, source_id, None, Some(&config), None, None) + .await?; + let connected = updated.config["refresh_token"] + .as_str() + .is_some_and(|t| !t.is_empty()); + if connected && !body.folders.is_empty() { + let _ = utopia_store::sources::mark_queued(&state.pool, source_id).await; + utopia_store::jobs::enqueue( + &state.pool, + "sync_source", + json!({ "source_id": source_id }), + ) + .await?; + } + state.emit_source(kb_id); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "source.updated", + "source", + Some(source_id), + json!({ "google_drive_folders": body.folders.len() }), + ) + .await; + Ok(Json(json!({ "source": updated.without_secrets() }))) +} + +/// 一张小页面,不带脚本。字符串都是这里写死的,不拼任何请求里来的东西 +fn page(title: &str, body: &str) -> Html { + Html(format!( + "{title}\ + \ + \ +

{title}

{body}

" + )) +} diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs index a0211913a..bce8995e3 100644 --- a/crates/utopia-server/src/api/mod.rs +++ b/crates/utopia-server/src/api/mod.rs @@ -8,6 +8,7 @@ mod datasource_routes; pub(crate) mod documents_routes; mod events_routes; mod export_routes; +mod google_drive_routes; mod graph_routes; mod jobs_routes; mod kbs; @@ -122,6 +123,9 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { .route("/auth/oidc/callback", get(oidc_routes::callback)) // 受信的另一个应用拿它用户刚到手的 ID token 换一个会话(0066) .route("/auth/oidc/exchange", post(oidc_routes::exchange)) + // Google Drive 来源(0068):部署配了 OAuth 客户端才有;回调落在连接弹窗里 + .route("/google-drive/status", get(google_drive_routes::status)) + .route("/google-drive/callback", get(google_drive_routes::callback)) // 我自己的绑定:看、解绑。绑定走 `/auth/oidc/start?link=1`,由本人完成 .route( "/auth/oidc/me", @@ -498,6 +502,18 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { 4 * sources_routes::STATEMENTS_MAX_BYTES, )), ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/connect", + post(google_drive_routes::connect), + ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/connection", + axum::routing::delete(google_drive_routes::disconnect), + ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/folders", + get(google_drive_routes::folders).put(google_drive_routes::select), + ) .route( "/kbs/{id}/sources/{source_id}/token", get(sources_routes::get_token), diff --git a/crates/utopia-server/src/google_drive.rs b/crates/utopia-server/src/google_drive.rs new file mode 100644 index 000000000..4ae3fdb86 --- /dev/null +++ b/crates/utopia-server/src/google_drive.rs @@ -0,0 +1,741 @@ +//! Google Drive 来源:一个人用自己的 Google 账号授权,按他选的文件夹同步(0068)。 +//! +//! 按 [0013](../../docs/decisions/0013-a-source-should-hand-over-its-history.md) 的四条判据: +//! +//! | 判据 | Google Drive | +//! |---|---| +//! | 真实时间戳 | `modifiedTime`,文件内容最后一次改动的时刻 | +//! | 会不会自我推翻 | 方案、纪要、合同的新版本覆盖旧版本 | +//! | 稳定身份 | 文件 id,改名、挪目录都不变 | +//! | 企业知识住不住在那儿 | 大多数团队的文档就住在这儿 | +//! +//! 与 Notion 一样**只交出现状**:修订历史在 API 里有,但 Google 文档的旧修订只能 +//! 导出成当时的全文,抽样哪几个是另一个决定,这一版不做。 +//! +//! ## 授权是一个人的,不是部署的 +//! +//! Notion、对象存储是管理员贴一把密钥;这里是**这个来源的编辑者**点「连接 Google +//! Drive」,在 Google 那边同意只读授权,我们存下刷新令牌(封印,`SOURCE_SECRET_KEYS`)。 +//! OAuth 客户端是部署级的(`UTOPIA_GOOGLE_DRIVE_CLIENT_ID` 等),没配就没有这个按钮。 +//! 授权顺带要 `openid email`,只为在界面上说清「这是以谁的身份在读」。 +//! +//! ## 几个容易踩的 +//! +//! **没有递归查询。** `q` 只认 `'{id}' in parents`,一层一层走;两个选中的文件夹 +//! 互相嵌套时同一个文件会出现两次,按 id 去重。 +//! +//! **共享云端硬盘里的东西默认看不见。** 每个请求都得带 `supportsAllDrives=true`, +//! 列表还得带 `includeItemsFromAllDrives=true`,少了不报错,只是结果是空的。 +//! +//! **Google 自己的格式要导出。** Docs/Sheets/Slides 没有字节,`alt=media` 回 403; +//! 走 `export` 换成 DOCX/XLSX/PPTX,交给已有的解析器(标题层级、表格、多张工作表都在)。 +//! 导出上限 10 MB,超了是 403 `exportSizeLimitExceeded`,按读不出来的那一份处理。 +//! +//! **导出的字节每次都不同**(压缩包里带时间戳),所以「变没变」不能靠内容哈希, +//! 靠 Drive 给的标记:二进制用 `md5Checksum`,Google 格式用 `modifiedTime`。 +//! +//! 错误文案是英文:它会原样落到来源的同步状态里给用户看。 + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::time::Duration; + +use anyhow::Context as _; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use chrono::{DateTime, Utc}; +use reqwest::StatusCode; +use serde_json::Value; +use uuid::Uuid; + +pub const AUTHORIZE_URL: &str = "https://accounts.google.com/o/oauth2/v2/auth"; +const TOKEN_URL: &str = "https://oauth2.googleapis.com/token"; +const API_ROOT: &str = "https://www.googleapis.com/drive/v3"; + +/// 只读整个 Drive。`drive.file` 不够:它只给应用自己建的、或经 Picker 逐个选中的 +/// 文件,选一个文件夹并不连带它里面的东西 +pub const DRIVE_SCOPE: &str = "https://www.googleapis.com/auth/drive.readonly"; +pub const SCOPES: &str = "openid email https://www.googleapis.com/auth/drive.readonly"; + +const FOLDER_MIME: &str = "application/vnd.google-apps.folder"; + +/// 一次同步最多收多少个文件,与 WebDAV、对象存储同一个量级:摄入不可逆, +/// 而一个共享盘可以有几万个文件 +const MAX_FILES_PER_SYNC: usize = 2_000; +/// 单个文件的上限,同对象存储 +const MAX_FILE_BYTES: u64 = 32 * 1024 * 1024; +/// 选中文件夹往下走几层。再深的留着不走,比一轮同步卡在一棵树上好 +const MAX_DEPTH: usize = 16; +/// 文件夹浏览器一次最多回多少个子文件夹 +const MAX_FOLDERS_LISTED: usize = 1_000; +/// 429 / 5xx 最多再试几回 +const MAX_RETRIES: u32 = 3; +const JSON_LIMIT: usize = 4 * 1024 * 1024; + +/// 部署级的 OAuth 客户端。三项都得有,缺一项就当没配 +pub struct OAuthConfig { + pub client_id: String, + pub client_secret: String, + pub redirect: String, +} + +pub fn oauth_config() -> Option { + let get = |key: &str| { + std::env::var(key) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + }; + Some(OAuthConfig { + client_id: get("UTOPIA_GOOGLE_DRIVE_CLIENT_ID")?, + client_secret: get("UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET")?, + redirect: get("UTOPIA_GOOGLE_DRIVE_REDIRECT_URI")?, + }) +} + +/// 文件、文件夹、共享盘的 id 只有这些字符。它们要拼进 `q` 的单引号里, +/// 不在这张表上的一律不收——比转义可靠 +pub fn valid_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= 128 + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} + +/// 来源配置里选中的文件夹:`folders: [{id, name}]`。名字只给界面看,同步只认 id +pub fn selected_folders(config: &Value) -> Vec { + config["folders"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|f| f["id"].as_str().or_else(|| f.as_str())) + .map(str::trim) + .filter(|id| valid_id(id)) + .map(String::from) + .collect() + }) + .unwrap_or_default() +} + +/// 授权码换来的东西 +#[derive(Debug)] +pub struct Grant { + pub refresh_token: String, + /// 授权的那个 Google 账号,ID token 里核验过的邮箱;只用来在界面上显示 + pub account_email: Option, +} + +fn http() -> anyhow::Result { + Ok(reqwest::Client::builder() + .timeout(Duration::from_secs(60)) + .build()?) +} + +/// 授权码 + PKCE verifier 换令牌。要的是刷新令牌:访问令牌一小时就过期, +/// 同步是在没人看着的时候跑的 +pub async fn exchange_code(cfg: &OAuthConfig, code: &str, verifier: &str) -> anyhow::Result { + exchange_code_at(&http()?, TOKEN_URL, cfg, code, verifier).await +} + +async fn exchange_code_at( + http: &reqwest::Client, + token_url: &str, + cfg: &OAuthConfig, + code: &str, + verifier: &str, +) -> anyhow::Result { + let resp = http + .post(token_url) + .form(&[ + ("grant_type", "authorization_code"), + ("code", code), + ("code_verifier", verifier), + ("client_id", &cfg.client_id), + ("client_secret", &cfg.client_secret), + ("redirect_uri", &cfg.redirect), + ]) + .send() + .await + .context("google token")?; + let v = token_response(resp).await?; + // 同意页上 Google 允许人把某一项权限的勾去掉;去掉了 Drive 的那一项, + // 令牌照发,只是什么都读不到。在这里说清楚,比让第一次同步报 403 好 + let scopes = v["scope"].as_str().unwrap_or(""); + anyhow::ensure!( + scopes.split_whitespace().any(|s| s == DRIVE_SCOPE), + "Google did not grant read access to Drive; connect again and allow it" + ); + // 只有第一次同意(或 `prompt=consent`)才发刷新令牌;`begin` 每次都带 consent + let refresh_token = v["refresh_token"] + .as_str() + .filter(|t| !t.is_empty()) + .context("Google returned no refresh token; connect again")? + .to_string(); + let account_email = match v["id_token"].as_str() { + Some(t) => id_token_email(t, &cfg.client_id)?, + None => None, + }; + Ok(Grant { + refresh_token, + account_email, + }) +} + +/// 刷新令牌换一个访问令牌 +async fn access_token_at( + http: &reqwest::Client, + token_url: &str, + cfg: &OAuthConfig, + refresh_token: &str, +) -> anyhow::Result { + let resp = http + .post(token_url) + .form(&[ + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("client_id", &cfg.client_id), + ("client_secret", &cfg.client_secret), + ]) + .send() + .await + .context("google token")?; + let v = token_response(resp).await?; + Ok(v["access_token"] + .as_str() + .filter(|t| !t.is_empty()) + .context("google token: response has no access_token")? + .to_string()) +} + +/// 令牌端点的响应。`invalid_grant` 单独说:那是人撤了授权、改了密码,或者 +/// 刷新令牌半年没用过期了——重连能好,重试不能 +async fn token_response(resp: reqwest::Response) -> anyhow::Result { + let status = resp.status(); + let v: Value = resp + .json() + .await + .context("google token: reading the response")?; + if status.is_success() { + return Ok(v); + } + if v["error"] == "invalid_grant" { + anyhow::bail!( + "Google Drive access was revoked or has expired; reconnect the source in its settings" + ); + } + anyhow::bail!( + "google token returned {status}: {}", + v["error_description"] + .as_str() + .or(v["error"].as_str()) + .unwrap_or("no message") + ) +} + +/// ID token 里的邮箱。 +/// +/// **不验签名**:这枚令牌是我们自己拿授权码、经 TLS 直接从 Google 的令牌端点换回来的, +/// OIDC Core 3.1.3.7 允许这时以 TLS 代替签名校验。issuer 与 audience 照查——查的是 +/// 「这是 Google 签给本部署的」。邮箱只拿来显示,不认身份、不开账号; +/// 没验证过的邮箱不显示,免得界面上写着一个那人并不拥有的地址 +fn id_token_email(id_token: &str, client_id: &str) -> anyhow::Result> { + let payload = id_token + .split('.') + .nth(1) + .context("google id_token is not a JWT")?; + let claims: Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(payload.trim_end_matches('=')) + .context("google id_token payload is not base64url")?, + ) + .context("google id_token payload is not JSON")?; + anyhow::ensure!( + matches!( + claims["iss"].as_str(), + Some("https://accounts.google.com" | "accounts.google.com") + ), + "google id_token has an unexpected issuer" + ); + anyhow::ensure!( + claims["aud"].as_str() == Some(client_id), + "google id_token was issued to a different client" + ); + Ok(match claims["email_verified"].as_bool() { + Some(true) => claims["email"].as_str().map(String::from), + _ => None, + }) +} + +/// 文件夹浏览器里的一项 +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct FolderEntry { + pub id: String, + pub name: String, +} + +/// 选中的文件夹里的一个文件 +#[derive(Debug, Clone)] +pub struct DriveFile { + pub id: String, + pub name: String, + pub mime: String, + pub modified: Option>, + md5: Option, + modified_raw: Option, +} + +impl DriveFile { + /// `gdrive://{file_id}`——文件 id 是它最稳的身份 + pub fn external_key(&self) -> String { + format!("gdrive://{}", self.id) + } + + /// 「变没变」的标记,见模块注释 + pub fn marker(&self) -> String { + self.md5 + .clone() + .or_else(|| self.modified_raw.clone()) + .unwrap_or_default() + } +} + +/// 一轮遍历的结果 +#[derive(Debug, Default)] +pub struct Listing { + pub files: Vec, + /// 到了文件数上限,或有文件夹深过 `MAX_DEPTH` + pub truncated: bool, + /// 选中却打不开的文件夹(删了、或不再共享给这个人)。有它时不对账: + /// 打不开不等于里面的东西都没了 + pub unreachable: Vec, + /// 跳过的文件,每个一句原因:表单、快捷方式、超大的 + pub skipped: usize, +} + +/// 读一个文件拿到的字节 +pub struct Content { + pub bytes: Vec, + pub filename: String, + pub mime: &'static str, +} + +/// Google 自己的格式导出成什么。不在这张表上的 Google 格式(表单、绘图、 +/// 快捷方式、站点……)没有能读的正文,跳过 +fn export_as(mime: &str) -> Option<(&'static str, &'static str)> { + Some(match mime { + "application/vnd.google-apps.document" => ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "docx", + ), + "application/vnd.google-apps.spreadsheet" => ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlsx", + ), + "application/vnd.google-apps.presentation" => ( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "pptx", + ), + _ => return None, + }) +} + +/// 拿着一个访问令牌的 Drive 客户端 +pub struct Drive { + http: reqwest::Client, + token: String, + // 私有地址入口让 HTTP 测试走真实的遍历逻辑,不增加用户配置 + api_root: String, +} + +impl Drive { + /// 用刷新令牌换访问令牌,得到一个能用的客户端 + pub async fn connect(cfg: &OAuthConfig, refresh_token: &str) -> anyhow::Result { + let http = http()?; + let token = access_token_at(&http, TOKEN_URL, cfg, refresh_token).await?; + Ok(Self { + http, + token, + api_root: API_ROOT.into(), + }) + } + + /// 发一次 GET。429 与 5xx 退避重试;其余非 2xx 带上 Google 自己的 message 报错, + /// 状态码与 `reason` 留在错误里给调用方分辨(404 = 打不开) + async fn get( + &self, + what: &str, + url: &str, + query: &[(&str, &str)], + ) -> anyhow::Result { + let url = reqwest::Url::parse_with_params(url, query) + .with_context(|| format!("google drive {what}: building the URL"))?; + let mut attempt = 0u32; + loop { + let resp = self + .http + .get(url.clone()) + .bearer_auth(&self.token) + .send() + .await + .with_context(|| format!("google drive {what}"))?; + let status = resp.status(); + if (status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error()) + && attempt < MAX_RETRIES + { + attempt += 1; + let wait = Duration::from_millis(500 * 2u64.pow(attempt)); + tracing::info!(what, attempt, %status, "google drive busy, waiting before the next attempt"); + tokio::time::sleep(wait).await; + continue; + } + if status.is_success() { + return Ok(resp); + } + let v: Value = resp.json().await.unwrap_or(Value::Null); + let reason = v["error"]["errors"][0]["reason"].as_str().unwrap_or(""); + return Err(DriveError { + status, + message: format!( + "google drive {what} returned {status}{}: {}", + if reason.is_empty() { + String::new() + } else { + format!(" ({reason})") + }, + v["error"]["message"].as_str().unwrap_or("no message") + ), + } + .into()); + } + } + + async fn get_json( + &self, + what: &str, + url: &str, + query: &[(&str, &str)], + ) -> anyhow::Result { + let resp = self.get(what, url, query).await?; + let bytes = read_capped(resp, JSON_LIMIT as u64) + .await + .with_context(|| format!("google drive {what}"))?; + serde_json::from_slice(&bytes).with_context(|| format!("google drive {what}: not JSON")) + } + + /// 按 `q` 列文件,翻完所有页或到 `limit` 为止。第二项:是不是因为到了上限才停的 + async fn list( + &self, + q: &str, + fields: &str, + limit: usize, + ) -> anyhow::Result<(Vec, bool)> { + let url = format!("{}/files", self.api_root); + let fields = format!("nextPageToken,files({fields})"); + let mut out = Vec::new(); + let mut page_token: Option = None; + loop { + let mut query = vec![ + ("q", q), + ("fields", fields.as_str()), + ("pageSize", "1000"), + ("orderBy", "name"), + ("supportsAllDrives", "true"), + ("includeItemsFromAllDrives", "true"), + ]; + if let Some(t) = &page_token { + query.push(("pageToken", t)); + } + let v = self.get_json("files.list", &url, &query).await?; + for f in v["files"].as_array().into_iter().flatten() { + if out.len() >= limit { + return Ok((out, true)); + } + out.push(f.clone()); + } + page_token = v["nextPageToken"].as_str().map(String::from); + if page_token.is_none() { + return Ok((out, false)); + } + } + } + + /// 文件夹浏览器的第一层:我的云端硬盘、与我共享的、每个看得见的共享云端硬盘。 + /// 前两项的名字由界面按 id 换成它的语言 + pub async fn roots(&self) -> anyhow::Result> { + let mut out = vec![ + FolderEntry { + id: "root".into(), + name: "My Drive".into(), + }, + FolderEntry { + id: SHARED_WITH_ME.into(), + name: "Shared with me".into(), + }, + ]; + let url = format!("{}/drives", self.api_root); + let mut page_token: Option = None; + loop { + let mut query = vec![ + ("pageSize", "100"), + ("fields", "nextPageToken,drives(id,name)"), + ]; + if let Some(t) = &page_token { + query.push(("pageToken", t)); + } + let v = self.get_json("drives.list", &url, &query).await?; + for d in v["drives"].as_array().into_iter().flatten() { + if let (Some(id), Some(name)) = (d["id"].as_str(), d["name"].as_str()) { + out.push(FolderEntry { + id: id.into(), + name: name.into(), + }); + } + } + page_token = v["nextPageToken"].as_str().map(String::from); + if page_token.is_none() || out.len() >= MAX_FOLDERS_LISTED { + return Ok(out); + } + } + } + + /// 一个文件夹(或共享盘、或「与我共享」)下面的子文件夹 + pub async fn child_folders(&self, parent: &str) -> anyhow::Result> { + let q = if parent == SHARED_WITH_ME { + format!("sharedWithMe = true and mimeType = '{FOLDER_MIME}' and trashed = false") + } else { + anyhow::ensure!(valid_id(parent), "invalid folder id"); + format!("'{parent}' in parents and mimeType = '{FOLDER_MIME}' and trashed = false") + }; + let (items, _) = self.list(&q, "id,name", MAX_FOLDERS_LISTED).await?; + Ok(items + .iter() + .filter_map(|f| { + Some(FolderEntry { + id: f["id"].as_str()?.into(), + name: f["name"].as_str()?.into(), + }) + }) + .collect()) + } + + /// 走遍选中的文件夹,收集文件。 + /// + /// 广度优先,访问过的文件夹不再进(两个选中的文件夹互相嵌套时),文件按 id 去重 + pub async fn walk(&self, folders: &[String]) -> anyhow::Result { + let mut listing = Listing::default(); + let mut visited: HashSet = HashSet::new(); + let mut seen_files: HashSet = HashSet::new(); + let mut queue: VecDeque<(String, usize)> = VecDeque::new(); + + for id in folders { + match self.folder_is_open(id).await { + Ok(true) => queue.push_back((id.clone(), 0)), + // 进了回收站:里面的东西对这个人已经不算数了,不进去,让对账把它们标出来 + Ok(false) => {} + Err(e) if is_not_found(&e) => listing.unreachable.push(id.clone()), + Err(e) => return Err(e), + } + } + + while let Some((folder, depth)) = queue.pop_front() { + if !visited.insert(folder.clone()) { + continue; + } + let q = format!("'{folder}' in parents and trashed = false"); + let room = MAX_FILES_PER_SYNC.saturating_sub(listing.files.len()) + MAX_FOLDERS_LISTED; + let (items, cut) = self + .list(&q, "id,name,mimeType,modifiedTime,md5Checksum,size", room) + .await?; + if cut { + listing.truncated = true; + } + for f in items { + let (Some(id), Some(name), Some(mime)) = + (f["id"].as_str(), f["name"].as_str(), f["mimeType"].as_str()) + else { + continue; + }; + if mime == FOLDER_MIME { + if depth + 1 > MAX_DEPTH { + listing.truncated = true; + } else { + queue.push_back((id.to_string(), depth + 1)); + } + continue; + } + let google_format = mime.starts_with("application/vnd.google-apps."); + if google_format && export_as(mime).is_none() { + listing.skipped += 1; + continue; + } + // Drive 的 size 是字符串里的整数;Google 格式没有 size + let size = f["size"].as_str().and_then(|s| s.parse::().ok()); + if !google_format && size.is_none_or(|n| n == 0 || n > MAX_FILE_BYTES) { + listing.skipped += 1; + continue; + } + if !seen_files.insert(id.to_string()) { + continue; + } + if listing.files.len() >= MAX_FILES_PER_SYNC { + listing.truncated = true; + return Ok(listing); + } + let modified_raw = f["modifiedTime"].as_str().map(String::from); + listing.files.push(DriveFile { + id: id.into(), + name: name.into(), + mime: mime.into(), + modified: modified_raw + .as_deref() + .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) + .map(|d| d.with_timezone(&Utc)), + md5: f["md5Checksum"].as_str().map(String::from), + modified_raw, + }); + } + } + Ok(listing) + } + + /// 选中的这个还在不在:在且没进回收站 → true;进了回收站 → false; + /// 打不开 → 404 的错误。`root` 是「我的云端硬盘」,共享盘的 id 也能当文件夹用 + async fn folder_is_open(&self, id: &str) -> anyhow::Result { + let url = format!("{}/files/{id}", self.api_root); + let v = self + .get_json( + "files.get", + &url, + &[ + ("fields", "id,mimeType,trashed"), + ("supportsAllDrives", "true"), + ], + ) + .await?; + anyhow::ensure!( + v["mimeType"] == FOLDER_MIME, + "google drive: selected item {id} is not a folder" + ); + Ok(v["trashed"].as_bool() != Some(true)) + } + + /// 一个文件的字节:Google 格式导出,其余原样下载 + pub async fn content(&self, file: &DriveFile) -> anyhow::Result { + if let Some((mime, ext)) = export_as(&file.mime) { + let url = format!("{}/files/{}/export", self.api_root, file.id); + let resp = self.get("export", &url, &[("mimeType", mime)]).await?; + let bytes = read_capped(resp, MAX_FILE_BYTES).await?; + let filename = if file.name.to_lowercase().ends_with(&format!(".{ext}")) { + file.name.clone() + } else { + format!("{}.{ext}", file.name) + }; + return Ok(Content { + bytes, + filename, + mime, + }); + } + let url = format!("{}/files/{}", self.api_root, file.id); + let resp = self + .get( + "download", + &url, + &[("alt", "media"), ("supportsAllDrives", "true")], + ) + .await?; + // 不猜 mime,理由同对象存储:解析先看魔数、再看扩展名 + Ok(Content { + bytes: read_capped(resp, MAX_FILE_BYTES).await?, + filename: file.name.clone(), + mime: "application/octet-stream", + }) + } +} + +/// 文件夹浏览器里「与我共享」的占位 id。真的 id 里没有冒号,撞不上 +pub const SHARED_WITH_ME: &str = "shared:with-me"; + +/// Drive 的一个非 2xx 响应 +#[derive(Debug)] +struct DriveError { + status: StatusCode, + message: String, +} + +impl std::fmt::Display for DriveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for DriveError {} + +fn is_not_found(e: &anyhow::Error) -> bool { + e.downcast_ref::() + .is_some_and(|d| d.status == StatusCode::NOT_FOUND) +} + +/// 读一个限了体积的响应体。先看 Content-Length,再边读边数 +async fn read_capped(mut resp: reqwest::Response, limit: u64) -> anyhow::Result> { + if resp.content_length().is_some_and(|n| n > limit) { + anyhow::bail!("response is larger than {limit} bytes"); + } + let mut body = Vec::new(); + while let Some(chunk) = resp.chunk().await? { + body.extend_from_slice(&chunk); + if body.len() as u64 > limit { + anyhow::bail!("response is larger than {limit} bytes"); + } + } + Ok(body) +} + +/// 上一轮摄入时每个文件的标记 +pub async fn markers( + pool: &sqlx::PgPool, + source_id: Uuid, +) -> anyhow::Result> { + let rows: Vec<(String, String)> = + sqlx::query_as("SELECT file_id, marker FROM google_drive_files WHERE source_id = $1") + .bind(source_id) + .fetch_all(pool) + .await?; + Ok(rows.into_iter().collect()) +} + +pub async fn remember( + pool: &sqlx::PgPool, + source_id: Uuid, + file_id: &str, + marker: &str, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO google_drive_files (source_id, file_id, marker) VALUES ($1, $2, $3) + ON CONFLICT (source_id, file_id) DO UPDATE SET marker = EXCLUDED.marker", + ) + .bind(source_id) + .bind(file_id) + .bind(marker) + .execute(pool) + .await?; + Ok(()) +} + +/// 这一轮没出现的文件忘掉标记:它再出现时得重新读一遍,不能凭旧标记跳过。 +/// 只在遍历完整的轮次调用,与对账同一个条件 +pub async fn forget_others( + pool: &sqlx::PgPool, + source_id: Uuid, + seen: &[String], +) -> anyhow::Result<()> { + sqlx::query("DELETE FROM google_drive_files WHERE source_id = $1 AND NOT (file_id = ANY($2))") + .bind(source_id) + .bind(seen) + .execute(pool) + .await?; + Ok(()) +} + +#[cfg(test)] +#[path = "google_drive_tests.rs"] +mod tests; diff --git a/crates/utopia-server/src/google_drive_tests.rs b/crates/utopia-server/src/google_drive_tests.rs new file mode 100644 index 000000000..dcf08401b --- /dev/null +++ b/crates/utopia-server/src/google_drive_tests.rs @@ -0,0 +1,460 @@ +//! Google Drive 的遍历、导出与授权。响应的形状照 Drive v3 与 Google 令牌端点的文档写, +//! **没有对一个真实的 Drive 跑过**(0013 决定 4 要求说出来):真连的那条在最下面, +//! 给了令牌才跑。 +use super::*; +use serde_json::json; +use wiremock::matchers::{body_string_contains, method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn drive(server: &MockServer) -> Drive { + Drive { + // 本地 mock 不走开发机的系统代理 + http: reqwest::Client::builder().no_proxy().build().unwrap(), + token: "ya29.test".into(), + api_root: server.uri(), + } +} + +fn cfg() -> OAuthConfig { + OAuthConfig { + client_id: "client-1.apps.googleusercontent.com".into(), + client_secret: "secret".into(), + redirect: "https://utopia.example/api/v1/google-drive/callback".into(), + } +} + +fn folder_meta(id: &str, trashed: bool) -> serde_json::Value { + json!({"id": id, "mimeType": FOLDER_MIME, "trashed": trashed}) +} + +async fn mount_folder(server: &MockServer, id: &str, trashed: bool) { + Mock::given(method("GET")) + .and(path(format!("/files/{id}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(folder_meta(id, trashed))) + .mount(server) + .await; +} + +async fn mount_children(server: &MockServer, parent: &str, files: serde_json::Value) { + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("'{parent}' in parents and trashed = false"), + )) + .and(query_param("supportsAllDrives", "true")) + .and(query_param("includeItemsFromAllDrives", "true")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "files": files }))) + .mount(server) + .await; +} + +fn id_token(claims: serde_json::Value) -> String { + format!( + "{}.{}.sig", + URL_SAFE_NO_PAD.encode(br#"{"alg":"RS256"}"#), + URL_SAFE_NO_PAD.encode(claims.to_string()) + ) +} + +/// 子文件夹往下走;Google 表单这类没有正文的格式与超大文件跳过; +/// 两个选中的文件夹互相嵌套时同一个文件只收一次 +#[tokio::test] +async fn a_walk_descends_skips_what_cannot_be_read_and_counts_each_file_once() -> anyhow::Result<()> +{ + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + mount_folder(&server, "sub", false).await; + mount_children( + &server, + "top", + json!([ + {"id": "sub", "name": "Sub", "mimeType": FOLDER_MIME}, + {"id": "doc1", "name": "Plan", "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-01T09:30:00.000Z"}, + {"id": "form1", "name": "Survey", "mimeType": "application/vnd.google-apps.form"}, + {"id": "big", "name": "video.mp4", "mimeType": "video/mp4", "size": "999999999"} + ]), + ) + .await; + mount_children( + &server, + "sub", + json!([ + {"id": "pdf1", "name": "contract.pdf", "mimeType": "application/pdf", + "size": "1024", "md5Checksum": "abc", "modifiedTime": "2026-09-01T00:00:00Z"} + ]), + ) + .await; + + // `sub` 也被单独选中:它的文件不该出现两次 + let listing = drive(&server) + .walk(&["top".to_string(), "sub".to_string()]) + .await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(ids, ["doc1", "pdf1"]); + assert_eq!(listing.skipped, 2, "表单与超大文件"); + assert!(!listing.truncated); + assert!(listing.unreachable.is_empty()); + + let doc = &listing.files[0]; + assert_eq!(doc.external_key(), "gdrive://doc1"); + assert_eq!( + doc.marker(), + "2026-10-01T09:30:00.000Z", + "Google 格式没有 md5,用修改时刻" + ); + assert_eq!( + doc.modified.map(|d| d.to_rfc3339()), + Some("2026-10-01T09:30:00+00:00".into()) + ); + assert_eq!(listing.files[1].marker(), "abc", "二进制文件用 md5"); + Ok(()) +} + +/// 选中的文件夹打不开(删了、或不再共享)不是错误,但要点名——同步据此不对账 +#[tokio::test] +async fn an_unreachable_folder_is_named_not_fatal() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/gone")) + .respond_with(ResponseTemplate::new(404).set_body_json(json!({ + "error": {"code": 404, "message": "File not found: gone.", + "errors": [{"reason": "notFound"}]} + }))) + .mount(&server) + .await; + mount_folder(&server, "here", false).await; + mount_children(&server, "here", json!([])).await; + + let listing = drive(&server) + .walk(&["gone".to_string(), "here".to_string()]) + .await?; + assert_eq!(listing.unreachable, ["gone"]); + Ok(()) +} + +/// 进了回收站的文件夹不进去,也不算打不开:里面的东西对这个人已经不算数了 +#[tokio::test] +async fn a_trashed_folder_is_not_walked() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "binned", true).await; + let listing = drive(&server).walk(&["binned".to_string()]).await?; + assert!(listing.files.is_empty()); + assert!(listing.unreachable.is_empty()); + // 没有任何 files.list 请求 + let lists = server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path() == "/files") + .count(); + assert_eq!(lists, 0); + Ok(()) +} + +/// 列表翻页:`nextPageToken` 一直跟到没有为止 +#[tokio::test] +async fn a_listing_follows_page_tokens() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + let file = |id: &str| { + json!({"id": id, "name": format!("{id}.txt"), "mimeType": "text/plain", + "size": "3", "md5Checksum": id}) + }; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param("pageToken", "p2")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"files": [file("b")]}))) + .with_priority(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"files": [file("a")], "nextPageToken": "p2"})), + ) + .with_priority(2) + .mount(&server) + .await; + + let listing = drive(&server).walk(&["top".to_string()]).await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(ids, ["a", "b"]); + Ok(()) +} + +/// Google 文档走 export 换成 DOCX,文件名补扩展名;别的原样下载,不猜 mime +#[tokio::test] +async fn google_formats_are_exported_and_files_are_downloaded() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/doc1/export")) + .and(query_param( + "mimeType", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + )) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"PK-docx".to_vec())) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files/pdf1")) + .and(query_param("alt", "media")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"%PDF-1.7".to_vec())) + .mount(&server) + .await; + + let d = drive(&server); + let doc = DriveFile { + id: "doc1".into(), + name: "Plan".into(), + mime: "application/vnd.google-apps.document".into(), + modified: None, + md5: None, + modified_raw: None, + }; + let c = d.content(&doc).await?; + assert_eq!(c.bytes, b"PK-docx"); + assert_eq!(c.filename, "Plan.docx"); + + let pdf = DriveFile { + id: "pdf1".into(), + name: "contract.pdf".into(), + mime: "application/pdf".into(), + modified: None, + md5: Some("x".into()), + modified_raw: None, + }; + let c = d.content(&pdf).await?; + assert_eq!(c.bytes, b"%PDF-1.7"); + assert_eq!(c.filename, "contract.pdf"); + assert_eq!(c.mime, "application/octet-stream"); + Ok(()) +} + +/// 导出超过 10 MB 时 Google 回 403 带 `exportSizeLimitExceeded`;错误里要留着这个原因, +/// 同步把它报给人看 +#[tokio::test] +async fn an_export_refusal_keeps_googles_reason() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/huge/export")) + .respond_with(ResponseTemplate::new(403).set_body_json(json!({ + "error": {"code": 403, "message": "This file is too large to be exported.", + "errors": [{"reason": "exportSizeLimitExceeded"}]} + }))) + .mount(&server) + .await; + let f = DriveFile { + id: "huge".into(), + name: "Budget".into(), + mime: "application/vnd.google-apps.spreadsheet".into(), + modified: None, + md5: None, + modified_raw: None, + }; + let Err(e) = drive(&server).content(&f).await else { + panic!("a 403 must not read as content"); + }; + let msg = format!("{e:#}"); + assert!(msg.contains("exportSizeLimitExceeded"), "{msg}"); + assert!(msg.contains("too large"), "{msg}"); +} + +/// 文件夹浏览器:「与我共享」用 sharedWithMe 查询,别的按父文件夹查,只要文件夹 +#[tokio::test] +async fn the_folder_browser_lists_only_folders() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("sharedWithMe = true and mimeType = '{FOLDER_MIME}' and trashed = false"), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "files": [{"id": "s1", "name": "Client X"}] + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("'root' in parents and mimeType = '{FOLDER_MIME}' and trashed = false"), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "files": [{"id": "f1", "name": "Projects"}] + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/drives")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "drives": [{"id": "0AAdrive", "name": "Team"}] + }))) + .mount(&server) + .await; + + let d = drive(&server); + assert_eq!( + d.child_folders(SHARED_WITH_ME).await?, + [FolderEntry { + id: "s1".into(), + name: "Client X".into() + }] + ); + assert_eq!( + d.child_folders("root").await?, + [FolderEntry { + id: "f1".into(), + name: "Projects".into() + }] + ); + let roots: Vec = d.roots().await?.into_iter().map(|r| r.id).collect(); + assert_eq!(roots, ["root", SHARED_WITH_ME, "0AAdrive"]); + // 拼进 q 的 id 不收引号 + assert!(d.child_folders("x' or '1'='1").await.is_err()); + Ok(()) +} + +/// 换令牌:要刷新令牌、要 Drive 那一项权限;邮箱从 ID token 里拿 +#[tokio::test] +async fn a_code_exchange_needs_a_refresh_token_and_the_drive_scope() -> anyhow::Result<()> { + let token_ok = json!({ + "access_token": "ya29.a", "refresh_token": "1//r", "expires_in": 3599, + "scope": format!("openid {DRIVE_SCOPE} https://www.googleapis.com/auth/userinfo.email"), + "id_token": id_token(json!({ + "iss": "https://accounts.google.com", "aud": cfg().client_id, + "sub": "1", "email": "bruno@example.com", "email_verified": true + })) + }); + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=good")) + .and(body_string_contains("code_verifier=v")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_ok.clone())) + .mount(&server) + .await; + let mut no_drive = token_ok.clone(); + no_drive["scope"] = json!("openid email"); + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=nodrive")) + .respond_with(ResponseTemplate::new(200).set_body_json(no_drive)) + .mount(&server) + .await; + let mut no_refresh = token_ok.clone(); + no_refresh.as_object_mut().unwrap().remove("refresh_token"); + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=norefresh")) + .respond_with(ResponseTemplate::new(200).set_body_json(no_refresh)) + .mount(&server) + .await; + + let http = reqwest::Client::builder().no_proxy().build()?; + let url = format!("{}/token", server.uri()); + let grant = exchange_code_at(&http, &url, &cfg(), "good", "v").await?; + assert_eq!(grant.refresh_token, "1//r"); + assert_eq!(grant.account_email.as_deref(), Some("bruno@example.com")); + + let e = exchange_code_at(&http, &url, &cfg(), "nodrive", "v") + .await + .unwrap_err(); + assert!(e.to_string().contains("read access to Drive"), "{e}"); + let e = exchange_code_at(&http, &url, &cfg(), "norefresh", "v") + .await + .unwrap_err(); + assert!(e.to_string().contains("no refresh token"), "{e}"); + Ok(()) +} + +/// 撤了授权或刷新令牌过期:说「去重连」,而不是一句 400 +#[tokio::test] +async fn a_revoked_grant_says_reconnect() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({ + "error": "invalid_grant", "error_description": "Token has been expired or revoked." + }))) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let e = access_token_at(&http, &format!("{}/token", server.uri()), &cfg(), "1//old") + .await + .unwrap_err(); + assert!(e.to_string().contains("reconnect"), "{e}"); +} + +/// ID token:签给别的 client 的、别的 issuer 的不收;没验证过的邮箱不显示 +#[test] +fn an_id_token_must_be_googles_and_ours() { + let ours = cfg().client_id; + let ok = id_token(json!({"iss": "accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": true})); + assert_eq!( + id_token_email(&ok, &ours).unwrap().as_deref(), + Some("a@b.c") + ); + + let unverified = id_token(json!({"iss": "https://accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": false})); + assert_eq!(id_token_email(&unverified, &ours).unwrap(), None); + + let other_client = id_token( + json!({"iss": "https://accounts.google.com", "aud": "someone-else", + "email": "a@b.c", "email_verified": true}), + ); + assert!(id_token_email(&other_client, &ours).is_err()); + + let other_issuer = id_token(json!({"iss": "https://evil.example", "aud": ours, + "email": "a@b.c", "email_verified": true})); + assert!(id_token_email(&other_issuer, &ours).is_err()); +} + +/// 配置里的文件夹:对象或字符串都认,id 不合字符表的丢掉 +#[test] +fn selected_folders_keep_only_valid_ids() { + let cfg = json!({"folders": [ + {"id": "1AbC-_x", "name": "Clients"}, "root", {"id": "bad'id"}, {"name": "no id"} + ]}); + assert_eq!(selected_folders(&cfg), ["1AbC-_x", "root"]); + assert!(selected_folders(&json!({})).is_empty()); +} + +/// 真连一个 Google Drive。给了令牌才跑,CI 上永远跳过。 +/// +/// ```text +/// # 在 Google Cloud 建一个 OAuth 客户端(Web 应用),在 Utopia 里连接一个 google_drive 来源, +/// # 然后用那个客户端和一枚刷新令牌: +/// UTOPIA_GOOGLE_DRIVE_CLIENT_ID=… UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET=… \ +/// UTOPIA_GOOGLE_DRIVE_REDIRECT_URI=… UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN=1//… \ +/// UTOPIA_GOOGLE_DRIVE_TEST_FOLDER= cargo test -p utopia-server google_drive +/// ``` +#[tokio::test] +async fn it_reads_from_a_real_drive() -> anyhow::Result<()> { + let (Some(cfg), Ok(refresh), Ok(folder)) = ( + oauth_config(), + std::env::var("UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN"), + std::env::var("UTOPIA_GOOGLE_DRIVE_TEST_FOLDER"), + ) else { + eprintln!("跳过:未设 UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN / _FOLDER"); + return Ok(()); + }; + let d = Drive::connect(&cfg, &refresh).await?; + assert!(!d.roots().await?.is_empty()); + let listing = d.walk(&[folder]).await?; + assert!( + !listing.files.is_empty(), + "那个文件夹里一个能读的文件都没有" + ); + let first = &listing.files[0]; + assert!(first.modified.is_some(), "modifiedTime 是 doc_time 的来源"); + let c = d.content(first).await?; + assert!(!c.bytes.is_empty()); + Ok(()) +} diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 62f81c69d..3c1808241 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -89,6 +89,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() } Some(SourceKind::Webdav) => sync_webdav(state, &source).await, Some(SourceKind::Notion) => sync_notion(state, &source).await, + Some(SourceKind::GoogleDrive) => sync_google_drive(state, &source).await, // 被动容器:folder / api / memory / upload 没有拉取语义 Some( SourceKind::Folder @@ -1203,6 +1204,114 @@ async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result anyhow::Result { + use crate::google_drive; + + let cfg = google_drive::oauth_config().ok_or_else(|| { + anyhow::anyhow!( + "Google Drive is not configured on this server (UTOPIA_GOOGLE_DRIVE_CLIENT_ID)" + ) + })?; + let refresh = source.config["refresh_token"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + anyhow::anyhow!( + "google_drive source is not connected: connect a Google account in its settings" + ) + })?; + let folders = google_drive::selected_folders(&source.config); + if folders.is_empty() { + anyhow::bail!("google_drive source has no folders selected"); + } + + let drive = google_drive::Drive::connect(&cfg, refresh).await?; + let listing = drive.walk(&folders).await?; + if listing.truncated { + tracing::warn!("文件数或目录深度到达单次上限,其余留给下一次同步"); + } + + let known = google_drive::markers(&state.pool, source.id).await?; + let mut stats = SyncStats::default(); + let mut seen_keys = Vec::with_capacity(listing.files.len()); + let mut seen_ids = Vec::with_capacity(listing.files.len()); + let mut unread: Vec = Vec::new(); + for f in &listing.files { + let key = f.external_key(); + seen_keys.push(key.clone()); + seen_ids.push(f.id.clone()); + let marker = f.marker(); + if known.get(&f.id) == Some(&marker) { + let live = utopia_store::documents::find_by_external_key(&state.pool, source.id, &key) + .await? + .is_some_and(|d| d.deleted_at.is_none()); + if live { + continue; + } + } + // 读不出来的文件跳过,库里已有的原样留着,其余照常同步——同 Notion + let content = match drive.content(f).await { + Ok(c) => c, + Err(error) => { + tracing::warn!(id = %f.id, error = %format!("{error:#}"), "google drive file could not be read, leaving the stored document as it is"); + unread.push(format!("{}: {error:#}", f.name)); + continue; + } + }; + let action = ingest_item( + state, + source.kb_id, + source.id, + &key, + &content.filename, + content.mime, + &content.bytes, + f.modified, + ) + .await?; + google_drive::remember(&state.pool, source.id, &f.id, &marker).await?; + stats.absorb(action); + } + + if listing.truncated || !listing.unreachable.is_empty() { + utopia_store::documents::clear_missing_keys(&state.pool, source.id, &seen_keys).await?; + } else { + utopia_store::documents::reconcile_missing(&state.pool, source.id, &seen_keys).await?; + google_drive::forget_others(&state.pool, source.id, &seen_ids).await?; + } + + if !listing.unreachable.is_empty() { + anyhow::bail!( + "{} file(s) synced; {} selected folder(s) could not be opened (deleted, or no longer shared with the connected account): {}", + stats.total(), + listing.unreachable.len(), + listing.unreachable.join(", ") + ); + } + if !unread.is_empty() { + anyhow::bail!( + "{} file(s) synced; {} could not be read and keep their stored content: {}", + stats.total(), + unread.len(), + unread + .iter() + .take(3) + .map(String::as_str) + .collect::>() + .join("; ") + ); + } + Ok(stats) +} + /// 标题无扩展名时按 mime 补一个,接入解析矩阵的分派。 fn ensure_extension(title: &str, mime: &str) -> String { let has_ext = title diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index 03e98a174..1df1f1105 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -10,6 +10,7 @@ mod error; mod extraction; mod extraction_open; mod github_issues; +mod google_drive; mod governance; mod http_fetch; mod implication; diff --git a/docs/decisions/0068-a-google-drive-source-is-connected-by-a-person.md b/docs/decisions/0068-a-google-drive-source-is-connected-by-a-person.md new file mode 100644 index 000000000..5b954f5f8 --- /dev/null +++ b/docs/decisions/0068-a-google-drive-source-is-connected-by-a-person.md @@ -0,0 +1,44 @@ +# 0068 · A Google Drive source is connected by a person + +- **Status**: Implemented · 2026-10-14 · migration 0112 · tried by hand against a real Drive; the test fixtures are written from the documentation, not recorded (0013 decision 4) · open: revision history, change-feed sync +- **Written**: 2026-10-14 (conventions in the [README](README.md)) +- **Related**: [0013](0013-a-source-should-hand-over-its-history.md) (what a source must pass, fixtures from real responses); [0014](0014-identity-from-the-person-scope-from-the-token.md) (identity comes from the person); migration `0056` (`oidc_flows`, the shape the connection flow copies); [#246](https://github.com/deeplethe/utopia/issues/246) (a credential never reaches a viewer) + +## Problem + +Most teams keep their documents in Google Drive. Today the way in is to export them by hand, or to copy them into a bucket and sync that, which loses each file's identity and its modification time. Drive passes the tests of 0013: `modifiedTime` is the content's own clock, documents are rewritten in place, a file id survives renames and moves, and the knowledge actually lives there. + +Every existing connector is configured with a key an administrator pastes in. Drive has no such key for a person's files. A service account sees only what is shared with it, so every folder would first have to be shared with a robot address, and domain-wide delegation lets one key read everyone's Drive, far more than anyone chose to include. The natural grant is the one a person gives: "read these folders of mine, as me". + +## Decisions + +1. **A person connects the source with OAuth, and it reads Drive as them.** An editor of the knowledge base opens the source's settings and clicks Connect. Google's consent screen asks for `drive.readonly` plus `openid email`, and the refresh token is stored in the source's config under `refresh_token`, which is in `SOURCE_SECRET_KEYS`: sealed at rest, never in a listing, never in a response (#246). The browser never holds a Google token. Even the folder browser asks the server, which reads Drive with the stored grant. `drive.file` was not enough: it covers only files the app created or the person picked one by one, and picking a folder does not include what is inside it. + +2. **The OAuth client belongs to the deployment.** `UTOPIA_GOOGLE_DRIVE_CLIENT_ID`, `_CLIENT_SECRET` and `_REDIRECT_URI` name a Google "web application" client the operator registers. When any of the three is missing, `GET /google-drive/status` says so, the kind cannot be created from the dialog, and a sync fails with that sentence rather than a 401. This client is separate from the SSO client of 0056. The two ask for different scopes, and one deployment may sign people in elsewhere and still read Drive. + +3. **The flow is bound to the person who started it.** `POST …/google-drive/connect` records a single-use `state`, a PKCE verifier, the user and the source in `google_drive_flows` (ten minutes, at most 1000 pending), and returns Google's URL. The callback deletes the row before using it, requires the browser's session to be that same user, and checks again that the user is still an editor. Without the session check, someone could start a flow for their own source and get another person to click through consent, and the other person's Drive would be connected to their source. `prompt=consent` and `access_type=offline` are always sent, so a reconnect after revocation gets a new refresh token. + +4. **The ID token names the account, and that is all it does.** `openid email` exists so the settings can say "connected as bruno@…". The ID token comes straight from Google's token endpoint over TLS, so issuer and audience are checked and the signature is not (OIDC Core 3.1.3.7). An unverified email is not shown. The email never identifies a Utopia account and never links one; that stays as 0056 decided. `connected_at` in the config is what the interface reads as "connected", because the email may legitimately be absent. + +5. **The person chooses folders, and subfolders come with them.** `config.folders` is `[{id, name}]`. The name is only for the interface. The browser lists My Drive, Shared with me and every shared drive. Saving the selection (`PUT …/google-drive/folders`) queues a sync at once. Ids are checked against `[A-Za-z0-9_-]` before they go into a Drive query, which is simpler to trust than escaping. The limit is 50 folders, 16 levels, 2000 files and 32 MB per file, the same order as WebDAV and object storage. Hitting any of them is reported, not silent. + +6. **Google formats are exported to Office formats.** Docs become DOCX, Sheets XLSX and Slides PPTX, and the existing parsers read them with headings, tables and every sheet. Plain text or CSV would have lost the headings that 0039's chunks and 0064's section dates depend on, and CSV keeps only the first sheet. Forms, drawings, shortcuts and sites have no readable body and are skipped. Everything else is downloaded as bytes with `application/octet-stream`, because the parser trusts magic bytes over names, as object storage already does. + +7. **Change is detected from Drive's marker, not from the bytes.** An export of an unchanged Doc is a new zip with new timestamps, so a content hash changes on every run, and every run would re-extract every Doc. `google_drive_files` keeps one marker per file: `md5Checksum` for binary files, and `modifiedTime` for Google formats, which have no checksum. When the marker is unchanged and the document is live, the file is not downloaded at all. + +8. **Retraction follows a complete walk only.** Files not seen are marked "not in source" (`reconcile_missing`) only when every selected folder opened and no limit was hit. A folder that returns 404, because it was deleted or is no longer shared with the connected person, fails the run by name and marks nothing. Not being able to look is not the same as finding it empty. A file that cannot be read (Google refuses exports over 10 MB) keeps its stored document, the rest of the run continues, and the run is reported as failed with the names, as Notion does. + +9. **Disconnecting forgets the grant and asks Google to revoke it.** `DELETE …/google-drive/connection` removes the token from the config first and then calls Google's revoke endpoint. If the revoke fails, that is logged, and the token is already gone from Utopia. Connecting and disconnecting are audited with the account email and never the token. + +## Dead ends + +- **A service account with folders shared to it.** It is robust, and nobody's departure breaks it. But selecting a folder becomes "share it with a robot address and paste an id", and a shared drive must add the robot as a member. It may still be worth having for organisations that will not tie a sync to one person; it would be a second way to fill the same `refresh_token`-shaped slot, not a different source. +- **The Google Picker.** It needs a browser API key and an app id beside the OAuth client, puts a Google token in the browser, and with `drive.file` would still not grant a folder's contents. A server-side folder list costs one endpoint and keeps decision 1 intact. +- **Comparing content hashes**, as every other connector does. That costs a re-extraction of every Google Doc on every run (decision 7). +- **The changes feed** (`changes.list` with a saved page token). It is cheaper on large drives, but it reports changes across the whole Drive and not under a folder, so each change needs an ancestry walk, and moves out of a selected folder are the hard case. The full walk is simple and bounded. Revisit when a deployment hits the 2000-file limit. + +## Open questions + +- **Revision history.** Drive keeps revisions, and 0013 prefers a source that hands its history over. Exporting old revisions of Google formats is possible but expensive; which revisions to sample is its own decision, as it is for Feishu and Confluence. +- **When the person leaves.** The grant is theirs. When their account is suspended the refresh token stops working, and the sync fails with "reconnect the source". That is honest, but someone has to notice it. A `document.needs_reader`-style alert for a dead grant would make it visible. +- **Verification by Google.** `drive.readonly` is a restricted scope. An "internal" OAuth client in a Workspace organisation needs no verification; an "external" one used beyond a hundred test users needs Google's security assessment. The operator decides that, but the setup guide should say so. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index b4d4490dc..0a7d4d1dc 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -112,6 +112,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0052 | [Document content is a read contract over the retained ledger](0052-document-content-is-a-read-contract.md) | Proposed | | | 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Implemented | | | 0065 | [The OCR reader chooses its provider](0065-readers-choose-their-provider.md) | Implemented | | +| 0068 | [A Google Drive source is connected by a person](0068-a-google-drive-source-is-connected-by-a-person.md) | Implemented | | ### [lakehouse-and-actions](../design/lakehouse-and-actions.md) diff --git a/docs/design/sources.md b/docs/design/sources.md index bbc34da19..ec94ff9c2 100644 --- a/docs/design/sources.md +++ b/docs/design/sources.md @@ -1,6 +1,6 @@ # A source hands over its history -Records: [0013] (connectors), [0023] and [0033] (RSS), [0039] (chunks), [0040] (origin and media), +Records: [0013] (connectors), [0068] (Google Drive), [0023] and [0033] (RSS), [0039] (chunks), [0040] (origin and media), [0035] (vector index), [0036] d7 (schema documents), [0005] (sync alerts), [0022] and [0045] d3 (document dates), [0016] E. @@ -14,13 +14,15 @@ source, upload_time, none) is the document's own date; upload time is recorded t document date, so an upload-dated document counts as undated for attestation and ordering [0022, 0045, #714]. -**Connectors.** GitHub issues, Jira, Notion, RSS, WebDAV, S3 / Azure / GCS, and custom push with an +**Connectors.** GitHub issues, Jira, Notion, Google Drive, RSS, WebDAV, S3 / Azure / GCS, and custom push with an `ingest_token`; the kinds come from one `SourceKind` enum that the allowlist, the dispatcher and the web list derive from; credentials never leave the server (`SOURCE_SECRET_KEYS`) and a blank field keeps the stored value; a truncated round says so [0013]. A ticket is one document whose body carries its history as dated declarative sentences and a `## History` list; GitHub fetches events per ticket; timestamps are written to the second [0013]. +**Google Drive.** A person connects it with OAuth and it reads Drive as them; the refresh token is a sealed source secret and the browser never holds a Google token; the flow is bound to the session that started it; the person picks folders and subfolders come along; Docs, Sheets and Slides are exported to DOCX, XLSX and PPTX; change is Drive's marker (`md5Checksum`, or `modifiedTime` for Google formats) because an export's bytes differ every time; "not in source" only after a complete walk [0068]. + **Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054]. **RSS.** Observations are not documents: one table with baseline, candidate and no_source rows; diff --git a/migrations/0112_a_google_drive_source_is_connected_by_a_person.sql b/migrations/0112_a_google_drive_source_is_connected_by_a_person.sql new file mode 100644 index 000000000..7a9b6fe9c --- /dev/null +++ b/migrations/0112_a_google_drive_source_is_connected_by_a_person.sql @@ -0,0 +1,25 @@ +-- Google Drive 来源(0068):由一个人授权连上,按他选的文件夹同步。 +-- +-- 一次连接尝试的临时状态:授权码换令牌之前,state 与 PKCE verifier 得先落着, +-- 还要记下是谁、替哪个来源发起的——回调时这个浏览器里的会话必须还是他。 +-- 短命、一次性:回调一到就删,半途弃单的过期扫走。与 oidc_flows 同一个形状。 +CREATE TABLE google_drive_flows ( + state TEXT PRIMARY KEY, + verifier TEXT NOT NULL, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX google_drive_flows_expiry_idx ON google_drive_flows (expires_at); + +-- 每个文件上一次摄入时 Drive 给的变更标记:二进制文件是 md5Checksum,Google 文档 +-- 没有 md5,用 modifiedTime。**不能靠内容哈希判「没变」**:Docs/Sheets/Slides 每次 +-- 导出成 Office 格式,压缩包里的时间戳都不同,哈希每轮都变,每轮都会重新抽取。 +-- 标记没变、文档还在,就连下载都省了。 +CREATE TABLE google_drive_files ( + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + file_id TEXT NOT NULL, + marker TEXT NOT NULL, + PRIMARY KEY (source_id, file_id) +); diff --git a/web/src/api.ts b/web/src/api.ts index a821a4435..ed4038dc8 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -233,6 +233,11 @@ export interface SourceView { base_url?: string; /** jira_issues:项目 key,如 KAFKA */ project?: string; + /** google_drive:选中的文件夹(名字只给界面看,同步只认 id) */ + folders?: { id: string; name: string }[]; + /** google_drive:连上的 Google 账号;`connected_at` 有值即已连接(令牌从不下发) */ + account_email?: string | null; + connected_at?: string; /** false = 这个来源下的文档只检索、不抽取(schema 文档,0035 决定 7);缺省抽取 */ extract?: boolean; } | null; @@ -2461,6 +2466,32 @@ export const api = { method: "POST", }, ), + googleDriveStatus: () => + request<{ enabled: boolean }>(`/api/v1/google-drive/status`), + googleDriveConnect: (kbId: string, sourceId: string) => + request<{ url: string }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/connect`, + { method: "POST" }, + ), + googleDriveDisconnect: (kbId: string, sourceId: string) => + request<{ ok: boolean }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/connection`, + { method: "DELETE" }, + ), + googleDriveFolders: (kbId: string, sourceId: string, parent?: string) => + request<{ folders: { id: string; name: string }[] }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/folders` + + (parent ? `?parent=${encodeURIComponent(parent)}` : ""), + ), + googleDriveSelect: ( + kbId: string, + sourceId: string, + folders: { id: string; name: string }[], + ) => + request<{ source: SourceView }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/folders`, + { method: "PUT", body: JSON.stringify({ folders }) }, + ), syncSource: (kbId: string, sourceId: string) => request<{ queued: boolean }>( `/api/v1/kbs/${kbId}/sources/${sourceId}/sync`, diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 6676918fb..73e9dc8aa 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -124,6 +124,13 @@ export const en = { "This data source is not granted to this workspace. Ask a deployment admin to grant it in Administration → Data sources.", memory_source_permanent: "The Memory source is part of the knowledge base and stays.", + google_drive_not_configured: + "Google Drive is not set up on this server. An administrator sets UTOPIA_GOOGLE_DRIVE_CLIENT_ID, _CLIENT_SECRET and _REDIRECT_URI.", + google_drive_not_connected: "Connect a Google account first.", + google_drive_unavailable: "Google Drive could not be read.", + google_drive_busy: "Too many Google Drive connections are in progress. Try again in a few minutes.", + google_drive_too_many_folders: "Choose at most 50 folders. Choose a common parent instead.", + google_drive_invalid_folder: "That is not a Google Drive folder id.", source_name_required: "Give this source a name.", bad_cron: "That cron expression could not be parsed.", bad_cron_fields: @@ -636,6 +643,7 @@ export const en = { gcs: "Google Cloud Storage", webdav: "WebDAV", notion: "Notion", + google_drive: "Google Drive", }, sourceKindHints: { folder: "Holds the files you upload: add, drag in and remove them here.", @@ -667,6 +675,10 @@ export const en = { "Syncs the pages a Notion integration can see — share a page with the integration " + "and it appears here. Dated by when the page was last edited, which is the page's " + "own clock rather than ours.", + google_drive: + "Syncs the folders you choose from your Google Drive, shared drives included. " + + "**You connect with your own Google account**, read-only; Docs, Sheets and Slides " + + "arrive as Word, Excel and PowerPoint files. Dated by each file's last modification.", api: "External systems push JSON documents here, authenticated with this source's own token.", statements: "Your system pushes statements already in the extraction contract; no model reads them. Send events, not the state of a table.", @@ -743,6 +755,31 @@ export const en = { davPassField: "Password", notionTokenField: "Internal integration token", notionQueryField: "Search term (optional — empty takes every shared page)", + googleDrive: { + createHint: "After creating the source, connect a Google account and choose the folders.", + notConfigured: + "Google Drive is not set up on this server. An administrator sets UTOPIA_GOOGLE_DRIVE_CLIENT_ID, _CLIENT_SECRET and _REDIRECT_URI.", + account: "Google account", + connect: "Connect Google Drive", + reconnect: "Reconnect", + disconnect: "Disconnect", + waiting: "Finish in the Google window…", + connectedAs: (email: string) => `Connected as ${email}`, + connectedNoEmail: "Connected", + notConnected: "Not connected. Syncing reads Drive with the permissions of whoever connects it.", + folders: "Folders to sync", + foldersHint: "Subfolders are included. Files outside these folders are never read.", + myDrive: "My Drive", + sharedWithMe: "Shared with me", + top: "Google Drive", + empty: "No folders here.", + open: "Open", + noneSelected: "No folders chosen yet.", + remove: "Remove", + saveFolders: "Save folders and sync", + saved: "Folders saved. A sync has started.", + disconnected: "Google Drive disconnected.", + }, tokenField: "GitHub token (optional, never shown again)", includePullRequests: "Treat pull requests as tickets too", interval: "Sync schedule", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index cc493763a..b80e6848f 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -113,6 +113,13 @@ export const zh: Strings = { source_not_granted: "这个数据源没有授权给本工作区。请部署管理员在「管理 → 数据源」里授权。", memory_source_permanent: "「记忆」是知识库自带的来源,会一直在。", + google_drive_not_configured: + "这台服务器没有配置 Google Drive。需要管理员设置 UTOPIA_GOOGLE_DRIVE_CLIENT_ID、_CLIENT_SECRET 与 _REDIRECT_URI。", + google_drive_not_connected: "先连接一个 Google 账号。", + google_drive_unavailable: "读不了 Google Drive。", + google_drive_busy: "同时进行的 Google Drive 连接太多,过几分钟再试。", + google_drive_too_many_folders: "最多选 50 个文件夹,可以改选它们共同的上级。", + google_drive_invalid_folder: "这不是一个 Google Drive 文件夹 id。", source_name_required: "给这个来源起个名字。", bad_cron: "这个 cron 表达式解析不了。", bad_cron_fields: "cron 表达式有五段:分 时 日 月 周。", @@ -577,6 +584,7 @@ export const zh: Strings = { gcs: "Google Cloud Storage", webdav: "WebDAV", notion: "Notion", + google_drive: "Google Drive", }, sourceKindHints: { folder: "可在文件夹中管理上传的文件:上传、拖入、删除都在这里。", @@ -605,6 +613,10 @@ export const zh: Strings = { notion: "同步一个 Notion 集成能看见的页面——把页面分享给集成,它就会出现在这里。" + "日期取页面最后一次编辑的时刻,那是页面自己的时钟,不是我们抓它的时刻。", + google_drive: + "同步你在 Google Drive 里选的文件夹,共享云端硬盘也算。" + + "**用你自己的 Google 账号连接**,只读;Docs、Sheets、Slides 以 Word、Excel、PowerPoint " + + "文件进来。日期取每个文件最后一次修改的时刻。", api: "外部系统把 JSON 文档推送到这里,用这个来源自己的令牌认证。", statements: "外部系统把已经是抽取契约形状的陈述推送到这里,不经模型;发事件,别发整张表的状态。", @@ -678,6 +690,31 @@ export const zh: Strings = { davPassField: "密码", notionTokenField: "内部集成令牌", notionQueryField: "搜索词(可选——留空则取全部已分享页面)", + googleDrive: { + createHint: "建好来源之后,连接一个 Google 账号并选择文件夹。", + notConfigured: + "这台服务器没有配置 Google Drive。需要管理员设置 UTOPIA_GOOGLE_DRIVE_CLIENT_ID、_CLIENT_SECRET 与 _REDIRECT_URI。", + account: "Google 账号", + connect: "连接 Google Drive", + reconnect: "重新连接", + disconnect: "断开", + waiting: "请在 Google 的窗口里完成……", + connectedAs: (email: string) => `已连接:${email}`, + connectedNoEmail: "已连接", + notConnected: "尚未连接。同步以连接者的权限读取 Drive。", + folders: "要同步的文件夹", + foldersHint: "包含子文件夹。这些文件夹之外的文件一概不读。", + myDrive: "我的云端硬盘", + sharedWithMe: "与我共享", + top: "Google Drive", + empty: "这里没有文件夹。", + open: "打开", + noneSelected: "还没有选文件夹。", + remove: "移除", + saveFolders: "保存并同步", + saved: "文件夹已保存,同步已开始。", + disconnected: "Google Drive 已断开。", + }, tokenField: "GitHub 令牌(可选,不再显示)", includePullRequests: "把 PR 也当工单收进来", interval: "同步计划", diff --git a/web/src/pages/GoogleDrivePanel.tsx b/web/src/pages/GoogleDrivePanel.tsx new file mode 100644 index 000000000..265a0fcc0 --- /dev/null +++ b/web/src/pages/GoogleDrivePanel.tsx @@ -0,0 +1,236 @@ +/* Google Drive 来源的连接与选文件夹(0068),住在来源设置弹窗里。 + * + * 连接走弹窗而不是整页跳转:设置弹窗里的东西(改了一半的名字、日程)不会丢。 + * 回调页不带脚本,不 postMessage 回来——这边在弹窗开着的时候轮询来源列表, + * 看见 `connected_at` 变了就关掉它。人自己关了弹窗,轮询也就停了。 + * + * 浏览器从头到尾拿不到 Google 的令牌:文件夹列表是服务器以连接者的身份去取的。 */ +import { useEffect, useRef, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ChevronRight, Folder, X } from "lucide-react"; +import { api, type SourceView } from "../api"; +import { S } from "../i18n"; +import { toast } from "../toast"; +import { Button, Checkbox, Chip, IconButton, LinkButton } from "../ui"; + +type Entry = { id: string; name: string }; + +/** 服务器给的两个固定入口按 id 换成界面语言;其余是 Drive 里的真名 */ +function label(e: Entry): string { + if (e.id === "root") return S.library.googleDrive.myDrive; + if (e.id === "shared:with-me") return S.library.googleDrive.sharedWithMe; + return e.name; +} + +export function GoogleDrivePanel({ kbId, source }: { kbId: string; source: SourceView }) { + const G = S.library.googleDrive; + const queryClient = useQueryClient(); + const cfg = source.config ?? {}; + const connectedAt = cfg.connected_at ?? null; + const [selected, setSelected] = useState(cfg.folders ?? []); + // 浏览路径:空 = 第一层(我的云端硬盘、与我共享、共享云端硬盘) + const [path, setPath] = useState([]); + const [waiting, setWaiting] = useState(false); + const popup = useRef(null); + const startedFrom = useRef(null); + + const status = useQuery({ + queryKey: ["google-drive-status"], + queryFn: api.googleDriveStatus, + }); + const parent = path.at(-1)?.id; + const folders = useQuery({ + queryKey: ["google-drive-folders", source.id, connectedAt, parent ?? ""], + queryFn: () => api.googleDriveFolders(kbId, source.id, parent), + enabled: Boolean(connectedAt), + retry: false, + }); + + // 弹窗开着时轮询:连上了(connected_at 变了)就关弹窗;人关了弹窗就停 + useEffect(() => { + if (!waiting) return; + const timer = window.setInterval(() => { + if (popup.current?.closed) { + setWaiting(false); + return; + } + queryClient.invalidateQueries({ queryKey: ["sources", kbId] }); + }, 1500); + return () => window.clearInterval(timer); + }, [waiting, kbId, queryClient]); + useEffect(() => { + if (waiting && connectedAt && connectedAt !== startedFrom.current) { + popup.current?.close(); + setWaiting(false); + setPath([]); + } + }, [waiting, connectedAt]); + + const connect = useMutation({ + mutationFn: async () => { + // 先同步地开一个空弹窗,再填地址:等请求回来再 window.open 会被拦 + const w = window.open("", "utopia-google-drive", "width=520,height=680"); + try { + const { url } = await api.googleDriveConnect(kbId, source.id); + if (w) w.location.href = url; + else window.location.href = url; + popup.current = w; + startedFrom.current = connectedAt; + setWaiting(true); + } catch (e) { + w?.close(); + throw e; + } + }, + onError: (e) => toast.error((e as Error).message), + }); + + const disconnect = useMutation({ + mutationFn: () => api.googleDriveDisconnect(kbId, source.id), + onSuccess: () => { + toast.success(G.disconnected); + queryClient.invalidateQueries({ queryKey: ["sources", kbId] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const save = useMutation({ + mutationFn: () => api.googleDriveSelect(kbId, source.id, selected), + onSuccess: () => { + toast.success(G.saved); + queryClient.invalidateQueries({ queryKey: ["sources", kbId] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const isSelected = (id: string) => selected.some((f) => f.id === id); + const toggle = (e: Entry) => + setSelected((cur) => + cur.some((f) => f.id === e.id) + ? cur.filter((f) => f.id !== e.id) + : [...cur, { id: e.id, name: label(e) }], + ); + const unchanged = + JSON.stringify(selected.map((f) => f.id)) === + JSON.stringify((cfg.folders ?? []).map((f) => f.id)); + + if (status.data && !status.data.enabled) { + return

{G.notConfigured}

; + } + + return ( +
+
{G.account}
+
+ + {waiting + ? G.waiting + : connectedAt + ? cfg.account_email + ? G.connectedAs(cfg.account_email) + : G.connectedNoEmail + : G.notConnected} + +
+ {connectedAt && ( + + )} + +
+
+ + {connectedAt && ( + <> +
{G.folders}
+

{G.foldersHint}

+ + {/* 面包屑:点哪一级回哪一级 */} +
+ setPath([])}>{G.top} + {path.map((p, i) => ( + + + setPath(path.slice(0, i + 1))}> + {label(p)} + + + ))} +
+
+ {folders.isPending ? ( +
…
+ ) : folders.isError ? ( +
{(folders.error as Error).message}
+ ) : folders.data.folders.length === 0 ? ( +
{G.empty}
+ ) : ( + folders.data.folders.map((f) => ( +
+ {/* 「与我共享」只是一个列表,不是一个能同步的文件夹 */} + {f.id === "shared:with-me" ? ( + + + {label(f)} + + ) : ( + toggle(f)} + label={label(f)} + /> + )} + setPath([...path, f])} + > + + +
+ )) + )} +
+ + {selected.length === 0 ? ( +

{G.noneSelected}

+ ) : ( +
+ {/* 点一下就从选中里拿掉 */} + {selected.map((f) => ( + toggle(f)}> + {f.name} + + + ))} +
+ )} + + + )} +
+ ); +} diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx index 31d815f5c..e7a430251 100644 --- a/web/src/pages/Library.tsx +++ b/web/src/pages/Library.tsx @@ -49,6 +49,7 @@ import { SYNCING_KINDS, type LibrarySelection, } from "./SourcesRail"; +import { GoogleDrivePanel } from "./GoogleDrivePanel"; const PAGE_SIZE = 15; @@ -799,8 +800,10 @@ export function Library() { onDone={(id, pushKind) => { setAdding(false); if (id) setSelection(id); + // Google Drive 建好直接打开设置:连接账号、选文件夹都在那里 + if (id && pushKind === "google_drive") setEditing(true); // 推送来源建好直接打开密钥弹窗(onboarding:端点 + 密钥一步拿全) - if (id && pushKind) setTokenReveal({ sourceId: id, kind: pushKind }); + else if (id && pushKind) setTokenReveal({ sourceId: id, kind: pushKind }); invalidate(); }} /> @@ -1328,7 +1331,8 @@ function SourceModal({ }: { kbId: string; /** isApi=true 时父级紧接着打开密钥弹窗 */ - /** 第二个参数:建的是推送来源时它的 kind(`api` / `statements`),别的不给 */ + /** 第二个参数:建的是推送来源时它的 kind(`api` / `statements`)、或 `google_drive` + * (建好还得连账号、选文件夹),别的不给 */ onDone: (id?: string, pushKind?: string) => void; }) { const [kind, setKind] = useState("folder"); @@ -1375,7 +1379,14 @@ function SourceModal({ kind === "azure_blob" || kind === "gcs" || kind === "webdav" || - kind === "notion"; + kind === "notion" || + kind === "google_drive"; + // 部署没配 OAuth 客户端时建出来也连不上,不让建 + const driveStatus = useQuery({ + queryKey: ["google-drive-status"], + queryFn: api.googleDriveStatus, + enabled: kind === "google_drive", + }); const create = useMutation({ mutationFn: () => { @@ -1454,7 +1465,10 @@ function SourceModal({ }); }, onSuccess: (data) => - onDone(data.source.id, kind === "api" || kind === "statements" ? kind : undefined), + onDone( + data.source.id, + kind === "api" || kind === "statements" || kind === "google_drive" ? kind : undefined, + ), }); const valid = @@ -1467,6 +1481,8 @@ function SourceModal({ ? endpoint.trim() : kind === "notion" ? notionToken.trim() + : kind === "google_drive" + ? driveStatus.data?.enabled : kind === "webdav" ? davUrl.trim() : kind === "s3" || kind === "azure_blob" || kind === "gcs" @@ -1779,6 +1795,13 @@ function SourceModal({ )} )} + {kind === "google_drive" && ( +

+ {driveStatus.data && !driveStatus.data.enabled + ? S.library.googleDrive.notConfigured + : S.library.googleDrive.createHint} +

+ )} {kind === "notion" && ( <> {field( @@ -2070,6 +2093,8 @@ function SourceEditModal({ )} + {kind === "google_drive" && } + {ingestChanged && (

{S.library.editKeepNote} diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsx index 9929bb0b0..a5558b263 100644 --- a/web/src/pages/SourcesRail.tsx +++ b/web/src/pages/SourcesRail.tsx @@ -78,6 +78,7 @@ export const KIND_ICON: Record = { gcs: Cloud, webdav: FolderOpen, notion: Notebook, + google_drive: HardDrive, memory: Brain, upload: Upload, }; @@ -94,6 +95,7 @@ export const SYNCING_KINDS = new Set([ "gcs", "webdav", "notion", + "google_drive", ]); export const SYNC_DOT: Record = { diff --git a/web/src/sourceKinds.ts b/web/src/sourceKinds.ts index c6e30af41..60fbc285a 100644 --- a/web/src/sourceKinds.ts +++ b/web/src/sourceKinds.ts @@ -19,6 +19,7 @@ export const CREATABLE_SOURCE_KINDS = [ "gcs", "webdav", "notion", + "google_drive", "api", "statements", "custom",