diff --git a/package.json b/package.json index 7cc505e..bf77a69 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "coderelay", - "version": "0.1.13", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/sidecars/coderelay-proxy/UPSTREAM.md b/sidecars/coderelay-proxy/UPSTREAM.md index ca528eb..7b6f743 100644 --- a/sidecars/coderelay-proxy/UPSTREAM.md +++ b/sidecars/coderelay-proxy/UPSTREAM.md @@ -4,8 +4,8 @@ original MIT license and copyright notice are retained in `LICENSE`. The outer `coderelay-proxy` package contains the CodeBuddy CN relay, request -policy, account selection, model synchronization, vision routing and the -newline-delimited JSON lifecycle event protocol consumed by the Tauri host. +policy, account selection, model synchronization and the newline-delimited JSON +lifecycle event protocol consumed by the Tauri host. The sidecar accepts: diff --git a/sidecars/coderelay-proxy/codebuddy.go b/sidecars/coderelay-proxy/codebuddy.go index df5275f..f70c9f4 100644 --- a/sidecars/coderelay-proxy/codebuddy.go +++ b/sidecars/coderelay-proxy/codebuddy.go @@ -34,19 +34,6 @@ func equalStringSlices(a, b []string) bool { return true } -// visionProxyEnabled reports whether the vision-proxy layer is active (mode is -// "routing" or "preprocess"). Used by /v1/models to report `input_modalities: -// ["text","image"]` for non-vision models that the proxy will transparently -// handle — otherwise clients (e.g. Cursor) filter image inputs client-side and -// the image never reaches the relay. -func (m *manifest) visionProxyEnabled() bool { - if m == nil { - return false - } - mode := strings.ToLower(strings.TrimSpace(m.VisionMode)) - return mode == "routing" || mode == "preprocess" || mode == "agentic" -} - // modelIDs returns a snapshot of the current model ID list. It is safe for // concurrent use with setModelIDs. func (m *manifest) modelIDs() []string { diff --git a/sidecars/coderelay-proxy/codebuddy_model_cache.go b/sidecars/coderelay-proxy/codebuddy_model_cache.go index cdcf5ef..ed98660 100644 --- a/sidecars/coderelay-proxy/codebuddy_model_cache.go +++ b/sidecars/coderelay-proxy/codebuddy_model_cache.go @@ -19,7 +19,7 @@ const codebuddyModelCacheFilename = "codebuddy_models_cache.json" // codebuddyModelCache is the persisted form of the CodeBuddy model catalog. // Models are stored in full (not just IDs) so capability fields such as // SupportsImages / ContextLength / MaxCompletionTokens survive a reload, which -// vision-proxy routing and max_tokens clamping depend on. +// max_tokens clamping and image-capability checks depend on. type codebuddyModelCache struct { Version int `json:"version"` SyncedAt string `json:"syncedAt,omitempty"` diff --git a/sidecars/coderelay-proxy/manifest_policy.go b/sidecars/coderelay-proxy/manifest_policy.go index bf31a71..d4b0036 100644 --- a/sidecars/coderelay-proxy/manifest_policy.go +++ b/sidecars/coderelay-proxy/manifest_policy.go @@ -78,10 +78,6 @@ var ( streamIdleTimeout = 60 * time.Second imageStreamOpenTimeout = 10 * time.Second imageStreamIdleTimeout = 60 * time.Second - // visionAgenticStreamIdleTimeout covers the codebuddy vision sub-agent - // loop: up to several rounds of text-model + vision-model upstream calls - // (each 10-30s), so the relay idle watchdog does not trip mid-loop. - visionAgenticStreamIdleTimeout = 300 * time.Second ) type accountModelRule struct { @@ -102,8 +98,6 @@ type manifest struct { ImmediateSSEResponse bool `json:"immediateSseResponse"` MaxConcurrentImageRequests int `json:"maxConcurrentImageRequests"` DebugLogs *bool `json:"debugLogs,omitempty"` - VisionMode string `json:"visionMode"` - VisionModel string `json:"visionModel"` apiKeyByValue map[string]*apiKeySpec accountByID map[string]*accountSpec @@ -397,7 +391,6 @@ type requestDiagnosticPayload struct { Path string `json:"path,omitempty"` RequestKind string `json:"requestKind,omitempty"` Model string `json:"model,omitempty"` - VisionSubagent bool `json:"visionSubagent,omitempty"` APIKeyID string `json:"apiKeyId,omitempty"` APIKeyLabel string `json:"apiKeyLabel,omitempty"` Transport string `json:"transport,omitempty"` @@ -979,7 +972,7 @@ func (p *requestPolicy) middleware() gin.HandlerFunc { if isCodexClientModelsRequest(c.Request) { c.JSON(http.StatusOK, buildCodexClientModelsResponse(models, spec, contextWindowsForAPIKey(p.manifest, spec))) } else { - c.JSON(http.StatusOK, buildModelsResponse(models, p.manifest.visionProxyEnabled())) + c.JSON(http.StatusOK, buildModelsResponse(models)) } c.Abort() return @@ -1138,13 +1131,12 @@ func (p *requestPolicy) emitRequestCompleted(c *gin.Context, requestID string, s RequestID: requestID, Method: c.Request.Method, Path: requestPath(c.Request), - RequestKind: requestKind, - Model: model, - VisionSubagent: internallogging.GetVisionSubagent(c.Request.Context()), - APIKeyID: stringFromAPIKey(spec, "id"), - APIKeyLabel: stringFromAPIKey(spec, "label"), - Transport: diagnosticTransport(c.Request), - Status: status, + RequestKind: requestKind, + Model: model, + APIKeyID: stringFromAPIKey(spec, "id"), + APIKeyLabel: stringFromAPIKey(spec, "label"), + Transport: diagnosticTransport(c.Request), + Status: status, LatencyMS: latencyMS, CompletedAtMS: completedAtMS, Aborted: c.IsAborted(), @@ -1248,15 +1240,16 @@ func isCodexClientModelsRequest(r *http.Request) bool { // carries `input_modalities` so clients (e.g. Cursor) can detect vision-capable // models instead of defaulting to text-only and filtering images client-side. // -// A model reports image capability when either: -// 1. the backend natively supports images for it (registry.CodebuddyModelSupportsImages), or -// 2. the vision-proxy layer is active and will transparently describe/handle -// images for it (preprocess/routing via hy3-preview). -func buildModelsResponse(models []string, visionProxyEnabled bool) gin.H { +// A model reports image capability when the backend natively supports images +// for it (registry.CodebuddyModelSupportsImages, backed by the online catalog +// plus the measured capability overrides in the registry). Models without +// native image support report text-only so clients do not send images the +// upstream would reject. +func buildModelsResponse(models []string) gin.H { data := make([]gin.H, 0, len(models)) for _, model := range models { modalities := []any{"text"} - if internalregistry.CodebuddyModelSupportsImages(model) || visionProxyEnabled { + if internalregistry.CodebuddyModelSupportsImages(model) { modalities = []any{"text", "image"} } data = append(data, gin.H{ @@ -1799,41 +1792,6 @@ func canonicalModelForClientModel(m *manifest, spec *apiKeySpec, model string) s return resolveSupportedModelAlias(m, withoutPrefix) } -// requestHasVisionInput reports whether a request body carries image input -// (OpenAI image_url parts or Responses input_image parts). It is used by the -// stream watchdog to extend the idle timeout for vision sub-agent loops. -func requestHasVisionInput(body []byte) bool { - if len(body) == 0 || !json.Valid(body) { - return false - } - var payload any - if err := json.Unmarshal(body, &payload); err != nil { - return false - } - return valueHasVisionInput(payload) -} - -func valueHasVisionInput(value any) bool { - switch typed := value.(type) { - case map[string]any: - if typ, _ := typed["type"].(string); strings.EqualFold(strings.TrimSpace(typ), "input_image") || strings.EqualFold(strings.TrimSpace(typ), "image_url") { - return true - } - for _, child := range typed { - if valueHasVisionInput(child) { - return true - } - } - case []any: - for _, child := range typed { - if valueHasVisionInput(child) { - return true - } - } - } - return false -} - func stripModelPrefix(model string, spec *apiKeySpec) string { trimmed := strings.TrimSpace(model) if spec == nil || strings.TrimSpace(spec.ModelPrefix) == "" { diff --git a/sidecars/coderelay-proxy/manifest_policy_test.go b/sidecars/coderelay-proxy/manifest_policy_test.go index b7ef589..6c62150 100644 --- a/sidecars/coderelay-proxy/manifest_policy_test.go +++ b/sidecars/coderelay-proxy/manifest_policy_test.go @@ -255,7 +255,7 @@ func TestCodexClientModelsResponseShape(t *testing.T) { } func TestCodebuddyModelsResponseReportsInputModalities(t *testing.T) { - models := []string{"hy3", "hy3-preview", "deepseek-v4-pro", "deepseek-v4-flash", "hunyuan-2.0-instruct"} + models := []string{"hy3", "hy3-preview", "deepseek-v4-pro", "deepseek-v4-flash", "hunyuan-2.0-instruct", "glm-5v-turbo"} modalitiesFor := func(response gin.H, modelID string) []any { data, ok := response["data"].([]gin.H) @@ -281,22 +281,20 @@ func TestCodebuddyModelsResponseReportsInputModalities(t *testing.T) { } } - // 情况1：vision-proxy 启用（默认 preprocess）——所有模型都报 image， - // 因为反代能透明处理纯文本模型的图片（描述后回填）。 - enabled := buildModelsResponse(models, true) - for _, m := range models { - assertModalities(enabled, m, []any{"text", "image"}) - } - - // 情况2：vision-proxy 关闭——仅后端原生支持视觉的模型报 image。 - // hy3/hy3-preview（app.asar supportsImages）报 image； - // deepseek（已移出白名单，后端返回拒绝）与 hunyuan（假视觉）报 text。 - disabled := buildModelsResponse(models, false) - assertModalities(disabled, "hy3", []any{"text", "image"}) - assertModalities(disabled, "hy3-preview", []any{"text", "image"}) - assertModalities(disabled, "deepseek-v4-pro", []any{"text"}) - assertModalities(disabled, "deepseek-v4-flash", []any{"text"}) - assertModalities(disabled, "hunyuan-2.0-instruct", []any{"text"}) + // 图片能力判定 = 在线清单 supportsImages + registry 里的实测校正表 + // （排除 glm-5v-turbo，补入 glm-5.1 / deepseek-v3-2-volc）。 + // 单测环境下 codebuddySynced 为空、回退静态 models.json： + // hy3/hy3-preview 在静态目录中含 supportsImages，报 image； + // deepseek（静态目录未标记图片能力）与 hunyuan（无图片能力）报 text； + // glm-5v-turbo 由校正表强制排除，即使目录误标也报 text。 + // 注意：运行态同步到在线清单后，deepseek-v4.x 会被标记为图片能力 → 报 image。 + response := buildModelsResponse(models) + assertModalities(response, "hy3", []any{"text", "image"}) + assertModalities(response, "hy3-preview", []any{"text", "image"}) + assertModalities(response, "deepseek-v4-pro", []any{"text"}) + assertModalities(response, "deepseek-v4-flash", []any{"text"}) + assertModalities(response, "hunyuan-2.0-instruct", []any{"text"}) + assertModalities(response, "glm-5v-turbo", []any{"text"}) } func TestCodexClientModelsResponsePreserves56Template(t *testing.T) { @@ -446,23 +444,6 @@ func TestCodexClientModelsResponseDoesNotInjectFastMode(t *testing.T) { } } -func TestRequestVisionDetectionIgnoresToolSchemaFieldNames(t *testing.T) { - body := []byte(`{ - "model":"deepseek-v4-pro", - "tools":[{ - "type":"function", - "name":"inspect_url", - "parameters":{ - "type":"object", - "properties":{"image_url":{"type":"string"}} - } - }] - }`) - if requestHasVisionInput(body) { - t.Fatal("tool schema field names must not be treated as image input") - } -} - func TestCodexClientModelsResponseEnablesWebsocketsWhenConfigured(t *testing.T) { response := buildCodexClientModelsResponse([]string{"gpt-5.6-sol"}, &apiKeySpec{ ResponsesWebsockets: true, diff --git a/sidecars/coderelay-proxy/relay_server.go b/sidecars/coderelay-proxy/relay_server.go index 1c9351b..2337a89 100644 --- a/sidecars/coderelay-proxy/relay_server.go +++ b/sidecars/coderelay-proxy/relay_server.go @@ -574,7 +574,7 @@ func (s *relayServer) handleModels(c *gin.Context) { c.JSON(http.StatusOK, buildCodexClientModelsResponse(models, spec, contextWindowsForAPIKey(s.manifest, spec))) return } - c.JSON(http.StatusOK, buildModelsResponse(models, s.manifest.visionProxyEnabled())) + c.JSON(http.StatusOK, buildModelsResponse(models)) } func (s *relayServer) handleResponses(c *gin.Context) { diff --git a/sidecars/coderelay-proxy/stream_protocol.go b/sidecars/coderelay-proxy/stream_protocol.go index 399fd29..78facfc 100644 --- a/sidecars/coderelay-proxy/stream_protocol.go +++ b/sidecars/coderelay-proxy/stream_protocol.go @@ -62,15 +62,6 @@ func (s *relayServer) streamTimeoutsForRequest(r *http.Request, body []byte, mod profile.open = durationFromConfigMillis(s.cfg.Streaming.StreamOpenTimeoutMS, profile.open) profile.idle = durationFromConfigMillis(s.cfg.Streaming.StreamIdleTimeoutMS, profile.idle) } - // Requests carrying vision input are handled by the codebuddy vision - // sub-agent loop, which performs multiple rounds of upstream calls before - // emitting content. Give them a much longer idle timeout so the relay - // watchdog does not cancel mid-loop. - if requestHasVisionInput(body) { - if profile.idle < visionAgenticStreamIdleTimeout { - profile.idle = visionAgenticStreamIdleTimeout - } - } if !isImageGenerationRequest(r, body, model) { return profile } @@ -158,12 +149,6 @@ func relayContext(c *gin.Context) context.Context { if c == nil || c.Request == nil { return context.Background() } - // Attach the vision sub-agent holder to the request context so the - // downstream executor's SetVisionSubagent and the request-completed - // diagnostic's GetVisionSubagent observe the same holder instance. Without - // this, both sides would hold distinct (or missing) holders and the - // request_completed event's visionSubagent flag would always be false. - c.Request = c.Request.WithContext(internallogging.WithVisionSubagentHolder(c.Request.Context())) endpoint := c.Request.Method if c.Request.URL != nil { endpoint += " " + c.Request.URL.Path diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/config/sdk_config.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/config/sdk_config.go index 80a1df7..62360cb 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/config/sdk_config.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/config/sdk_config.go @@ -4,8 +4,6 @@ // debug settings, proxy configuration, and API keys. package config -import "strings" - // SDKConfig represents the application's configuration, loaded from a YAML file. type SDKConfig struct { EnableGeminiCLIEndpoint bool `yaml:"enable-gemini-cli-endpoint,omitempty" json:"enable-gemini-cli-endpoint,omitempty"` @@ -64,12 +62,6 @@ type SDKConfig struct { // NonStreamKeepAliveInterval controls how often blank lines are emitted for non-streaming responses. // <= 0 disables keep-alives. Value is in seconds. NonStreamKeepAliveInterval int `yaml:"nonstream-keepalive-interval,omitempty" json:"nonstream-keepalive-interval,omitempty"` - - // CodebuddyVision configures the CodeBuddy vision-proxy layer. When a chat - // request carries image input for a model that does not natively support - // images, the proxy either swaps the model to a vision model (routing) or - // converts the images to text descriptions first (preprocess). - CodebuddyVision CodebuddyVisionConfig `yaml:"codebuddy-vision" json:"codebuddy-vision"` } // ClaudeCodeConfig configures Claude Code compatibility behavior. @@ -97,72 +89,4 @@ type StreamingConfig struct { BootstrapRetryMaxDelayMS int `yaml:"bootstrap-retry-max-delay-ms,omitempty" json:"bootstrap-retry-max-delay-ms,omitempty"` } -// CodebuddyVisionConfig controls the CodeBuddy vision-proxy layer. -// -// The Tencent CodeBuddy backend accepts image input on a per-model basis. Some -// text-only models (e.g. hunyuan-2.0-instruct) silently ignore images and reply -// with "this model does not support image input" instead of an error. When -// enabled, the proxy detects image input and handles it for non-vision models. -type CodebuddyVisionConfig struct { - // Mode selects the strategy: - // - "off" (default): disabled; images pass through unchanged. - // - "routing": swap the request model to Model for non-vision models. - // - "preprocess": describe images with Model first, then continue with the - // original model. - // - "agentic": inject an inspect_image tool and run a server-side tool-calling - // loop so the text-only model can autonomously query the vision model - // multiple times during reasoning. - // Any other value falls back to "off". - Mode string `yaml:"mode" json:"mode"` - - // Model is the vision model used as the routing target / preprocess engine. - // Default "hy4-preview". - Model string `yaml:"model" json:"model"` - - // PreprocessPrompt overrides the user-visible prompt sent to the vision model - // in preprocess mode. Empty uses a built-in default. - PreprocessPrompt string `yaml:"preprocess-prompt" json:"preprocess-prompt"` - - // MaxToolRounds caps the number of inspect_image tool-call iterations in - // agentic mode. Non-positive falls back to a default of 3. - MaxToolRounds int `yaml:"max-tool-rounds" json:"max-tool-rounds"` -} - -// VisionMode constants for CodebuddyVisionConfig.Mode. -const ( - CodebuddyVisionModeOff = "off" - CodebuddyVisionModeRouting = "routing" - CodebuddyVisionModePreprocess = "preprocess" - CodebuddyVisionModeAgentic = "agentic" -) - -// NormalizedVisionMode returns the effective mode, mapping unknown values to "off". -func (c CodebuddyVisionConfig) NormalizedVisionMode() string { - switch strings.ToLower(strings.TrimSpace(c.Mode)) { - case CodebuddyVisionModeRouting: - return CodebuddyVisionModeRouting - case CodebuddyVisionModePreprocess: - return CodebuddyVisionModePreprocess - case CodebuddyVisionModeAgentic: - return CodebuddyVisionModeAgentic - default: - return CodebuddyVisionModeOff - } -} - -// MaxVisionToolRounds returns the effective agentic iteration cap (default 3). -func (c CodebuddyVisionConfig) MaxVisionToolRounds() int { - if c.MaxToolRounds > 0 { - return c.MaxToolRounds - } - return 3 -} -// VisionModel returns the configured vision model, defaulting to "hy4-preview". -func (c CodebuddyVisionConfig) VisionModel() string { - model := strings.TrimSpace(c.Model) - if model == "" { - return "hy4-preview" - } - return model -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/logging/requestmeta.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/logging/requestmeta.go index 14681b2..576bf5d 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/logging/requestmeta.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/logging/requestmeta.go @@ -19,16 +19,10 @@ type ClientRequestMetadata struct { UserAgent string } -type visionSubagentKey struct{} - type responseStatusHolder struct { status atomic.Int32 } -type visionSubagentHolder struct { - flag atomic.Bool -} - type responseHeadersHolder struct { mu sync.RWMutex headers http.Header @@ -148,43 +142,3 @@ func cloneHTTPHeader(src http.Header) http.Header { } return dst } - -// WithVisionSubagentHolder attaches a pointer holder so that a downstream -// executor can flag the request as handled by the pure-text vision sub-agent -// loop, which the upstream request policy later reads when emitting request -// diagnostics. -func WithVisionSubagentHolder(ctx context.Context) context.Context { - if ctx == nil { - ctx = context.Background() - } - if holder, ok := ctx.Value(visionSubagentKey{}).(*visionSubagentHolder); ok && holder != nil { - return ctx - } - return context.WithValue(ctx, visionSubagentKey{}, &visionSubagentHolder{}) -} - -// SetVisionSubagent flags (or clears) the vision sub-agent marker on the -// request held by ctx. Safe to call from downstream executors. -func SetVisionSubagent(ctx context.Context, value bool) { - if ctx == nil { - return - } - holder, ok := ctx.Value(visionSubagentKey{}).(*visionSubagentHolder) - if !ok || holder == nil { - return - } - holder.flag.Store(value) -} - -// GetVisionSubagent reports whether the request was flagged as handled by the -// vision sub-agent loop. -func GetVisionSubagent(ctx context.Context) bool { - if ctx == nil { - return false - } - holder, ok := ctx.Value(visionSubagentKey{}).(*visionSubagentHolder) - if !ok || holder == nil { - return false - } - return holder.flag.Load() -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/redisqueue/plugin.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/redisqueue/plugin.go index a27cd07..d91c8a2 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/redisqueue/plugin.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/redisqueue/plugin.go @@ -114,7 +114,6 @@ func (p *usageQueuePlugin) HandleUsage(ctx context.Context, record coreusage.Rec APIKey: apiKey, RequestID: requestID, ReasoningEffort: reasoningEffort, - VisionSubagent: record.VisionSubagent, ServiceTier: serviceTier, ResponseServiceTier: responseServiceTier, }) @@ -137,7 +136,6 @@ type queuedUsageDetail struct { APIKey string `json:"api_key"` RequestID string `json:"request_id"` ReasoningEffort string `json:"reasoning_effort"` - VisionSubagent bool `json:"vision_subagent"` ServiceTier string `json:"service_tier"` ResponseServiceTier string `json:"response_service_tier,omitempty"` } diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/codebuddy_model_vision_test.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/codebuddy_model_vision_test.go index 69ee6b8..039d9b7 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/codebuddy_model_vision_test.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/codebuddy_model_vision_test.go @@ -2,58 +2,128 @@ package registry import "testing" -// TestCodebuddyModelSupportsImagesWhitelist verifies that models confirmed by -// live-backend measurement to support images (despite app.asar marking them -// text-only) report vision support via the whitelist. -func TestCodebuddyModelSupportsImagesWhitelist(t *testing.T) { - whitelisted := []string{ - "glm-5.1", - "glm-5.2", +// installCodebuddyTestCatalog pins the package-level synced catalog to a fixed +// model list for the duration of a test. The catalog is a process-wide global, +// so every capability test installs its own catalog and restores the previous +// one on cleanup to avoid cross-test pollution. +func installCodebuddyTestCatalog(t *testing.T, models []*ModelInfo) { + t.Helper() + codebuddySyncMu.Lock() + prev := codebuddySynced + codebuddySynced = models + codebuddySyncMu.Unlock() + t.Cleanup(func() { + codebuddySyncMu.Lock() + codebuddySynced = prev + codebuddySyncMu.Unlock() + }) +} + +// TestCodebuddyModelSupportsImagesFollowsCatalog verifies that native vision +// capability is taken verbatim from the online model catalog's supportsImages +// field for models without a measured override: models the catalog flags as +// image-capable report support (including deepseek-v4.x, which used to be +// hard-coded as text-only), while models the catalog does not flag report no +// support. +func TestCodebuddyModelSupportsImagesFollowsCatalog(t *testing.T) { + installCodebuddyTestCatalog(t, []*ModelInfo{ + {ID: "deepseek-v4.1-flash", SupportsImages: true}, + {ID: "deepseek-v4-flash", SupportsImages: true}, + {ID: "deepseek-v4-pro", SupportsImages: true}, + {ID: "glm-5.3-flash", SupportsImages: true}, + {ID: "glm-5.3", SupportsImages: true}, + {ID: "glm-5.2", SupportsImages: false}, + {ID: "hunyuan-2.0-thinking", SupportsImages: false}, + }) + + visionCapable := []string{ + "deepseek-v4.1-flash", + "deepseek-v4-flash", + "deepseek-v4-pro", + "glm-5.3-flash", + "glm-5.3", } - for _, id := range whitelisted { + for _, id := range visionCapable { if !CodebuddyModelSupportsImages(id) { - t.Errorf("whitelisted model %q should report vision support", id) + t.Errorf("catalog marks %q image-capable; CodebuddyModelSupportsImages should be true", id) + } + } + + textOnly := []string{ + "glm-5.2", + "hunyuan-2.0-thinking", + } + for _, id := range textOnly { + if CodebuddyModelSupportsImages(id) { + t.Errorf("catalog does not mark %q image-capable; CodebuddyModelSupportsImages should be false", id) } } } -// TestCodebuddyModelSupportsImagesCaseInsensitive verifies the whitelist is -// case-insensitive. -func TestCodebuddyModelSupportsImagesCaseInsensitive(t *testing.T) { +// TestCodebuddyModelSupportsImagesOverrides verifies that the measured +// capability override tables win over the catalog (2026-09-11 live measurement, +// see 《模型视觉能力实测与校正表.md》): +// - glm-5v-turbo: catalog says image-capable, upstream refuses images → false. +// - glm-5.1 / deepseek-v3-2-volc: catalog omits capability, upstream reads +// images correctly → true. +func TestCodebuddyModelSupportsImagesOverrides(t *testing.T) { + installCodebuddyTestCatalog(t, []*ModelInfo{ + // The catalog's own values must be overridden for these three. + {ID: "glm-5v-turbo", SupportsImages: true}, + {ID: "glm-5.1", SupportsImages: false}, + {ID: "deepseek-v3-2-volc", SupportsImages: false}, + // A control model without an override keeps the catalog value. + {ID: "glm-5.3", SupportsImages: true}, + }) + + if CodebuddyModelSupportsImages("glm-5v-turbo") { + t.Error("glm-5v-turbo must be excluded from native vision (measured: upstream refuses images)") + } + if !CodebuddyModelSupportsImages("glm-5.1") { + t.Error("glm-5.1 must be included in native vision (measured: upstream reads images)") + } + if !CodebuddyModelSupportsImages("deepseek-v3-2-volc") { + t.Error("deepseek-v3-2-volc must be included in native vision (measured: upstream reads images)") + } + // Case-insensitivity and whitespace trimming apply to the overrides too. + if CodebuddyModelSupportsImages(" GLM-5V-TURBO ") { + t.Error("glm-5v-turbo exclusion must be case-insensitive and trimmed") + } if !CodebuddyModelSupportsImages("GLM-5.1") { - t.Error("whitelist lookup should be case-insensitive") + t.Error("glm-5.1 inclusion must be case-insensitive") + } + // Control: no override → catalog value applies. + if !CodebuddyModelSupportsImages("glm-5.3") { + t.Error("un-overridden model must follow the catalog") } } -// TestCodebuddyModelSupportsImagesDeepSeekNotWhitelisted verifies that -// deepseek-v4-flash / deepseek-v4-pro no longer report native vision support: -// live testing showed the backend returns a refusal text for them, so the -// vision-proxy layer (preprocess) must handle their image inputs instead. -func TestCodebuddyModelSupportsImagesDeepSeekNotWhitelisted(t *testing.T) { - notWhitelisted := []string{ - "deepseek-v4-flash", - "deepseek-v4-pro", - } - for _, id := range notWhitelisted { - if CodebuddyModelSupportsImages(id) { - t.Errorf("model %q should NOT report native vision support (backend returns refusal)", id) - } +// TestCodebuddyModelSupportsImagesCaseInsensitive verifies the catalog lookup is +// case-insensitive. +func TestCodebuddyModelSupportsImagesCaseInsensitive(t *testing.T) { + installCodebuddyTestCatalog(t, []*ModelInfo{ + {ID: "GLM-5.3-Flash", SupportsImages: true}, + }) + if !CodebuddyModelSupportsImages("glm-5.3-flash") { + t.Error("catalog lookup should be case-insensitive") } } -// TestCodebuddyModelSupportsImagesFakeVision verifies that models which return -// a "model does not support images" refusal text (or unknown/empty IDs) do not -// report vision support, so the vision-proxy layer still routes them. -func TestCodebuddyModelSupportsImagesFakeVision(t *testing.T) { - fake := []string{ +// TestCodebuddyModelSupportsImagesUnknown verifies that models absent from the +// catalog (or an empty ID) report no native vision support, so clients do not +// send images the upstream would reject. +func TestCodebuddyModelSupportsImagesUnknown(t *testing.T) { + installCodebuddyTestCatalog(t, []*ModelInfo{ + {ID: "glm-5.3", SupportsImages: true}, + }) + unknown := []string{ "hunyuan-2.0-instruct", - "hunyuan-2.0-thinking", "unknown-model", "", } - for _, id := range fake { + for _, id := range unknown { if CodebuddyModelSupportsImages(id) { - t.Errorf("model %q should NOT report vision support", id) + t.Errorf("model %q is not in the catalog; should NOT report vision support", id) } } } diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/model_definitions.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/model_definitions.go index 0f5c5e1..0c1a744 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/model_definitions.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/registry/model_definitions.go @@ -130,31 +130,6 @@ func GetCodebuddyModels() []*ModelInfo { return WithCodebuddyBuiltins(cloneModelInfos(getModels().Codebuddy)) } -// codebuddyVisionBackendWhitelist lists CodeBuddy models whose official app.asar -// supportsImages flag is false but which the live backend verifiably routes to a -// vision sub-model. They are treated as vision-capable so the vision-proxy layer -// does not re-route them. -// -// NOTE (2026-08-21): deepseek-v4-flash / deepseek-v4-pro were removed from this -// list — live testing through the CLIProxy relay showed the backend returns a -// "this model does not support image input" refusal text for them (the images -// reach the text model unchanged instead of being routed to a vision sub-model). -// They now fall through to the vision-proxy layer (preprocess via hy3-preview). -var codebuddyVisionBackendWhitelist = map[string]struct{}{ - "glm-5.1": {}, - "glm-5.2": {}, -} - -// codebuddyVisionBlacklist lists CodeBuddy models whose backend supportsImages -// flag is true but which verifiably reject image input at inference time (the -// backend returns a "this model does not support image input" refusal text). -// They are treated as text-only so the vision-proxy layer routes them through -// the configured vision model (hy3-preview) instead of passing images through. -var codebuddyVisionBlacklist = map[string]struct{}{ - "deepseek-v4-pro": {}, - "deepseek-v4-flash": {}, -} - // CodebuddyMaxCompletionTokensDefault is the fallback max completion token // ceiling shared by CodeBuddy models when a specific value is unknown. The // synced catalog and the static models.json fallback both declare 32768 for @@ -176,14 +151,42 @@ func CodebuddyModelMaxCompletionTokens(modelID string) int { return CodebuddyMaxCompletionTokensDefault } -// CodebuddyModelSupportsImages reports whether the given CodeBuddy model accepts -// image input. It consults, in order: -// 1. The measured blacklist (models the backend flags as vision-capable but -// which verifiably reject image input, e.g. deepseek-v4-pro/flash). -// 2. The measured backend whitelist (models the live backend verifiably routes -// to a vision sub-model despite app.asar marking them text-only). -// 3. The official client's app.asar supportsImages flag (or the static -// models.json fallback when the client is not installed). +// codebuddyVisionExcluded lists models the online catalog marks as +// image-capable (supportsImages=true) but which verifiably reject image input +// at inference time. They must report text-only so clients do not send images +// the upstream would refuse with a "not a vision model" reply. +// +// Measured 2026-09-11 against the live upstream (4 keys × 2 images, 7 requests, +// every one refused: "作为 GLM 大语言模型…不具备处理视觉信息的能力"). +// See 《模型视觉能力实测与校正表.md》 §3.1. +var codebuddyVisionExcluded = map[string]struct{}{ + "glm-5v-turbo": {}, +} + +// codebuddyVisionIncluded lists models the online catalog does not mark as +// image-capable (supportsImages=false or absent) but which verifiably accept +// and read image input. They must report image support so clients keep sending +// images instead of filtering them out client-side. +// +// Measured 2026-09-11 against the live upstream (4/4 keys read the test image +// correctly, A/B confirmed). See 《模型视觉能力实测与校正表.md》 §3.2. +var codebuddyVisionIncluded = map[string]struct{}{ + "glm-5.1": {}, + "deepseek-v3-2-volc": {}, +} + +// CodebuddyModelSupportsImages reports whether the given CodeBuddy model natively +// accepts image input. It consults, in order: +// +// 1. codebuddyVisionExcluded — catalog wrongly says image-capable; forced off. +// 2. codebuddyVisionIncluded — catalog wrongly omits image capability; forced on. +// 3. the online model catalog's supportsImages field (synced from the official +// backend endpoint, cached locally) — the capability source for everything +// else. Models absent from the catalog report false. +// +// The two override tables exist because the catalog is not a reliable capability +// source by itself (2026-09-11 measurement found both false positives and false +// negatives). Keep them minimal and dated; they are the only local overrides. // // Unknown or empty model IDs report false. func CodebuddyModelSupportsImages(modelID string) bool { @@ -192,10 +195,10 @@ func CodebuddyModelSupportsImages(modelID string) bool { return false } key := strings.ToLower(modelID) - if _, ok := codebuddyVisionBlacklist[key]; ok { + if _, ok := codebuddyVisionExcluded[key]; ok { return false } - if _, ok := codebuddyVisionBackendWhitelist[key]; ok { + if _, ok := codebuddyVisionIncluded[key]; ok { return true } for _, m := range GetCodebuddyModels() { diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor.go index 01bc3c3..63242cf 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor.go @@ -13,7 +13,6 @@ import ( "github.com/router-for-me/CLIProxyAPI/v7/internal/auth/codebuddy" "github.com/router-for-me/CLIProxyAPI/v7/internal/config" - internallogging "github.com/router-for-me/CLIProxyAPI/v7/internal/logging" "github.com/router-for-me/CLIProxyAPI/v7/internal/registry" "github.com/router-for-me/CLIProxyAPI/v7/internal/runtime/executor/helps" cliproxyauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth" @@ -94,37 +93,9 @@ func (e *CodebuddyExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth body = normalizeCodebuddyChatImageContent(body) // Diagnostic: capture the exact shape of a CodeBuddy read-tool image workflow - // that the vision router does not recognize (problem-two investigation). + // that the image input detection does not recognize (problem-two investigation). codebuddyDumpReadToolDiagnostic(body) - // Read-tool image backfill: when the client read an image via read/read_file - // and the tool result was reduced to a placeholder (base64 omitted), re-attach - // the image from the tool_calls filePath so the vision router recognizes it. - body = codebuddyBackfillReadToolImages(body) - - // Historical image rewrite: replace truncated historical image stubs with a - // text marker so text-only models do not choke on them (preprocess/routing). - body = e.rewriteCodebuddyHistoricalImagesForTextModel(body, baseModel) - - // Vision proxy: transparently handle image input for non-vision models. - body, _ = e.applyCodebuddyVisionProxy(ctx, auth, body, baseModel, nil, reporter) - - // Agentic vision: server-side tool-calling loop for text-only models to - // autonomously inspect images via the vision model. - if e.codebuddyVisionAgenticEnabled() { - if codebuddyChatHasImageInput(body) { - helps.DumpCodebuddyDebugBody("vision-reporter-trigger", - []byte(fmt.Sprintf("path=Execute baseModel=%s visionModel=%s", baseModel, e.cfg.CodebuddyVision.VisionModel()))) - internallogging.SetVisionSubagent(ctx, true) - visionReporter := helps.NewUsageReporter(ctx, e.Identifier(), baseModel, auth) - defer visionReporter.TrackFailure(ctx, &err) - return e.executeCodebuddyVisionAgentic(ctx, auth, req, opts, body, baseModel, creds, baseURL, visionReporter) - } - // Text-only turn in agentic mode: strip any stale images re-sent by the - // client so they don't reach the text-only model and get filtered. - body = replaceCodebuddyImagesWithText(body, codebuddyHistoricalImageText) - } - // Prompt cache: inject a stable session-bound key so repeated turns in the // same conversation hit the backend prefix cache (lower credit). body = applyCodebuddyPromptCache(body, codebuddyExecutionSessionID(req, opts)) @@ -150,6 +121,15 @@ func (e *CodebuddyExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth return resp, err } + // Read-tool image backfill: when the client read an image via read/read_file + // and the tool result was reduced to a placeholder (base64 omitted), re-attach + // the image from the tool_calls filePath so it reaches the upstream model. + // It MUST run after normalizeCodebuddyToolMessages: that pass appends a + // synthetic user message when the body ends with a tool message, and the + // upstream only adopts images carried by the LAST user message. Backfilling + // first would hide the image behind that synthetic message. + body = codebuddyBackfillReadToolImages(body) + // Clamp oversized max_tokens (Cursor sends 65536) to the model's declared // MaxCompletionTokens ceiling so strict backend routes do not reject it. body = clampCodebuddyMaxTokens(body, baseModel) @@ -245,45 +225,9 @@ func (e *CodebuddyExecutor) ExecuteStream(ctx context.Context, auth *cliproxyaut body = normalizeCodebuddyChatImageContent(body) // Diagnostic: capture the exact shape of a CodeBuddy read-tool image workflow - // that the vision router does not recognize (problem-two investigation). + // that the image input detection does not recognize (problem-two investigation). codebuddyDumpReadToolDiagnostic(body) - // Read-tool image backfill: when the client read an image via read/read_file - // and the tool result was reduced to a placeholder (base64 omitted), re-attach - // the image from the tool_calls filePath so the vision router recognizes it. - body = codebuddyBackfillReadToolImages(body) - - // Historical image rewrite: replace truncated historical image stubs with a - // text marker so text-only models do not choke on them (preprocess/routing). - body = e.rewriteCodebuddyHistoricalImagesForTextModel(body, baseModel) - - // Vision proxy: transparently handle image input for non-vision models. - // Routing is applied synchronously (cheap model swap). Preprocess is deferred - // into the stream goroutine below so the vision description can be forwarded - // to the client in real time (it takes seconds and must not block the first - // byte / trip the relay stream-open watchdog). When preprocess is needed we - // keep the original image-bearing body intact here and describe it later. - needsPreprocess := e.codebuddyVisionNeedsPreprocess(body, baseModel) - if !needsPreprocess { - body, _ = e.applyCodebuddyVisionProxy(ctx, auth, body, baseModel, nil, reporter) - } - - // Agentic vision: server-side tool-calling loop for text-only models to - // autonomously inspect images via the vision model. - if e.codebuddyVisionAgenticEnabled() { - if codebuddyChatHasImageInput(body) { - helps.DumpCodebuddyDebugBody("vision-reporter-trigger", - []byte(fmt.Sprintf("path=ExecuteStream baseModel=%s visionModel=%s", baseModel, e.cfg.CodebuddyVision.VisionModel()))) - internallogging.SetVisionSubagent(ctx, true) - visionReporter := helps.NewUsageReporter(ctx, e.Identifier(), baseModel, auth) - defer visionReporter.TrackFailure(ctx, &err) - return e.executeCodebuddyVisionAgenticStream(ctx, auth, req, opts, body, baseModel, creds, baseURL, visionReporter) - } - // Text-only turn in agentic mode: strip any stale images re-sent by the - // client so they don't reach the text-only model and get filtered. - body = replaceCodebuddyImagesWithText(body, codebuddyHistoricalImageText) - } - // Prompt cache: inject a stable session-bound key so repeated turns in the // same conversation hit the backend prefix cache (lower credit). body = applyCodebuddyPromptCache(body, codebuddyExecutionSessionID(req, opts)) @@ -302,12 +246,11 @@ func (e *CodebuddyExecutor) ExecuteStream(ctx context.Context, auth *cliproxyaut body = helps.ApplyPayloadConfigWithRequest(e.cfg, baseModel, to.String(), from.String(), "", body, originalTranslated, requestedModel, requestPath, opts.Headers) // NOTE: normalizeCodebuddyToolMessages is intentionally NOT called here. - // It appends a synthetic user message when the body ends with a tool - // message, which would shift lastCodebuddyUserMessageIndex and break the - // deferred streaming preprocess below (image extraction and the - // description replacement both key off the last user message). It runs - // inside the stream goroutine, after the images have been swapped for - // their descriptions. + // It runs inside the stream goroutine below, immediately before + // codebuddyBackfillReadToolImages. The two must stay in that order: the + // backfill depends on the LAST user message (the upstream only adopts + // images carried by it), and normalization may append a synthetic user + // message when the body ends with a tool message. // Clamp oversized max_tokens (Cursor sends 65536) to the model's declared // MaxCompletionTokens ceiling so strict backend routes do not reject it. @@ -335,53 +278,9 @@ func (e *CodebuddyExecutor) ExecuteStream(ctx context.Context, auth *cliproxyaut return true } - // Preprocess streaming: describe the images via the vision model first, - // forwarding each description delta to the client in real time, then - // rewrite the image parts into text and continue with the text-only model. - // The initial role chunk opens the stream immediately so the relay's - // stream-open watchdog does not trip during the multi-second vision call. - if needsPreprocess { - visionModel := e.cfg.CodebuddyVision.VisionModel() - initChunk := buildCodebuddyVisionChunk("", baseModel, 0, nil, "assistant") - if initChunk != nil && !emit(initChunk) { - return - } - descriptions, visionUsage, descErr := e.describeImagesWithVisionModel(ctx, auth, body, visionModel, e.cfg.CodebuddyVision.PreprocessPrompt, baseModel, emit) - if descErr != nil { - log.Warnf("codebuddy vision proxy: preprocess stream failed for %s (vision=%s): %v; omitting images", baseModel, visionModel, descErr) - body = replaceCodebuddyImagesWithText(body, codebuddyOmittedImageText) - } else { - log.Infof("codebuddy vision proxy: preprocessed %d image(s) for %s via %s", len(descriptions), baseModel, visionModel) - body = replaceCodebuddyImagesWithDescriptions(body, descriptions, codebuddyOmittedImageText) - // Report the vision model's usage as a separate additional-model - // record, aligned with the agentic path. - reporter.PublishAdditionalModelAlways(ctx, visionModel, visionUsage) - } - // Re-apply the stream forcing so the (rewritten) text-only body still - // carries stream=true / include_usage after the image swap. - var errSet error - body, errSet = sjson.SetBytes(body, "stream", true) - if errSet == nil { - body, errSet = sjson.SetBytes(body, "stream_options.include_usage", true) - } - if errSet != nil { - select { - case out <- cliproxyexecutor.StreamChunk{Err: errSet}: - case <-ctx.Done(): - } - return - } - // Separate the text-model stream translator state from the vision - // delta state so the client stream stays coherent. - param = nil - } - // Normalize tool-related message fields so the strict backend does not - // reject tool-calling rounds with 400 invalid_parameter_value. This must - // run AFTER the preprocess block above: it may append a synthetic user - // message when the body ends with a tool message, and doing so before - // the image extraction/replacement would shift the last-user-message - // boundary and hide the backfilled images from the vision call. + // reject tool-calling rounds with 400 invalid_parameter_value. It may + // append a synthetic user message when the body ends with a tool message. body, errNorm := normalizeCodebuddyToolMessages(body) if errNorm != nil { select { @@ -391,6 +290,13 @@ func (e *CodebuddyExecutor) ExecuteStream(ctx context.Context, auth *cliproxyaut return } + // Read-tool image backfill: re-attach an image the client read via + // read/read_file but whose tool result was reduced to a placeholder. + // It MUST run after the normalization above: that pass appends a + // synthetic user message when the body ends with a tool message, and the + // upstream only adopts images carried by the LAST user message. + body = codebuddyBackfillReadToolImages(body) + httpReq, errReq := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) if errReq != nil { select { diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_backfill.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_backfill.go new file mode 100644 index 0000000..7f456e7 --- /dev/null +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_backfill.go @@ -0,0 +1,548 @@ +package executor + +// Read-tool image backfill. +// +// This file is all that remains of the former vision-proxy layer (preprocess / +// routing / agentic sub-agent), which was removed in 2026-09-11 because every +// in-catalog model that can be served either handles images natively or should +// not receive images at all, so the extra vision call, its latency, and its +// failure modes were pure cost. +// +// What is kept is NOT part of that layer: CodeBuddy (and Cursor's Read File V2) +// reach images through the read/read_file tool whose role=tool result is a +// placeholder ("image already analyzed...", "Read image file: ") with the +// base64 omitted. Native-vision models are equally blind to that placeholder, +// so the backfill — re-reading the image from the tool_calls filePath and +// re-attaching it as an image_url part — is required for them too. It is +// independent of any vision configuration and always runs. + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "mime" + "os" + "path/filepath" + "strings" + + "github.com/router-for-me/CLIProxyAPI/v7/internal/runtime/executor/helps" + log "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// lastCodebuddyUserMessageIndex returns the index of the last role=="user" +// message, or -1 if there is none. Detecting images only in the last user +// message isolates the "current request" from historical turns, so a client's +// text-only follow-up in the same session is not misclassified by a previous +// image. +func lastCodebuddyUserMessageIndex(messages []gjson.Result) int { + for i := len(messages) - 1; i >= 0; i-- { + if messages[i].Get("role").String() == "user" { + return i + } + } + return -1 +} + +// codebuddyContinuationReminderMarkers identify IDE-injected user messages that +// only nudge the model to continue after a tool result. CodeBuddy IDE appends +// `The tool call completed.` as a role=user +// message after every tool result; it carries no real user input and must not be +// treated as the start of a new turn. +var codebuddyContinuationReminderMarkers = []string{ + "", + "the tool call completed", +} + +// codebuddyTurnStartUserMessageIndex returns the index of the user message that +// starts the CURRENT turn, skipping trailing IDE-injected continuation +// reminders. +// +// CodeBuddy IDE appends a `` user message after every tool +// result, so that reminder becomes the last user message and the assistant's +// read tool_call of the very same turn ends up BEFORE it. A scan that treats +// "at/after the last user message" as the current turn would then classify the +// in-flight read as historical and never backfill its image — the exact reason +// deepseek-v4-pro reported "I cannot see the image" (2026-09-11). Falling back +// to the last *substantive* user message fixes that while still excluding tool +// reads from genuinely earlier turns. Returns lastCodebuddyUserMessageIndex +// (possibly -1) when every user message is a reminder. +func codebuddyTurnStartUserMessageIndex(messages []gjson.Result) int { + for i := len(messages) - 1; i >= 0; i-- { + if messages[i].Get("role").String() != "user" { + continue + } + if isCodebuddyContinuationReminder(messages[i]) { + continue + } + return i + } + return lastCodebuddyUserMessageIndex(messages) +} + +// isCodebuddyContinuationReminder reports whether a user message is an +// IDE-injected continuation nudge rather than real user input. +func isCodebuddyContinuationReminder(msg gjson.Result) bool { + text := strings.ToLower(strings.TrimSpace(codebuddyMessageText(msg.Get("content")))) + if text == "" { + return false + } + for _, marker := range codebuddyContinuationReminderMarkers { + if strings.Contains(text, marker) { + return true + } + } + return false +} + +// codebuddyMessageText flattens a message content (plain string or OpenAI +// content-part array) into its concatenated text. +func codebuddyMessageText(content gjson.Result) string { + if !content.Exists() { + return "" + } + if content.IsArray() { + parts := make([]string, 0, len(content.Array())) + for _, part := range content.Array() { + if text := strings.TrimSpace(part.Get("text").String()); text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, "\n") + } + return content.String() +} + +// codebuddyImageStubMaxPayloadChars is the threshold below which a data-URL +// image part is considered a truncated stub rather than a real image. Clients +// (CodeBuddy IDE, Cursor) truncate historical images to ~80-char stubs +// (e.g. "data:image/jpeg;base64,/9j/4AAQSkZJRgABA") when re-sending +// conversation history; such parts carry no usable pixels (~30 bytes). +// Real images are essentially always > 1KB of base64 payload. +const codebuddyImageStubMaxPayloadChars = 512 + +// codebuddyImagePartIsStub reports whether an image part carries no usable +// image data: a data: URL whose payload is shorter than the stub threshold, +// or a part with no URL at all. Remote (http/https) URLs are never stubs. +func codebuddyImagePartIsStub(raw []byte) bool { + url := gjson.GetBytes(raw, "image_url.url").String() + if url == "" { + // input_image / Anthropic-style forms carry the URL directly as a string. + if direct := gjson.GetBytes(raw, "image_url"); direct.Type == gjson.String { + url = direct.String() + } + } + if url == "" { + return true + } + if !strings.HasPrefix(url, "data:") { + return false + } + idx := strings.Index(url, ",") + if idx < 0 { + return true + } + return len(url)-idx-1 < codebuddyImageStubMaxPayloadChars +} + +// codebuddyChatHasImageInput reports whether the OpenAI-style chat body carries +// at least one REAL image part (image_url or input_image with usable data) in +// the current turn — the last user message and any subsequent assistant/tool +// messages. Truncated historical stubs do not count: they carry no usable +// pixels, so treating them as image input would only block the backfill from +// re-attaching the real image. +func codebuddyChatHasImageInput(body []byte) bool { + messages := gjson.GetBytes(body, "messages") + if !messages.IsArray() { + return false + } + arr := messages.Array() + lastUserIdx := lastCodebuddyUserMessageIndex(arr) + if lastUserIdx < 0 { + return false + } + for mi := lastUserIdx; mi < len(arr); mi++ { + content := arr[mi].Get("content") + if !content.IsArray() { + continue + } + for _, part := range content.Array() { + if !isCodebuddyImagePartType(part.Get("type").String()) { + continue + } + if codebuddyImagePartIsStub([]byte(part.Raw)) { + continue + } + return true + } + } + return false +} + +// isCodebuddyImagePartType reports whether a content-part type carries image +// input (OpenAI image_url or Anthropic-style input_image). +func isCodebuddyImagePartType(typ string) bool { + return typ == "image_url" || typ == "input_image" +} + +// codebuddyDumpReadToolDiagnostic emits a debug dump when the request appears to +// carry a Read-tool image workflow that the image input detection did NOT +// recognize (codebuddyChatHasImageInput returned false). CodeBuddy reads images +// via its `read` tool rather than attaching them as image_url parts, so this +// dump captures the exact shape (tool_calls carrying a base64/path, or a +// role=tool message) for backfill debugging. It is a no-op unless +// CODEBUDDY_DEBUG_BODY=1. +func codebuddyDumpReadToolDiagnostic(body []byte) { + if !helps.CodebuddyDebugBodyEnabled() { + return + } + if codebuddyChatHasImageInput(body) { + return + } + if !codebuddyBodyMentionsReadTool(body) { + return + } + helps.DumpCodebuddyDebugBody("read-tool-diagnostic", body) +} + +// codebuddyBodyMentionsReadTool reports whether the body contains any trace of a +// read/read_file tool (a tool_calls entry, a tool declaration, or a role=tool +// message naming read). It is used as the fast short-circuit for the backfill +// and to gate the diagnostic dump above. +func codebuddyBodyMentionsReadTool(body []byte) bool { + if !gjson.ValidBytes(body) { + return false + } + // Top-level tool declarations. + for _, t := range gjson.GetBytes(body, "tools").Array() { + name := t.Get("function.name").String() + if name == "" { + name = t.Get("name").String() + } + if isCodebuddyReadToolName(name) { + return true + } + } + // Any message whose role is tool, or whose tool_calls name read. + for _, m := range gjson.GetBytes(body, "messages").Array() { + if m.Get("role").String() == "tool" { + return true + } + for _, tc := range m.Get("tool_calls").Array() { + name := tc.Get("function.name").String() + if name == "" { + name = tc.Get("name").String() + } + if isCodebuddyReadToolName(name) { + return true + } + } + } + return false +} + +// isCodebuddyReadToolName reports whether a tool name refers to file-reading +// (read / read_file, case-insensitive), which is how CodeBuddy inspects images. +func isCodebuddyReadToolName(name string) bool { + n := strings.ToLower(strings.TrimSpace(name)) + return n == "read" || n == "read_file" || n == "readfile" || n == "read-file" +} + +// codebuddyBackfillMaxImageBytes caps the size of a single local image file that +// the read-tool backfill will base64-encode and attach. Larger images are +// skipped (with a log) rather than bloating the request body into the tens of MB. +const codebuddyBackfillMaxImageBytes = 20 << 20 // 20MB + +// codebuddyImagePlaceholderMarkers are substrings that identify a role=tool +// content that is a placeholder for a previously-read image rather than real +// text. CodeBuddy (and its client) replace the image with a short note such as +// "[Image already analyzed in an earlier step; base64 content omitted to save +// memory. ...]" and drop the base64. The backfill detects this and re-attaches +// the image from the tool_calls filePath so the model can see it. +var codebuddyImagePlaceholderMarkers = []string{ + "image already analyzed", + "base64 content omitted", + "image omitted", + // Cursor's Read File V2 tool returns a bare confirmation string for image + // files (e.g. "Read image file: h:\...\home.png") instead of image data. + "read image file", +} + +// codebuddyBackfillReadToolImages detects the CodeBuddy read-tool image workflow +// — where images reach the model via the `read`/`read_file` tool whose result +// content is a placeholder (the base64 was omitted to save memory) — and, when +// the current turn has no recognizable image part, reads the image back from the +// tool_calls filePath/path and appends an image_url part to the last user +// message so the image reaches the upstream model intact. +// +// Data source decision (from packet capture): the role=tool content is a +// placeholder, NOT base64, so the image must be recovered from the tool_calls +// filePath. This only works when the relay runs on the same host as the client +// (the filePath is a local absolute path). On a remote/independent-server relay +// the file cannot be read and the function degrades to a no-op. +// +// It is idempotent and safe: it returns the original body unchanged unless all +// of the following hold — the body mentions a read tool, the current turn has no +// image input, and at least one read-tool placeholder maps to a readable image +// file. Failure to read/encode any single file is non-fatal. +func codebuddyBackfillReadToolImages(body []byte) []byte { + if len(body) == 0 || !gjson.ValidBytes(body) { + return body + } + // Fast short-circuit: nothing to do unless a read tool is mentioned and the + // request would not already carry an image. + if !codebuddyBodyMentionsReadTool(body) { + return body + } + if codebuddyChatHasImageInput(body) { + return body + } + + messages := gjson.GetBytes(body, "messages") + if !messages.IsArray() { + return body + } + arr := messages.Array() + lastUserIdx := lastCodebuddyUserMessageIndex(arr) + if lastUserIdx < 0 { + return body + } + + // Collect read-tool image filePaths whose role=tool result is a placeholder. + // Only tool reads from the CURRENT turn qualify: historical tool reads were + // already backfilled in their own turn, and re-attaching them to every later + // question injects stale, unrelated images (2026-09-05 Cursor incident: an + // anime picture the agent read two turns earlier kept being re-attached + // whenever the user asked about a different, freshly pasted photo). + // + // The turn start is the last SUBSTANTIVE user message, not the last user + // message: CodeBuddy IDE appends a `` continuation as a + // user message after each tool result, which would otherwise push the + // in-flight read before the last user message and hide it as "historical". + turnStartIdx := codebuddyTurnStartUserMessageIndex(arr) + if turnStartIdx < 0 { + return body + } + paths := collectCodebuddyReadImagePaths(arr, turnStartIdx) + if len(paths) == 0 { + return body + } + + out := body + // The last user message content may be a plain string (Cursor sends string + // content on continuation turns). sjson cannot append an array element to a + // string: `content.-1` would silently turn the string into a malformed + // {"-1": ...} object that the upstream does not accept. Normalize non-array + // content into a text part first so the image append below yields a valid + // OpenAI content-part array. + contentPath := fmt.Sprintf("messages.%d.content", lastUserIdx) + if content := gjson.GetBytes(out, contentPath); content.Exists() && !content.IsArray() { + textParts, err := json.Marshal([]map[string]string{{"type": "text", "text": content.String()}}) + if err != nil { + return body + } + next, err := sjson.SetRawBytes(out, contentPath, textParts) + if err != nil { + return body + } + out = next + } + appended := 0 + for _, p := range paths { + dataURL, mimeType, ok := readCodebuddyImageAsDataURL(p) + if !ok { + continue + } + part := codebuddyImagePartJSON(dataURL) + next, err := sjson.SetRawBytes(out, fmt.Sprintf("messages.%d.content.-1", lastUserIdx), part) + if err != nil { + log.Warnf("codebuddy read-tool backfill: append image_url for %s failed: %v", p, err) + return body + } + out = next + appended++ + log.Infof("codebuddy read-tool backfill: attached image %s (%s, %d bytes) to last user message", p, mimeType, len(dataURL)) + } + if appended == 0 { + return body + } + return out +} + +// collectCodebuddyReadImagePaths scans assistant tool_calls for read/read_file +// invocations whose arguments carry a filePath/path, and whose corresponding +// role=tool result content is an image placeholder. Only those pairs yield an +// image path. tool_call_id is matched between the assistant tool_calls entry and +// the following role=tool message (falling back to order-based matching when IDs +// are absent). The result preserves body order and de-duplicates paths. +// +// Assistant messages before minAssistantIdx (i.e. before the current turn's +// last user message) are ignored: their images were already backfilled in their +// own turn, and re-attaching them now would inject stale, unrelated pictures +// into the user's latest question. +func collectCodebuddyReadImagePaths(messages []gjson.Result, minAssistantIdx int) []string { + type pending struct { + id string + path string + } + var pendings []pending + seenIDs := map[string]bool{} + order := []string{} + + for mi, m := range messages { + role := m.Get("role").String() + switch role { + case "assistant": + if mi < minAssistantIdx { + // Historical tool read: belongs to an earlier turn, do not + // re-inject its image into the current question. + continue + } + for _, tc := range m.Get("tool_calls").Array() { + name := tc.Get("function.name").String() + if name == "" { + name = tc.Get("name").String() + } + if !isCodebuddyReadToolName(name) { + continue + } + args := tc.Get("function.arguments").String() + if args == "" { + args = tc.Get("arguments").String() + } + if p := extractCodebuddyToolFilePath(args); p != "" { + id := tc.Get("id").String() + pendings = append(pendings, pending{id: id, path: p}) + } + } + case "tool": + if len(pendings) == 0 { + continue + } + if !isCodebuddyImagePlaceholder(m.Get("content").String()) { + continue + } + // Match by tool_call_id when available, else consume in order. + tcID := m.Get("tool_call_id").String() + if tcID != "" { + for _, pd := range pendings { + if pd.id == tcID { + if !seenIDs[pd.id] { + seenIDs[pd.id] = true + order = append(order, pd.path) + } + break + } + } + continue + } + // No ID on the tool message: consume the oldest unmatched pending. + for _, pd := range pendings { + if !seenIDs[pd.id] { + seenIDs[pd.id] = true + order = append(order, pd.path) + break + } + } + } + } + return order +} + +// extractCodebuddyToolFilePath parses the JSON arguments of a read tool call and +// returns the file path from filePath (CodeBuddy read_file) or path (older Read +// tool). It tolerates malformed JSON by falling back to a substring scan. +func extractCodebuddyToolFilePath(args string) string { + args = strings.TrimSpace(args) + if args == "" { + return "" + } + if gjson.Valid(args) { + if p := gjson.Get(args, "filePath").String(); p != "" { + return strings.TrimSpace(p) + } + if p := gjson.Get(args, "path").String(); p != "" { + return strings.TrimSpace(p) + } + } + // Fallback: scan for a quoted filePath/path key. + for _, key := range []string{`"filePath"`, `"path"`} { + idx := strings.Index(args, key) + if idx < 0 { + continue + } + rest := args[idx+len(key):] + colon := strings.Index(rest, ":") + if colon < 0 { + continue + } + rest = rest[colon+1:] + q := strings.Index(rest, `"`) + if q < 0 { + continue + } + end := strings.Index(rest[q+1:], `"`) + if end < 0 { + continue + } + return strings.TrimSpace(rest[q+1 : q+1+end]) + } + return "" +} + +// isCodebuddyImagePlaceholder reports whether a role=tool content string looks +// like a placeholder for a previously-read image (rather than real text output). +func isCodebuddyImagePlaceholder(content string) bool { + lower := strings.ToLower(content) + for _, marker := range codebuddyImagePlaceholderMarkers { + if strings.Contains(lower, marker) { + return true + } + } + return false +} + +// readCodebuddyImageAsDataURL reads a local image file, detects its MIME type +// from the file extension, base64-encodes its contents, and returns the data URL +// plus the detected MIME type. It reports ok=false on any failure (missing file, +// oversized file, read error, unsupported/unknown extension) so the caller can +// degrade to leaving the request unchanged. +func readCodebuddyImageAsDataURL(path string) (dataURL, mimeType string, ok bool) { + info, err := os.Stat(path) + if err != nil { + log.Warnf("codebuddy read-tool backfill: image file not readable %s: %v", path, err) + return "", "", false + } + if info.IsDir() { + return "", "", false + } + if info.Size() > codebuddyBackfillMaxImageBytes { + log.Warnf("codebuddy read-tool backfill: image %s too large (%d bytes > %d), skipping", path, info.Size(), codebuddyBackfillMaxImageBytes) + return "", "", false + } + + ext := strings.ToLower(filepath.Ext(path)) + mt := mime.TypeByExtension(ext) + if mt == "" || !strings.HasPrefix(mt, "image/") { + log.Warnf("codebuddy read-tool backfill: unsupported image extension %q for %s, skipping", ext, path) + return "", "", false + } + + raw, err := os.ReadFile(path) + if err != nil { + log.Warnf("codebuddy read-tool backfill: read image %s failed: %v", path, err) + return "", "", false + } + return "data:" + mt + ";base64," + base64.StdEncoding.EncodeToString(raw), mt, true +} + +// codebuddyImagePartJSON renders the canonical image_url part used by the +// backend (see normalizeCodebuddyImagePart). +func codebuddyImagePartJSON(dataURL string) []byte { + part, _ := json.Marshal(map[string]any{ + "type": "image_url", + "image_url": map[string]any{"url": dataURL}, + }) + return part +} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_backfill_test.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_backfill_test.go index 192e907..4d15b09 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_backfill_test.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_backfill_test.go @@ -343,3 +343,117 @@ func TestIsCodebuddyImagePlaceholder(t *testing.T) { } } } + +// buildCodebuddyTrailingReminderBody models the EXACT shape CodeBuddy IDE sends +// during an agent loop: after every tool result the IDE appends a role=user +// `The tool call completed.` message. That +// trailing reminder becomes the last user message, so a scan that treats +// "at/after the last user message" as the current turn classifies the +// assistant's read tool_call of the very same turn as historical. +func buildCodebuddyTrailingReminderBody(path string) string { + args, _ := json.Marshal(map[string]string{"filePath": path}) + placeholder := "[Image already analyzed in an earlier step; base64 content omitted to save memory. Path: " + path + ".]" + body := map[string]any{ + "model": "deepseek-v4-pro", + "messages": []any{ + map[string]any{"role": "user", "content": []any{map[string]any{"type": "text", "text": "@photo.png 图片在这里"}}}, + map[string]any{ + "role": "assistant", + "content": nil, + "tool_calls": []any{ + map[string]any{ + "id": "call_1", + "type": "function", + "function": map[string]any{ + "name": "read", + "arguments": string(args), + }, + }, + }, + }, + map[string]any{"role": "tool", "tool_call_id": "call_1", "content": placeholder}, + map[string]any{"role": "user", "content": "The tool call completed."}, + }, + } + b, _ := json.Marshal(body) + return string(b) +} + +// TestCodebuddyBackfillReadToolImages_TrailingSystemReminderTurn is the +// regression for the 2026-09-11 deepseek-v4-pro report: CodeBuddy IDE appends a +// `` user message after every tool result, so the read pair of +// the CURRENT turn sits before the last user message. The backfill used to skip +// it as historical and never attach the image, which is why the model answered +// "I cannot see the image". The image must land on the LAST user message — the +// only one the upstream adopts. +func TestCodebuddyBackfillReadToolImages_TrailingSystemReminderTurn(t *testing.T) { + dir := t.TempDir() + img := writeTestPNG(t, dir, "photo.png") + in := buildCodebuddyTrailingReminderBody(img) + + out := codebuddyBackfillReadToolImages([]byte(in)) + if string(out) == in { + t.Fatalf("backfill did not fire for a trailing system_reminder turn; body=%s", out) + } + + msgs := gjson.GetBytes(out, "messages").Array() + last := msgs[len(msgs)-1] + if last.Get("role").String() != "user" { + t.Fatalf("last message role = %q, want user", last.Get("role").String()) + } + content := last.Get("content") + if !content.IsArray() { + t.Fatalf("last user content should be an array after backfill, got %s", content.Raw) + } + imgURL := "" + for _, part := range content.Array() { + if part.Get("type").String() == "image_url" { + imgURL = part.Get("image_url.url").String() + } + } + if imgURL == "" { + t.Fatalf("no image_url part on the last user message: %s", content.Raw) + } + // The asking user message must not be polluted with the image: the upstream + // would ignore it and the model would stay blind. + if ask := gjson.GetBytes(out, "messages.0.content"); ask.IsArray() { + for _, part := range ask.Array() { + if part.Get("type").String() == "image_url" { + t.Fatalf("image injected into the asking user message instead of the last one: %s", out) + } + } + } +} + +// TestCodebuddyBackfillReadToolImages_AfterNormalizeContinuation pins the +// production order normalize -> backfill. normalizeCodebuddyToolMessages appends +// a synthetic user message when the body ends with a tool message; the backfill +// must run AFTER it so the image lands on that new last user message instead of +// being hidden behind it (the upstream only adopts images on the last user +// message). +func TestCodebuddyBackfillReadToolImages_AfterNormalizeContinuation(t *testing.T) { + dir := t.TempDir() + img := writeTestPNG(t, dir, "photo.png") + in := []byte(buildCurrentTurnReadToolImageBody(img)) // ends with a tool message + + norm, err := normalizeCodebuddyToolMessages(in) + if err != nil { + t.Fatalf("normalizeCodebuddyToolMessages: %v", err) + } + out := codebuddyBackfillReadToolImages(norm) + + msgs := gjson.GetBytes(out, "messages").Array() + last := msgs[len(msgs)-1] + if last.Get("role").String() != "user" { + t.Fatalf("last message role = %q, want user", last.Get("role").String()) + } + found := false + for _, part := range last.Get("content").Array() { + if part.Get("type").String() == "image_url" && part.Get("image_url.url").String() != "" { + found = true + } + } + if !found { + t.Fatalf("image_url missing from the last user message after normalize+backfill: %s", out) + } +} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_image_input_test.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_image_input_test.go new file mode 100644 index 0000000..86f66b5 --- /dev/null +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_image_input_test.go @@ -0,0 +1,183 @@ +package executor + +// Tests for the image-input detection helpers that back the read-tool image +// backfill (codebuddy_executor_backfill.go). The former vision-proxy layer and +// its tests were removed on 2026-09-11. + +import ( + "strings" + "testing" +) + +func TestCodebuddyChatHasImageInput(t *testing.T) { + tests := []struct { + name string + in string + want bool + }{ + { + // payload 必须超过 codebuddyImageStubMaxPayloadChars(512)， + // 否则会被 codebuddyImagePartIsStub 判定为截断残片（stub）。 + name: "image_url part detected", + in: `{"messages":[{"role":"user","content":[{"type":"text","text":"hi"},{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 600) + `"}}]}]}`, + want: true, + }, + { + name: "input_image part detected", + in: `{"messages":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,` + strings.Repeat("B", 600) + `"}]}]}`, + want: true, + }, + { + name: "text only", + in: `{"messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}`, + want: false, + }, + { + name: "string content", + in: `{"messages":[{"role":"user","content":"hello"}]}`, + want: false, + }, + { + name: "no messages", + in: `{"model":"auto"}`, + want: false, + }, + { + name: "invalid json", + in: `not-json`, + want: false, + }, + { + name: "historical image ignored when last user message is text-only", + in: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]},{"role":"assistant","content":"ok"},{"role":"user","content":[{"type":"text","text":"继续"}]}]}`, + want: false, + }, + { + name: "image in last user message detected despite text history", + in: `{"messages":[{"role":"user","content":[{"type":"text","text":"之前"}]},{"role":"assistant","content":"ok"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 600) + `"}}]}]}`, + want: true, + }, + { + name: "image in tool message detected (Read tool result)", + in: `{"messages":[{"role":"user","content":[{"type":"text","text":"读一下这张图"}]},{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"Read","arguments":"{\"file_path\":\"a.png\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 600) + `"}}]}]}`, + want: true, + }, + { + name: "historical tool image ignored when last user message is text-only", + in: `{"messages":[{"role":"user","content":[{"type":"text","text":"读图"}]},{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"Read","arguments":"{\"file_path\":\"a.png\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,HIST"}}]},{"role":"assistant","content":"看完了"},{"role":"user","content":[{"type":"text","text":"继续"}]}]}`, + want: false, + }, + { + name: "no user message", + in: `{"messages":[{"role":"assistant","content":"ok"}]}`, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := codebuddyChatHasImageInput([]byte(tt.in)); got != tt.want { + t.Fatalf("codebuddyChatHasImageInput() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsCodebuddyReadToolName(t *testing.T) { + for _, in := range []string{"read", "Read", "read_file", "READ_FILE", "readfile", "read-file", " Read "} { + if !isCodebuddyReadToolName(in) { + t.Fatalf("isCodebuddyReadToolName(%q) = false, want true", in) + } + } + for _, in := range []string{"bash", "write", "write_file", "ReadFilex", "globs"} { + if isCodebuddyReadToolName(in) { + t.Fatalf("isCodebuddyReadToolName(%q) = true, want false", in) + } + } +} + +// TestCodebuddyBodyMentionsReadTool guards the backfill's fast short-circuit and +// the diagnostic gate across tool declarations, assistant tool_calls, and +// role=tool messages. +func TestCodebuddyBodyMentionsReadTool(t *testing.T) { + tests := []struct { + name string + in string + want bool + }{ + { + name: "tool declaration read", + in: `{"tools":[{"type":"function","function":{"name":"read"}}],"messages":[]}`, + want: true, + }, + { + name: "assistant tool_calls read", + in: `{"messages":[{"role":"assistant","tool_calls":[{"id":"c1","type":"function","function":{"name":"Read","arguments":"{\"file_path\":\"a.png\"}"}}]}]}`, + want: true, + }, + { + name: "role tool message", + in: `{"messages":[{"role":"tool","tool_call_id":"c1","content":"text"}]}`, + want: true, + }, + { + name: "no read tool", + in: `{"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, + want: false, + }, + { + name: "invalid json", + in: `not-json`, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := codebuddyBodyMentionsReadTool([]byte(tt.in)); got != tt.want { + t.Fatalf("codebuddyBodyMentionsReadTool() = %v, want %v", got, tt.want) + } + }) + } +} + +const stubURL = "data:image/jpeg;base64,/9j/4AAQSkZJRgABA" // 40-char payload, the real-world 80-char stub + +func TestCodebuddyImagePartIsStub(t *testing.T) { + tests := []struct { + name string + raw string + want bool + }{ + {"80-char truncated stub", `{"type":"image_url","image_url":{"url":"` + stubURL + `"}}`, true}, + {"real data url", `{"type":"image_url","image_url":{"url":"data:image/png;base64,` + string(make([]byte, 600)) + `"}}`, false}, + {"remote url is never stub", `{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}`, false}, + {"empty url", `{"type":"image_url","image_url":{"url":""}}`, true}, + {"input_image stub form", `{"type":"input_image","image_url":"` + stubURL + `"}`, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := codebuddyImagePartIsStub([]byte(tt.raw)); got != tt.want { + t.Fatalf("codebuddyImagePartIsStub() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCodebuddyChatHasImageInputIgnoresStubs(t *testing.T) { + // Tool-continuation turn: user message carries a truncated stub, followed by + // tool results. A stub carries no usable pixels and must not count as image + // input, so the backfill can re-attach the real image from the read tool. + body := `{"messages":[` + + `{"role":"user","content":[{"type":"text","text":"描述"},{"type":"image_url","image_url":{"url":"` + stubURL + `"}}]},` + + `{"role":"assistant","tool_calls":[{"id":"c1","function":{"name":"read_file","arguments":"{\"path\":\"h:////home.png/"}"}}]},` + + `{"role":"tool","tool_call_id":"c1","content":"Read image file: h://home.png"}]}` + if codebuddyChatHasImageInput([]byte(body)) { + t.Fatal("truncated stub must not count as image input") + } +} + +func TestCursorReadFileV2PlaceholderRecognized(t *testing.T) { + if !isCodebuddyImagePlaceholder("Read image file: h://Ai 自测空间文档\\home.png") { + t.Fatal("Cursor Read File V2 confirmation must be recognized as an image placeholder") + } +} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_regression_test.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_regression_test.go index 9b45a52..dceab16 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_regression_test.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_regression_test.go @@ -84,28 +84,4 @@ func TestCodebuddyEffectiveStatus_SuccessUntouched(t *testing.T) { } } -// --- agentic tool_choice reset --------------------------------------------- -func TestInjectCodebuddyInspectTool_ResetsStaleToolChoice(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","tool_choice":{"type":"function","function":{"name":"read_file"}},"messages":[{"role":"user","content":"hi"}]}`) - out := injectCodebuddyInspectTool(body, 1) - if got := gjson.GetBytes(out, "tool_choice").String(); got != "auto" { - t.Fatalf("tool_choice = %q, want %q; out=%s", got, "auto", out) - } -} - -func TestInjectCodebuddyInspectTool_ResetsRequiredToolChoice(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","tool_choice":"required","messages":[{"role":"user","content":"hi"}]}`) - out := injectCodebuddyInspectTool(body, 1) - if got := gjson.GetBytes(out, "tool_choice").String(); got != "auto" { - t.Fatalf("tool_choice = %q, want %q; out=%s", got, "auto", out) - } -} - -func TestInjectCodebuddyInspectTool_AddsToolChoiceWhenAbsent(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","messages":[{"role":"user","content":"hi"}]}`) - out := injectCodebuddyInspectTool(body, 1) - if got := gjson.GetBytes(out, "tool_choice").String(); got != "auto" { - t.Fatalf("tool_choice = %q, want %q; out=%s", got, "auto", out) - } -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision.go deleted file mode 100644 index c648c85..0000000 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision.go +++ /dev/null @@ -1,1180 +0,0 @@ -package executor - -import ( - "bufio" - "bytes" - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "mime" - "net/http" - "os" - "path/filepath" - "strings" - - "github.com/router-for-me/CLIProxyAPI/v7/internal/auth/codebuddy" - "github.com/router-for-me/CLIProxyAPI/v7/internal/config" - "github.com/router-for-me/CLIProxyAPI/v7/internal/registry" - "github.com/router-for-me/CLIProxyAPI/v7/internal/runtime/executor/helps" - cliproxyauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth" - "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/usage" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - log "github.com/sirupsen/logrus" -) - -// codebuddyVisionAction describes how the vision-proxy layer handles a request. -type codebuddyVisionAction int - -const ( - // codebuddyVisionPassThrough leaves the request unchanged. - codebuddyVisionPassThrough codebuddyVisionAction = iota - // codebuddyVisionRoute swaps the request model to the configured vision model. - codebuddyVisionRoute - // codebuddyVisionPreprocess describes images first, then continues with the - // original model. - codebuddyVisionPreprocess -) - -// defaultCodebuddyVisionPrompt is the system prompt sent to the vision model in -// preprocess mode when no override is configured. -const defaultCodebuddyVisionPrompt = "请仔细观察图片，用中文详细、准确地描述图片内容。如果用户针对图片提出了具体问题，请优先提取与用户问题直接相关的细节（如指定位置的文字、数字、颜色、图表数据等），确保这些关键信息不遗漏，然后再补充图片的其它内容。注意：只客观陈述图片中实际存在的内容，不要提出任何解决方案、修改建议、操作步骤或分析判断。" - -// codebuddyOmittedImageText replaces image parts when preprocess fails and the -// request degrades to omitting images. -const codebuddyOmittedImageText = "[图片因视觉代理失败被省略]" - -// codebuddyHistoricalImageText replaces historical image parts in agentic mode -// so a later turn does not re-send stale images to a text-only model that -// rejects image input. -const codebuddyHistoricalImageText = "[历史图片]" - -func (a codebuddyVisionAction) String() string { - switch a { - case codebuddyVisionRoute: - return "routing" - case codebuddyVisionPreprocess: - return "preprocess" - default: - return "pass-through" - } -} - -// lastCodebuddyUserMessageIndex returns the index of the last role=="user" -// message, or -1 if there is none. Detecting images only in the last user -// message isolates the "current request" from historical turns, so a text-only -// follow-up in the same session is not misclassified by a previous image. -func lastCodebuddyUserMessageIndex(messages []gjson.Result) int { - for i := len(messages) - 1; i >= 0; i-- { - if messages[i].Get("role").String() == "user" { - return i - } - } - return -1 -} - -// codebuddyUserQuestion extracts the text content of the last user message in -// the chat body. This is the user's actual question/instruction, used to focus -// the vision model's description on the details the user actually cares about. -// It returns an empty string when there is no user message or the message has -// no text content (e.g. an image-only turn). It is a pure function (no I/O). -func codebuddyUserQuestion(body []byte) string { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return "" - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx < 0 { - return "" - } - - content := arr[lastUserIdx].Get("content") - // Simple string content: "content": "the question". - if content.Type == gjson.String { - return strings.TrimSpace(content.String()) - } - if !content.IsArray() { - return "" - } - - // Array content: concatenate all text parts, preserving order. - var sb strings.Builder - for _, part := range content.Array() { - if part.Get("type").String() != "text" { - continue - } - text := part.Get("text").String() - if text == "" { - continue - } - if sb.Len() > 0 { - sb.WriteByte('\n') - } - sb.WriteString(text) - } - return strings.TrimSpace(sb.String()) -} - -// buildCodebuddyVisionPrompt composes the prompt sent to the vision model in -// preprocess mode. Priority: -// 1. If a custom PreprocessPrompt is configured, it is used verbatim. -// 2. Otherwise, if the user's question text is available, the prompt is focused -// on that question so the vision model extracts exactly the relevant details. -// 3. Otherwise, fall back to the generic default description prompt. -func buildCodebuddyVisionPrompt(preprocessPrompt, question string) string { - if strings.TrimSpace(preprocessPrompt) != "" { - return preprocessPrompt - } - if q := strings.TrimSpace(question); q != "" { - return "用户的问题是：「" + q + "」。请仔细观察图片，仅针对该问题从图片中精准提取与之直接相关的细节（如指定位置的文字、数字、颜色、图表数据等），用中文准确、详细地描述，确保关键信息不遗漏，不要臆测图片中不存在的内容。注意：只客观陈述图片中实际存在的内容，不要提出任何解决方案、修改建议、操作步骤或分析判断。" - } - return defaultCodebuddyVisionPrompt -} - -// codebuddyImagePart describes a single image part found in the current turn, -// along with its absolute path in the body so it can be replaced in place. -type codebuddyImagePart struct { - path string // e.g. "messages.0.content.2" - raw []byte // raw JSON of the image part ({"type":"image_url",...}) -} - -// codebuddyImageStubMaxPayloadChars is the threshold below which a data-URL -// image part is considered a truncated stub rather than a real image. Clients -// (CodeBuddy IDE, Cursor) truncate historical images to ~80-char stubs -// (e.g. "data:image/jpeg;base64,/9j/4AAQSkZJRgABA") when re-sending -// conversation history; such parts carry no usable pixels (~30 bytes). -// Real images are essentially always > 1KB of base64 payload. -const codebuddyImageStubMaxPayloadChars = 512 - -// codebuddyImagePartIsStub reports whether an image part carries no usable -// image data: a data: URL whose payload is shorter than the stub threshold, -// or a part with no URL at all. Remote (http/https) URLs are never stubs. -func codebuddyImagePartIsStub(raw []byte) bool { - url := gjson.GetBytes(raw, "image_url.url").String() - if url == "" { - // input_image / Anthropic-style forms carry the URL directly as a string. - if direct := gjson.GetBytes(raw, "image_url"); direct.Type == gjson.String { - url = direct.String() - } - } - if url == "" { - return true - } - if !strings.HasPrefix(url, "data:") { - return false - } - idx := strings.Index(url, ",") - if idx < 0 { - return true - } - return len(url)-idx-1 < codebuddyImageStubMaxPayloadChars -} - -// extractCodebuddyCurrentImages walks the current turn (the last user message -// and any subsequent assistant/tool messages) and returns every REAL image part -// in body order together with its replacement path. Historical images (before -// the last user message) and truncated stubs are ignored, matching -// codebuddyChatHasImageInput. -func extractCodebuddyCurrentImages(body []byte) []codebuddyImagePart { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return nil - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx < 0 { - return nil - } - - var images []codebuddyImagePart - for mi := lastUserIdx; mi < len(arr); mi++ { - content := arr[mi].Get("content") - if !content.IsArray() { - continue - } - for ci, part := range content.Array() { - if !isCodebuddyImagePartType(part.Get("type").String()) { - continue - } - if codebuddyImagePartIsStub([]byte(part.Raw)) { - continue - } - images = append(images, codebuddyImagePart{ - path: fmt.Sprintf("messages.%d.content.%d", mi, ci), - raw: append([]byte(nil), []byte(part.Raw)...), - }) - } - } - return images -} - -// replaceCodebuddyCurrentTurnStubsWithText replaces truncated image stubs in -// the current turn (the last user message onward) with a text marker. Real -// image parts are left untouched (they are handled by preprocess/routing). -// Stubs carry no usable pixels: if they were passed to the vision model the -// call would fail and poison the request with the "omitted" placeholder; if -// passed through to the text model they make it disavow earlier descriptions. -func replaceCodebuddyCurrentTurnStubsWithText(body []byte, text string) []byte { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return body - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx < 0 { - return body - } - - out := body - for mi := lastUserIdx; mi < len(arr); mi++ { - content := arr[mi].Get("content") - if !content.IsArray() { - continue - } - for ci, part := range content.Array() { - if !isCodebuddyImagePartType(part.Get("type").String()) { - continue - } - if !codebuddyImagePartIsStub([]byte(part.Raw)) { - continue - } - path := fmt.Sprintf("messages.%d.content.%d", mi, ci) - replacement, err := json.Marshal(map[string]string{"type": "text", "text": text}) - if err != nil { - return body - } - out, err = sjson.SetRawBytes(out, path, replacement) - if err != nil { - // Never corrupt the request on a path error; keep the original. - return body - } - } - } - return out -} - -// codebuddyChatHasImageInput reports whether the OpenAI-style chat body carries -// at least one REAL image part (image_url or input_image with usable data) in -// the current turn — the last user message and any subsequent assistant/tool -// messages. Truncated historical stubs do not count: they carry no usable -// pixels, so triggering a vision call on them would waste quota, fail, and -// poison the request with the omitted-image placeholder. -func codebuddyChatHasImageInput(body []byte) bool { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return false - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx < 0 { - return false - } - for mi := lastUserIdx; mi < len(arr); mi++ { - content := arr[mi].Get("content") - if !content.IsArray() { - continue - } - for _, part := range content.Array() { - if !isCodebuddyImagePartType(part.Get("type").String()) { - continue - } - if codebuddyImagePartIsStub([]byte(part.Raw)) { - continue - } - return true - } - } - return false -} - -// isCodebuddyImagePartType reports whether a content-part type carries image -// input (OpenAI image_url or Anthropic-style input_image). -func isCodebuddyImagePartType(typ string) bool { - return typ == "image_url" || typ == "input_image" -} - -// rewriteCodebuddyModel replaces the top-level model field with model. -func rewriteCodebuddyModel(body []byte, model string) []byte { - out, err := sjson.SetBytes(body, "model", model) - if err != nil { - return body - } - return out -} - -// replaceCodebuddyImagesWithText replaces every image part (image_url / -// input_image) with a text part carrying the provided description. Non-image -// parts are left untouched, and the replacement preserves the part's position -// within the message content array. -func replaceCodebuddyImagesWithText(body []byte, text string) []byte { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return body - } - - out := body - for mi, msg := range messages.Array() { - content := msg.Get("content") - if !content.IsArray() { - continue - } - for ci, part := range content.Array() { - if !isCodebuddyImagePartType(part.Get("type").String()) { - continue - } - path := fmt.Sprintf("messages.%d.content.%d", mi, ci) - replacement, err := json.Marshal(map[string]string{"type": "text", "text": text}) - if err != nil { - return body - } - out, err = sjson.SetRawBytes(out, path, replacement) - if err != nil { - // Never corrupt the request on a path error; keep the original. - return body - } - } - } - return out -} - -// replaceCodebuddyImagesWithDescriptions replaces each image part in the -// current turn (last user message and any subsequent messages) with a distinct -// text description, matched by body order. It is the multi-image counterpart of -// replaceCodebuddyImagesWithText: descriptions[i] replaces the i-th image part. -// If fewer descriptions are supplied than images, the remaining images fall -// back to fallbackText. Historical images are left untouched. -func replaceCodebuddyImagesWithDescriptions(body []byte, descriptions []string, fallbackText string) []byte { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return body - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx < 0 { - return body - } - - out := body - idx := 0 - for mi := lastUserIdx; mi < len(arr); mi++ { - content := arr[mi].Get("content") - if !content.IsArray() { - continue - } - for ci, part := range content.Array() { - if !isCodebuddyImagePartType(part.Get("type").String()) { - continue - } - text := fallbackText - if idx < len(descriptions) && strings.TrimSpace(descriptions[idx]) != "" { - text = descriptions[idx] - } - idx++ - - path := fmt.Sprintf("messages.%d.content.%d", mi, ci) - replacement, err := json.Marshal(map[string]string{"type": "text", "text": text}) - if err != nil { - return body - } - out, err = sjson.SetRawBytes(out, path, replacement) - if err != nil { - // Never corrupt the request on a path error; keep the original. - return body - } - } - } - return out -} - -// rewriteCodebuddyHistoricalImagesForTextModel replaces image parts in -// historical messages (everything before the current turn, i.e. before the last -// user message) with a plain-text marker when the request is headed to a -// text-only model under preprocess/routing mode. -// -// Rationale: clients truncate historical images to useless stubs (e.g. an -// 80-char truncated data URL) when re-sending conversation history. In -// preprocess/routing mode the current-turn images are described and replaced, -// but historical image parts used to pass through untouched, so the text-only -// upstream model received an unreadable stub and concluded "I cannot see the -// image", disavowing the earlier assistant description. Native-vision models -// and off/agentic modes are intentionally left untouched (agentic mode manages -// historical images in its own loop). -func (e *CodebuddyExecutor) rewriteCodebuddyHistoricalImagesForTextModel(body []byte, baseModel string) []byte { - visionCfg := e.cfg.CodebuddyVision - mode := visionCfg.NormalizedVisionMode() - if mode != config.CodebuddyVisionModePreprocess && mode != config.CodebuddyVisionModeRouting { - return body - } - currentModel := strings.TrimSpace(gjson.GetBytes(body, "model").String()) - if currentModel == "" { - currentModel = strings.TrimSpace(baseModel) - } - // The vision engine itself and native-vision models keep history intact. - if strings.EqualFold(currentModel, strings.TrimSpace(visionCfg.VisionModel())) { - return body - } - if registry.CodebuddyModelSupportsImages(currentModel) { - return body - } - body = replaceCodebuddyHistoricalImagesWithText(body, codebuddyHistoricalImageText) - // Truncated stubs inside the current turn (e.g. re-sent by tool-call - // continuation turns) get the same treatment: they carry no usable pixels - // and must never reach the vision model or the text model as "images". - body = replaceCodebuddyCurrentTurnStubsWithText(body, codebuddyHistoricalImageText) - return body -} - -// replaceCodebuddyHistoricalImagesWithText replaces every image part -// (image_url / input_image) in messages before the current turn (the last user -// message) with a text part carrying the provided marker. Current-turn image -// parts are left untouched; they are handled by the preprocess/routing logic. -func replaceCodebuddyHistoricalImagesWithText(body []byte, text string) []byte { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return body - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx <= 0 { - return body - } - - out := body - for mi := 0; mi < lastUserIdx; mi++ { - content := arr[mi].Get("content") - if !content.IsArray() { - continue - } - for ci, part := range content.Array() { - if !isCodebuddyImagePartType(part.Get("type").String()) { - continue - } - path := fmt.Sprintf("messages.%d.content.%d", mi, ci) - replacement, err := json.Marshal(map[string]string{"type": "text", "text": text}) - if err != nil { - return body - } - out, err = sjson.SetRawBytes(out, path, replacement) - if err != nil { - // Never corrupt the request on a path error; keep the original. - return body - } - } - } - return out -} - -// codebuddyVisionNeedsPreprocess reports whether the request should be handled -// by the preprocess strategy (describe images first, then continue with the -// original text-only model). It mirrors the routing decision of -// codebuddyVisionPlan without performing any I/O, so the streaming path can -// defer the (blocking, ~seconds) vision call into the stream goroutine. -func (e *CodebuddyExecutor) codebuddyVisionNeedsPreprocess(body []byte, baseModel string) bool { - visionCfg := e.cfg.CodebuddyVision - mode := visionCfg.NormalizedVisionMode() - if mode != config.CodebuddyVisionModePreprocess { - return false - } - if !codebuddyChatHasImageInput(body) { - return false - } - visionModel := visionCfg.VisionModel() - currentModel := strings.TrimSpace(gjson.GetBytes(body, "model").String()) - if currentModel == "" { - currentModel = strings.TrimSpace(baseModel) - } - // Never re-route the vision engine itself, and leave native image models alone. - if strings.EqualFold(strings.TrimSpace(currentModel), strings.TrimSpace(visionModel)) { - return false - } - if registry.CodebuddyModelSupportsImages(currentModel) { - return false - } - return true -} - -// codebuddyVisionPlan decides how the vision-proxy layer should handle the -// request. It is a pure function (no I/O) so routing decisions stay unit-testable. -func codebuddyVisionPlan(mode, visionModel, currentModel string, hasImage, currentSupportsImages bool) codebuddyVisionAction { - if mode != config.CodebuddyVisionModeRouting && mode != config.CodebuddyVisionModePreprocess { - return codebuddyVisionPassThrough - } - if !hasImage { - return codebuddyVisionPassThrough - } - // The vision engine model itself must never be re-routed (avoids recursion). - if strings.EqualFold(strings.TrimSpace(currentModel), strings.TrimSpace(visionModel)) { - return codebuddyVisionPassThrough - } - // Models that natively accept images are left to the backend. - if currentSupportsImages { - return codebuddyVisionPassThrough - } - if mode == config.CodebuddyVisionModePreprocess { - return codebuddyVisionPreprocess - } - return codebuddyVisionRoute -} - -// applyCodebuddyVisionProxy is the single entry point that both Execute and -// ExecuteStream call after image normalization. It returns the (possibly -// rewritten) request body and reports whether the request was rewritten. -// -// Routing mode swaps the model and returns immediately. Preprocess mode performs -// an extra upstream call to the vision model to describe the images, then swaps -// the image parts for the returned descriptions; on failure it degrades to -// omitting the images rather than failing the whole request. -// -// When emit is non-nil (streaming path), each vision description delta is -// forwarded through it so the user can watch the image being described in real -// time; otherwise deltas are only aggregated. -func (e *CodebuddyExecutor) applyCodebuddyVisionProxy(ctx context.Context, auth *cliproxyauth.Auth, body []byte, baseModel string, emit func([]byte) bool, reporter *helps.UsageReporter) ([]byte, bool) { - visionCfg := e.cfg.CodebuddyVision - mode := visionCfg.NormalizedVisionMode() - if mode == config.CodebuddyVisionModeOff { - return body, false - } - if !codebuddyChatHasImageInput(body) { - return body, false - } - - visionModel := visionCfg.VisionModel() - currentModel := strings.TrimSpace(gjson.GetBytes(body, "model").String()) - if currentModel == "" { - currentModel = strings.TrimSpace(baseModel) - } - - action := codebuddyVisionPlan(mode, visionModel, currentModel, true, registry.CodebuddyModelSupportsImages(currentModel)) - switch action { - case codebuddyVisionRoute: - log.Infof("codebuddy vision proxy: routing %s -> %s", currentModel, visionModel) - return rewriteCodebuddyModel(body, visionModel), true - - case codebuddyVisionPreprocess: - descriptions, visionUsage, err := e.describeImagesWithVisionModel(ctx, auth, body, visionModel, visionCfg.PreprocessPrompt, baseModel, emit) - if err != nil { - log.Warnf("codebuddy vision proxy: preprocess failed for %s (vision=%s): %v; omitting images", currentModel, visionModel, err) - return replaceCodebuddyImagesWithText(body, codebuddyOmittedImageText), true - } - log.Infof("codebuddy vision proxy: preprocessed %d image(s) for %s via %s", len(descriptions), currentModel, visionModel) - // Report the vision model's usage as a separate additional-model record - // so its credit/tokens are visible in the request log alongside the - // base model's own record (aligned with the agentic path). - reporter.PublishAdditionalModelAlways(ctx, visionModel, visionUsage) - return replaceCodebuddyImagesWithDescriptions(body, descriptions, codebuddyOmittedImageText), true - - default: - return body, false - } -} - -// describeImagesWithVisionModel describes every image in the current turn, one -// at a time (serial), via the vision model. It returns one description per -// image, in body order, plus the accumulated vision-model usage across all -// images. When emit is non-nil, each vision delta is forwarded through it (for -// streaming); otherwise deltas are only aggregated. This is the single -// implementation shared by both the non-streaming Execute path and the -// streaming ExecuteStream path. -func (e *CodebuddyExecutor) describeImagesWithVisionModel( - ctx context.Context, - auth *cliproxyauth.Auth, - body []byte, - visionModel, prompt string, - baseModel string, - emit func([]byte) bool, -) ([]string, usage.Detail, error) { - question := codebuddyUserQuestion(body) - fullPrompt := buildCodebuddyVisionPrompt(prompt, question) - - images := extractCodebuddyCurrentImages(body) - if len(images) == 0 { - return nil, usage.Detail{}, fmt.Errorf("vision model called with no images") - } - - var totalUsage usage.Detail - descriptions := make([]string, 0, len(images)) - for i, img := range images { - desc, u, err := e.describeSingleImageWithVisionModel(ctx, auth, body, visionModel, fullPrompt, baseModel, img, emit) - if err != nil { - log.Warnf("codebuddy vision proxy: image %d/%d description failed (vision=%s): %v", i+1, len(images), visionModel, err) - descriptions = append(descriptions, "") - continue - } - addCodebuddyVisionUsage(&totalUsage, u) - descriptions = append(descriptions, desc) - } - return descriptions, totalUsage, nil -} - -// describeSingleImageWithVisionModel sends a single image to the vision model -// and returns its text description plus the vision-model usage for that call. -// The request body is rebuilt so that only the target image (plus the injected -// user question, if any) is sent, avoiding redundant multi-image payloads and -// keeping each serial call cheap. baseModel is used as the model field on the -// emitted vision chunks so the client always sees the requested model. -func (e *CodebuddyExecutor) describeSingleImageWithVisionModel( - ctx context.Context, - auth *cliproxyauth.Auth, - body []byte, - visionModel, prompt string, - baseModel string, - img codebuddyImagePart, - emit func([]byte) bool, -) (string, usage.Detail, error) { - // Build a minimal request: model + the single image + the user question text. - question := codebuddyUserQuestion(body) - userContent := []any{json.RawMessage(img.raw)} - if question != "" { - userContent = append(userContent, map[string]any{"type": "text", "text": question}) - } - reqBody := map[string]any{ - "model": visionModel, - "messages": []any{ - map[string]any{"role": "user", "content": userContent}, - }, - } - descBody, err := json.Marshal(reqBody) - if err != nil { - return "", usage.Detail{}, err - } - descBody, err = prependCodebuddySystemMessage(descBody, prompt) - if err != nil { - return "", usage.Detail{}, err - } - descBody, err = sjson.SetBytes(descBody, "stream", true) - if err != nil { - return "", usage.Detail{}, err - } - descBody, err = sjson.SetBytes(descBody, "stream_options.include_usage", true) - if err != nil { - return "", usage.Detail{}, err - } - - creds := codebuddy.CredsFromAuth(auth) - url := creds.ResolveBaseURL() + codebuddy.ChatPath - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(descBody)) - if err != nil { - return "", usage.Detail{}, err - } - applyCodebuddyHeaders(httpReq, creds) - httpReq.Header.Set("Accept", "text/event-stream") - - httpClient := helps.NewProxyAwareHTTPClient(ctx, e.cfg, auth, 0) - httpResp, err := httpClient.Do(httpReq) - if err != nil { - return "", usage.Detail{}, err - } - defer func() { _ = httpResp.Body.Close() }() - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - b, _ := io.ReadAll(httpResp.Body) - return "", usage.Detail{}, statusErr{code: httpResp.StatusCode, msg: string(b)} - } - - var ( - id string - created int64 - content strings.Builder - firstEmitDone bool - usageDetail usage.Detail - ) - // The vision chunk's model field is always rewritten to baseModel so the - // client sees a single consistent model from start to finish. - model := baseModel - scanner := bufio.NewScanner(httpResp.Body) - scanner.Buffer(nil, 52_428_800) - for scanner.Scan() { - line := bytes.TrimSpace(scanner.Bytes()) - if len(line) == 0 || !bytes.HasPrefix(line, []byte("data:")) { - continue - } - payload := bytes.TrimSpace(line[len("data:"):]) - if len(payload) == 0 || bytes.Equal(payload, []byte("[DONE]")) { - continue - } - if !gjson.ValidBytes(payload) { - continue - } - // Accumulate the vision-model usage chunk (if any) for reporting. - if u, ok := helps.ParseOpenAIStreamUsage(line); ok { - addCodebuddyVisionUsage(&usageDetail, u) - } - res := gjson.ParseBytes(payload) - if id == "" { - id = res.Get("id").String() - } - if created == 0 { - created = res.Get("created").Int() - } - for _, ch := range res.Get("choices").Array() { - delta := ch.Get("delta") - // Forward content deltas to the caller (streaming path). role and - // reasoning deltas are skipped so the client only sees the visible - // description text, matching what would later replace the image. - if c := delta.Get("content"); c.Exists() && c.Type == gjson.String && c.String() != "" { - content.WriteString(c.String()) - if emit != nil { - if !firstEmitDone { - firstEmitDone = true - // Emit an assistant role delta first so the stream has a - // valid opening chunk before content deltas arrive. - roleChunk := buildCodebuddyVisionChunk(id, model, created, nil, "assistant") - if !emit(roleChunk) { - return "", usage.Detail{}, ctx.Err() - } - } - contentStr := c.String() - contentChunk := buildCodebuddyVisionChunk(id, model, created, &contentStr, "") - if !emit(contentChunk) { - return "", usage.Detail{}, ctx.Err() - } - } - } - } - } - if errScan := scanner.Err(); errScan != nil { - return "", usage.Detail{}, errScan - } - - desc := strings.TrimSpace(content.String()) - if desc == "" { - return "", usage.Detail{}, fmt.Errorf("vision model returned empty description") - } - return desc, usageDetail, nil -} - -// addCodebuddyVisionUsage accumulates a single vision-model call's usage into -// the running total for the preprocess loop. -func addCodebuddyVisionUsage(total *usage.Detail, add usage.Detail) { - if total == nil { - return - } - total.InputTokens += add.InputTokens - total.OutputTokens += add.OutputTokens - total.ReasoningTokens += add.ReasoningTokens - total.CachedTokens += add.CachedTokens - total.CacheReadTokens += add.CacheReadTokens - total.CacheCreationTokens += add.CacheCreationTokens - total.TotalTokens += add.TotalTokens - total.Credit += add.Credit - total.TokenBreakdown.TotalTokens += add.TokenBreakdown.TotalTokens - total.TokenBreakdown.Input.TotalTokens += add.TokenBreakdown.Input.TotalTokens - total.TokenBreakdown.Input.UncachedTokens += add.TokenBreakdown.Input.UncachedTokens - total.TokenBreakdown.Input.CacheReadTokens += add.TokenBreakdown.Input.CacheReadTokens - total.TokenBreakdown.Input.CacheWriteTokens += add.TokenBreakdown.Input.CacheWriteTokens - total.TokenBreakdown.Output.TotalTokens += add.TokenBreakdown.Output.TotalTokens - total.TokenBreakdown.Output.NonReasoningTokens += add.TokenBreakdown.Output.NonReasoningTokens - total.TokenBreakdown.Output.ReasoningTokens += add.TokenBreakdown.Output.ReasoningTokens - total.TokenBreakdown.UnclassifiedTokens += add.TokenBreakdown.UnclassifiedTokens -} - -// buildCodebuddyVisionChunk renders a single OpenAI stream chunk for the vision -// sub-agent's description stream. content may be nil (role-only chunk). -func buildCodebuddyVisionChunk(id, model string, created int64, content *string, role string) []byte { - delta := map[string]any{} - if role != "" { - delta["role"] = role - } - if content != nil { - delta["content"] = *content - } - chunk, err := json.Marshal(map[string]any{ - "id": id, - "object": "chat.completion.chunk", - "created": created, - "model": model, - "choices": []any{ - map[string]any{"index": 0, "delta": delta, "finish_reason": nil}, - }, - }) - if err != nil { - return nil - } - return append([]byte("data: "), chunk...) -} - -// codebuddyInspectImageSystemPrompt is the system constraint injected into every -// inspect_image vision sub-request (agentic path) so the vision model only -// extracts objective image details and never proposes solutions/actions. It is -// the agentic counterpart of defaultCodebuddyVisionPrompt on the preprocess path, -// and the wording is deliberately kept consistent with it. -const codebuddyInspectImageSystemPrompt = "你是一个图片细节提取助手。请只客观、准确、详细地描述图片中实际存在的内容（文字、数字、位置、颜色、图表数据等），直接回答用户针对图片的具体问题。不要提出任何解决方案、修改建议、操作步骤、分析判断或额外评论。" - -// codebuddyDumpReadToolDiagnostic emits a debug dump when the request appears to -// carry a Read-tool image workflow that the vision router did NOT recognize as an -// image input (codebuddyChatHasImageInput returned false). CodeBuddy reads images -// via its `read` tool rather than attaching them as image_url parts, so this dump -// captures the exact shape (tool_calls carrying a base64/path, or a role=tool -// message) so the router can be extended to handle it. It is a no-op unless -// CODEBUDDY_DEBUG_BODY=1. -func codebuddyDumpReadToolDiagnostic(body []byte) { - if !helps.CodebuddyDebugBodyEnabled() { - return - } - if codebuddyChatHasImageInput(body) { - return - } - if !codebuddyBodyMentionsReadTool(body) { - return - } - helps.DumpCodebuddyDebugBody("read-tool-diagnostic", body) -} - -// codebuddyBodyMentionsReadTool reports whether the body contains any trace of a -// read/read_file tool (a tool_calls entry, a tool declaration, or a role=tool -// message naming read). It is used only to gate the diagnostic dump above. -func codebuddyBodyMentionsReadTool(body []byte) bool { - if !gjson.ValidBytes(body) { - return false - } - // Top-level tool declarations. - for _, t := range gjson.GetBytes(body, "tools").Array() { - name := t.Get("function.name").String() - if name == "" { - name = t.Get("name").String() - } - if isCodebuddyReadToolName(name) { - return true - } - } - // Any message whose role is tool, or whose tool_calls name read. - for _, m := range gjson.GetBytes(body, "messages").Array() { - if m.Get("role").String() == "tool" { - return true - } - for _, tc := range m.Get("tool_calls").Array() { - name := tc.Get("function.name").String() - if name == "" { - name = tc.Get("name").String() - } - if isCodebuddyReadToolName(name) { - return true - } - } - } - return false -} - -// isCodebuddyReadToolName reports whether a tool name refers to file-reading -// (read / read_file, case-insensitive), which is how CodeBuddy inspects images. -func isCodebuddyReadToolName(name string) bool { - n := strings.ToLower(strings.TrimSpace(name)) - return n == "read" || n == "read_file" || n == "readfile" || n == "read-file" -} - -// prependCodebuddySystemMessage inserts a system message at the front of the -// request's messages array. -func prependCodebuddySystemMessage(body []byte, prompt string) ([]byte, error) { - systemJSON, err := json.Marshal(map[string]any{"role": "system", "content": prompt}) - if err != nil { - return nil, err - } - - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return sjson.SetRawBytes(body, "messages", append([]byte("["), append(systemJSON, ']')...)) - } - - var sb strings.Builder - sb.WriteByte('[') - sb.Write(systemJSON) - for _, msg := range messages.Array() { - sb.WriteByte(',') - sb.WriteString(msg.Raw) - } - sb.WriteByte(']') - return sjson.SetRawBytes(body, "messages", []byte(sb.String())) -} - -// codebuddyBackfillMaxImageBytes caps the size of a single local image file that -// the read-tool backfill will base64-encode and attach. Larger images are -// skipped (with a log) rather than bloating the request body into the tens of MB. -const codebuddyBackfillMaxImageBytes = 20 << 20 // 20MB - -// codebuddyImagePlaceholderMarkers are substrings that identify a role=tool -// content that is a placeholder for a previously-read image rather than real -// text. CodeBuddy (and its client) replace the image with a short note such as -// "[Image already analyzed in an earlier step; base64 content omitted to save -// memory. ...]" and drop the base64. The backfill detects this and re-attaches -// the image from the tool_calls filePath so the vision router can see it. -var codebuddyImagePlaceholderMarkers = []string{ - "image already analyzed", - "base64 content omitted", - "image omitted", - // Cursor's Read File V2 tool returns a bare confirmation string for image - // files (e.g. "Read image file: h:\...\home.png") instead of image data. - "read image file", -} - -// codebuddyBackfillReadToolImages detects the CodeBuddy read-tool image workflow -// — where images reach the model via the `read`/`read_file` tool whose result -// content is a placeholder (the base64 was omitted to save memory) — and, when -// the current turn has no recognizable image part, reads the image back from the -// tool_calls filePath/path and appends an image_url part to the last user -// message so the existing vision router (codebuddyChatHasImageInput) finally -// recognizes it. -// -// Data source decision (from packet capture): the role=tool content is a -// placeholder, NOT base64, so the image must be recovered from the tool_calls -// filePath. This only works when the relay runs on the same host as the client -// (the filePath is a local absolute path). On a remote/independent-server relay -// the file cannot be read and the function degrades to a no-op. -// -// It is idempotent and safe: it returns the original body unchanged unless all -// of the following hold — the body mentions a read tool, the current turn has no -// image input, and at least one read-tool placeholder maps to a readable image -// file. Failure to read/encode any single file is non-fatal. -func codebuddyBackfillReadToolImages(body []byte) []byte { - if len(body) == 0 || !gjson.ValidBytes(body) { - return body - } - // Fast short-circuit: nothing to do unless a read tool is mentioned and the - // vision router would not already see an image. - if !codebuddyBodyMentionsReadTool(body) { - return body - } - if codebuddyChatHasImageInput(body) { - return body - } - - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return body - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx < 0 { - return body - } - - // Collect read-tool image filePaths whose role=tool result is a placeholder. - // Only tool reads from the CURRENT turn (at/after the last user message) - // qualify: historical tool reads were already backfilled and described in - // their own turn, and re-attaching them to every later question injects - // stale, unrelated images (2026-09-05 Cursor incident: an anime picture - // the agent read two turns earlier kept being re-described whenever the - // user asked about a different, freshly pasted photo). - paths := collectCodebuddyReadImagePaths(arr, lastUserIdx) - if len(paths) == 0 { - return body - } - - out := body - // The last user message content may be a plain string (Cursor sends string - // content on continuation turns). sjson cannot append an array element to a - // string: `content.-1` would silently turn the string into a malformed - // {"-1": ...} object that neither the vision router nor the upstream - // accepts. Normalize non-array content into a text part first so the image - // append below yields a valid OpenAI content-part array. - contentPath := fmt.Sprintf("messages.%d.content", lastUserIdx) - if content := gjson.GetBytes(out, contentPath); content.Exists() && !content.IsArray() { - textParts, err := json.Marshal([]map[string]string{{"type": "text", "text": content.String()}}) - if err != nil { - return body - } - next, err := sjson.SetRawBytes(out, contentPath, textParts) - if err != nil { - return body - } - out = next - } - appended := 0 - for _, p := range paths { - dataURL, mimeType, ok := readCodebuddyImageAsDataURL(p) - if !ok { - continue - } - part := codebuddyImagePartJSON(dataURL) - next, err := sjson.SetRawBytes(out, fmt.Sprintf("messages.%d.content.-1", lastUserIdx), part) - if err != nil { - log.Warnf("codebuddy vision backfill: append image_url for %s failed: %v", p, err) - return body - } - out = next - appended++ - log.Infof("codebuddy vision backfill: attached image %s (%s, %d bytes) to last user message", p, mimeType, len(dataURL)) - } - if appended == 0 { - return body - } - return out -} - -// collectCodebuddyReadImagePaths scans assistant tool_calls for read/read_file -// invocations whose arguments carry a filePath/path, and whose corresponding -// role=tool result content is an image placeholder. Only those pairs yield an -// image path. tool_call_id is matched between the assistant tool_calls entry and -// the following role=tool message (falling back to order-based matching when IDs -// are absent). The result preserves body order and de-duplicates paths. -// -// Assistant messages before minAssistantIdx (i.e. before the current turn's -// last user message) are ignored: their images were already backfilled and -// described in their own turn, and re-attaching them now would inject stale, -// unrelated pictures into the user's latest question. -func collectCodebuddyReadImagePaths(messages []gjson.Result, minAssistantIdx int) []string { - type pending struct { - id string - path string - } - var pendings []pending - seenIDs := map[string]bool{} - order := []string{} - - for mi, m := range messages { - role := m.Get("role").String() - switch role { - case "assistant": - if mi < minAssistantIdx { - // Historical tool read: belongs to an earlier turn, do not - // re-inject its image into the current question. - continue - } - for _, tc := range m.Get("tool_calls").Array() { - name := tc.Get("function.name").String() - if name == "" { - name = tc.Get("name").String() - } - if !isCodebuddyReadToolName(name) { - continue - } - args := tc.Get("function.arguments").String() - if args == "" { - args = tc.Get("arguments").String() - } - if p := extractCodebuddyToolFilePath(args); p != "" { - id := tc.Get("id").String() - pendings = append(pendings, pending{id: id, path: p}) - } - } - case "tool": - if len(pendings) == 0 { - continue - } - if !isCodebuddyImagePlaceholder(m.Get("content").String()) { - continue - } - // Match by tool_call_id when available, else consume in order. - tcID := m.Get("tool_call_id").String() - if tcID != "" { - for _, pd := range pendings { - if pd.id == tcID { - if !seenIDs[pd.id] { - seenIDs[pd.id] = true - order = append(order, pd.path) - } - break - } - } - continue - } - // No ID on the tool message: consume the oldest unmatched pending. - for _, pd := range pendings { - if !seenIDs[pd.id] { - seenIDs[pd.id] = true - order = append(order, pd.path) - break - } - } - } - } - return order -} - -// extractCodebuddyToolFilePath parses the JSON arguments of a read tool call and -// returns the file path from filePath (CodeBuddy read_file) or path (older Read -// tool). It tolerates malformed JSON by falling back to a substring scan. -func extractCodebuddyToolFilePath(args string) string { - args = strings.TrimSpace(args) - if args == "" { - return "" - } - if gjson.Valid(args) { - if p := gjson.Get(args, "filePath").String(); p != "" { - return strings.TrimSpace(p) - } - if p := gjson.Get(args, "path").String(); p != "" { - return strings.TrimSpace(p) - } - } - // Fallback: scan for a quoted filePath/path key. - for _, key := range []string{`"filePath"`, `"path"`} { - idx := strings.Index(args, key) - if idx < 0 { - continue - } - rest := args[idx+len(key):] - colon := strings.Index(rest, ":") - if colon < 0 { - continue - } - rest = rest[colon+1:] - q := strings.Index(rest, `"`) - if q < 0 { - continue - } - end := strings.Index(rest[q+1:], `"`) - if end < 0 { - continue - } - return strings.TrimSpace(rest[q+1 : q+1+end]) - } - return "" -} - -// isCodebuddyImagePlaceholder reports whether a role=tool content string looks -// like a placeholder for a previously-read image (rather than real text output). -func isCodebuddyImagePlaceholder(content string) bool { - lower := strings.ToLower(content) - for _, marker := range codebuddyImagePlaceholderMarkers { - if strings.Contains(lower, marker) { - return true - } - } - return false -} - -// readCodebuddyImageAsDataURL reads a local image file, detects its MIME type -// from the file extension, base64-encodes its contents, and returns the data URL -// plus the detected MIME type. It reports ok=false on any failure (missing file, -// oversized file, read error, unsupported/unknown extension) so the caller can -// degrade to leaving the request unchanged. -func readCodebuddyImageAsDataURL(path string) (dataURL, mimeType string, ok bool) { - info, err := os.Stat(path) - if err != nil { - log.Warnf("codebuddy vision backfill: image file not readable %s: %v", path, err) - return "", "", false - } - if info.IsDir() { - return "", "", false - } - if info.Size() > codebuddyBackfillMaxImageBytes { - log.Warnf("codebuddy vision backfill: image %s too large (%d bytes > %d), skipping", path, info.Size(), codebuddyBackfillMaxImageBytes) - return "", "", false - } - - ext := strings.ToLower(filepath.Ext(path)) - mt := mime.TypeByExtension(ext) - if mt == "" || !strings.HasPrefix(mt, "image/") { - log.Warnf("codebuddy vision backfill: unsupported image extension %q for %s, skipping", ext, path) - return "", "", false - } - - raw, err := os.ReadFile(path) - if err != nil { - log.Warnf("codebuddy vision backfill: read image %s failed: %v", path, err) - return "", "", false - } - return "data:" + mt + ";base64," + base64.StdEncoding.EncodeToString(raw), mt, true -} - -// codebuddyImagePartJSON renders the canonical image_url part used by the -// backend and by the vision router (see normalizeCodebuddyImagePart). -func codebuddyImagePartJSON(dataURL string) []byte { - part, _ := json.Marshal(map[string]any{ - "type": "image_url", - "image_url": map[string]any{"url": dataURL}, - }) - return part -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_agentic.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_agentic.go deleted file mode 100644 index 7e55abb..0000000 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_agentic.go +++ /dev/null @@ -1,889 +0,0 @@ -package executor - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - - "github.com/router-for-me/CLIProxyAPI/v7/internal/auth/codebuddy" - "github.com/router-for-me/CLIProxyAPI/v7/internal/config" - "github.com/router-for-me/CLIProxyAPI/v7/internal/runtime/executor/helps" - cliproxyauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth" - cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/executor" - "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/usage" - sdktranslator "github.com/router-for-me/CLIProxyAPI/v7/sdk/translator" - log "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" -) - -// inspectImageToolName is the tool injected into the text-only model so it can -// autonomously query the vision model for image details during reasoning. -const inspectImageToolName = "inspect_image" - -// codebuddyAgenticImageRef holds an extracted image's original content part, -// kept in memory so inspect_image can re-send it to the vision model on demand. -type codebuddyAgenticImageRef struct { - id int - partJSON []byte // {"type":"image_url","image_url":{...}} raw part -} - -// codebuddyVisionAgenticEnabled reports whether the agentic vision-proxy mode -// is active. -func (e *CodebuddyExecutor) codebuddyVisionAgenticEnabled() bool { - return e.cfg.CodebuddyVision.NormalizedVisionMode() == config.CodebuddyVisionModeAgentic -} - -// inspectImageToolDef returns the OpenAI function-tool definition injected into -// the request so the text-only model can inspect attached images. -func inspectImageToolDef() map[string]any { - return map[string]any{ - "type": "function", - "function": map[string]any{ - "name": inspectImageToolName, - "description": "查看已附加图片的细节。当你需要确认图片中的具体内容（文字、数字、位置、颜色、图表数据等）时调用。可多次调用以查看不同细节。", - "parameters": map[string]any{ - "type": "object", - "properties": map[string]any{ - "image_id": map[string]any{ - "type": "integer", - "description": "图片编号，从 1 开始。", - }, - "question": map[string]any{ - "type": "string", - "description": "针对该图片的具体问题，例如“图片右上角的文字是什么”。", - }, - }, - "required": []string{"image_id", "question"}, - }, - }, - } -} - -// extractCodebuddyImagesForAgentic rewrites the body so no image part reaches a -// text-only model: images in the current turn (the last user message and any -// subsequent assistant/tool messages) are extracted as inspect_image targets, -// while images in earlier messages (stale history re-sent by the client) are -// replaced with a neutral placeholder. -func extractCodebuddyImagesForAgentic(body []byte) ([]byte, []codebuddyAgenticImageRef, error) { - messages := gjson.GetBytes(body, "messages") - if !messages.IsArray() { - return body, nil, nil - } - arr := messages.Array() - lastUserIdx := lastCodebuddyUserMessageIndex(arr) - if lastUserIdx < 0 { - return body, nil, nil - } - - out := body - var images []codebuddyAgenticImageRef - for mi, msg := range arr { - content := msg.Get("content") - if !content.IsArray() { - continue - } - isCurrent := mi >= lastUserIdx - for ci, part := range content.Array() { - if !isCodebuddyImagePartType(part.Get("type").String()) { - continue - } - path := fmt.Sprintf("messages.%d.content.%d", mi, ci) - - var replacement []byte - if isCurrent { - id := len(images) + 1 - images = append(images, codebuddyAgenticImageRef{ - id: id, - partJSON: append([]byte(nil), []byte(part.Raw)...), - }) - text := fmt.Sprintf("[图片 #%d 已附加，可用 inspect_image 工具查看，image_id=%d]", id, id) - var err error - replacement, err = json.Marshal(map[string]string{"type": "text", "text": text}) - if err != nil { - return body, nil, err - } - } else { - replacement, _ = json.Marshal(map[string]string{"type": "text", "text": codebuddyHistoricalImageText}) - } - - var err error - out, err = sjson.SetRawBytes(out, path, replacement) - if err != nil { - return body, nil, err - } - } - } - return out, images, nil -} - -// injectCodebuddyInspectTool prepends a system guidance message and injects the -// inspect_image tool definition into the request body. -func injectCodebuddyInspectTool(body []byte, imageCount int) []byte { - guide := fmt.Sprintf( - "用户消息中附带了 %d 张图片，但图片内容已从消息中移除，你无法直接看到。你需要使用 inspect_image 工具查看图片细节。当回答涉及图片具体内容（文字、数字、位置、颜色、图表数据等）时，请先调用 inspect_image 工具获取细节，再作答。你可以多次调用该工具查看不同细节。", - imageCount, - ) - out, err := prependCodebuddySystemMessage(body, guide) - if err != nil { - out = body - } - - toolsJSON, err := json.Marshal([]any{inspectImageToolDef()}) - if err != nil { - return out - } - out, err = sjson.SetRawBytes(out, "tools", toolsJSON) - if err != nil { - return body - } - // The client may have sent a `tool_choice` naming one of its own tools (or - // "required"). Since `tools` was just replaced with the single inspect_image - // tool, a stale tool_choice is now inconsistent and can be rejected by the - // strict backend with 400 — reset it to "auto". - out, err = sjson.SetBytes(out, "tool_choice", "auto") - if err != nil { - return body - } - return out -} - -// appendAgenticMessage appends a raw JSON message to the messages array. -func appendAgenticMessage(body []byte, msgRaw []byte) ([]byte, error) { - return sjson.SetRawBytes(body, "messages.-1", msgRaw) -} - -// doCodebuddyChatRequest performs a single forced-stream chat completion and -// aggregates it into a non-streaming ChatCompletion JSON. It returns the -// aggregated payload and the upstream response headers. -func (e *CodebuddyExecutor) doCodebuddyChatRequest( - ctx context.Context, - auth *cliproxyauth.Auth, - creds codebuddy.Creds, - baseURL string, - body []byte, -) ([]byte, http.Header, error) { - var err error - body, err = sjson.SetBytes(body, "stream", true) - if err != nil { - return nil, nil, err - } - body, err = sjson.SetBytes(body, "stream_options.include_usage", true) - if err != nil { - return nil, nil, err - } - // Clamp oversized max_tokens (Cursor sends 65536) to the model's declared - // ceiling, matching the normal request path. - body = clampCodebuddyMaxTokens(body, gjson.GetBytes(body, "model").String()) - - // Normalize tool-related message fields so the strict backend does not - // reject accumulated tool-calling rounds with 400 invalid_parameter_value, - // matching the normal request path. - body, err = normalizeCodebuddyToolMessages(body) - if err != nil { - return nil, nil, err - } - - url := baseURL + codebuddy.ChatPath - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) - if err != nil { - return nil, nil, err - } - applyCodebuddyHeaders(httpReq, creds) - httpReq.Header.Set("Accept", "text/event-stream") - - // Diagnostic: dump the agentic sub-request body (redacted) so the exact - // tools/tool_choice/max_tokens the loop sends can be inspected on failure. - helps.DumpCodebuddyDebugBody("agentic-request", body) - - httpClient := helps.NewProxyAwareHTTPClient(ctx, e.cfg, auth, 0) - httpResp, err := httpClient.Do(httpReq) - if err != nil { - return nil, nil, err - } - defer func() { _ = httpResp.Body.Close() }() - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - b, _ := io.ReadAll(httpResp.Body) - helps.DumpCodebuddyDebugBody("agentic-error", b) - log.Warnf("codebuddy vision agentic: round request failed status=%d body=%s", - httpResp.StatusCode, summarize(b)) - return nil, httpResp.Header.Clone(), statusErr{code: codebuddyEffectiveStatus(httpResp.StatusCode, b), msg: string(b)} - } - - lines := make([][]byte, 0, 64) - scanner := bufio.NewScanner(httpResp.Body) - scanner.Buffer(nil, 52_428_800) - for scanner.Scan() { - lines = append(lines, bytes.Clone(scanner.Bytes())) - } - if errScan := scanner.Err(); errScan != nil { - return nil, httpResp.Header.Clone(), errScan - } - return collectChatCompletion(lines), httpResp.Header.Clone(), nil -} - -// doCodebuddyChatRequestWithEmit is doCodebuddyChatRequest with an optional -// streaming sink: as each content delta is scanned, it is forwarded through -// emit (if non-nil). The aggregated ChatCompletion is still returned for the -// caller. This lets the vision sub-agent's answer stream to the client in real -// time instead of being buffered until the round completes. baseModel is used -// to rewrite the forwarded chunk's model field so the client always sees the -// requested model rather than the vision model's backend engine name. -func (e *CodebuddyExecutor) doCodebuddyChatRequestWithEmit( - ctx context.Context, - auth *cliproxyauth.Auth, - creds codebuddy.Creds, - baseURL string, - body []byte, - baseModel string, - emit func([]byte) bool, -) ([]byte, http.Header, error) { - var err error - body, err = sjson.SetBytes(body, "stream", true) - if err != nil { - return nil, nil, err - } - body, err = sjson.SetBytes(body, "stream_options.include_usage", true) - if err != nil { - return nil, nil, err - } - // Clamp oversized max_tokens (Cursor sends 65536) to the model's declared - // ceiling, matching the normal request path. - body = clampCodebuddyMaxTokens(body, gjson.GetBytes(body, "model").String()) - - // Normalize tool-related message fields so the strict backend does not - // reject accumulated tool-calling rounds with 400 invalid_parameter_value, - // matching the normal request path. - body, err = normalizeCodebuddyToolMessages(body) - if err != nil { - return nil, nil, err - } - - url := baseURL + codebuddy.ChatPath - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) - if err != nil { - return nil, nil, err - } - applyCodebuddyHeaders(httpReq, creds) - httpReq.Header.Set("Accept", "text/event-stream") - - // Diagnostic: dump the agentic sub-request body (redacted) so the exact - // tools/tool_choice/max_tokens the loop sends can be inspected on failure. - helps.DumpCodebuddyDebugBody("agentic-request", body) - - httpClient := helps.NewProxyAwareHTTPClient(ctx, e.cfg, auth, 0) - httpResp, err := httpClient.Do(httpReq) - if err != nil { - return nil, nil, err - } - defer func() { _ = httpResp.Body.Close() }() - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - b, _ := io.ReadAll(httpResp.Body) - helps.DumpCodebuddyDebugBody("agentic-error", b) - log.Warnf("codebuddy vision agentic: round request failed status=%d body=%s", - httpResp.StatusCode, summarize(b)) - return nil, httpResp.Header.Clone(), statusErr{code: codebuddyEffectiveStatus(httpResp.StatusCode, b), msg: string(b)} - } - - lines := make([][]byte, 0, 64) - scanner := bufio.NewScanner(httpResp.Body) - scanner.Buffer(nil, 52_428_800) - for scanner.Scan() { - rawLine := scanner.Bytes() - // Diagnostic: dump the vision sub-model's raw SSE line (redacted) so the - // exact field placement of thinking (reasoning_content vs content) and - // the usage token shape can be confirmed. This dump intentionally uses the - // raw line so it is unaffected by stripping. - helps.DumpCodebuddyDebugBody("vision-raw-sse", rawLine) - - // Strip thinking from the line before it enters either the aggregation - // buffer (collectChatCompletion) or the client-forwarding path, so any - // thinking that landed inside delta.content never reaches the text-only - // model's tool result nor the client. - cleaned, drop := stripVisionThinkingLine(rawLine) - if drop { - continue - } - lines = append(lines, cleaned) - - if emit != nil { - // Forward only visible content deltas to the client. role/reasoning/ - // usage/finish lines are skipped so the client only sees answer text. - trimmed := bytes.TrimSpace(cleaned) - if bytes.HasPrefix(trimmed, []byte("data:")) { - payload := bytes.TrimSpace(trimmed[len("data:"):]) - if len(payload) > 0 && !bytes.Equal(payload, []byte("[DONE]")) && gjson.ValidBytes(payload) { - for _, ch := range gjson.GetBytes(payload, "choices").Array() { - delta := ch.Get("delta") - if c := delta.Get("content"); c.Exists() && c.Type == gjson.String && c.String() != "" { - // Rewrite the chunk's model field to baseModel so the - // client sees a single consistent model name. - outLine := trimmed - if baseModel != "" { - if rewritten, errSet := sjson.SetBytes(payload, "model", baseModel); errSet == nil { - outLine = append([]byte("data: "), rewritten...) - } - } - if !emit(outLine) { - return nil, httpResp.Header.Clone(), ctx.Err() - } - break - } - } - } - } - } - } - if errScan := scanner.Err(); errScan != nil { - return nil, httpResp.Header.Clone(), errScan - } - return collectChatCompletion(lines), httpResp.Header.Clone(), nil -} - -// stripVisionThinkingLine cleans a single vision sub-model SSE line so that any -// thinking text that landed inside delta.content (as opposed to the standard -// delta.reasoning_content field) is removed before the line is either forwarded -// to the client or aggregated by collectChatCompletion. -// -// It returns: -// - cleaned: the line with thinking-only content deltas emptied out. When no -// stripping applies, cleaned is a clone of line so the caller can buffer it -// without aliasing scanner memory. -// - drop: true when the whole line carries nothing but thinking and should be -// discarded entirely (no visible content, no structural fields worth keeping). -// -// The function is a pure transform with no network/context dependency, so its -// rules are exhaustively unit-testable. -// -// Rules (in priority order): -// - A. A delta carrying non-empty reasoning_content and no non-empty content is -// thinking-only and is dropped. (This mirrors the existing emit guard.) -// - B. A delta with empty content is left structurally intact (finish_reason, -// usage, etc. survive) but contributes no visible text. -// - C. content text is passed through stripThinkingMarkersFromContent, a -// conservative marker table that is currently empty (defensive extension -// point only). hy3-preview reports thinking via reasoning_content, so no -// content-level marker has been confirmed; we do NOT guess to avoid stripping -// legitimate image descriptions. -func stripVisionThinkingLine(line []byte) (cleaned []byte, drop bool) { - trimmed := bytes.TrimSpace(line) - if !bytes.HasPrefix(trimmed, []byte("data:")) { - return bytes.Clone(line), false - } - payload := bytes.TrimSpace(trimmed[len("data:"):]) - if len(payload) == 0 || bytes.Equal(payload, []byte("[DONE]")) || !gjson.ValidBytes(payload) { - return bytes.Clone(line), false - } - - choices := gjson.GetBytes(payload, "choices") - if !choices.IsArray() || len(choices.Array()) == 0 { - return bytes.Clone(line), false - } - - changed := false - hasVisibleContent := false - for i, ch := range choices.Array() { - delta := ch.Get("delta") - reasoning := delta.Get("reasoning_content") - hasReasoning := reasoning.Exists() && reasoning.Type == gjson.String && reasoning.String() != "" - - content := delta.Get("content") - hasContent := content.Exists() && content.Type == gjson.String && content.String() != "" - - if hasContent { - // Rule C: strip confirmed thinking markers from content (currently a - // no-op table; see stripThinkingMarkersFromContent). - stripped := stripThinkingMarkersFromContent(content.String()) - if stripped != content.String() { - changed = true - payload, _ = sjson.SetBytes(payload, fmt.Sprintf("choices.%d.delta.content", i), stripped) - } - if strings.TrimSpace(stripped) != "" { - hasVisibleContent = true - } - } else if hasReasoning { - // Rule A: reasoning-only delta (no visible content) → mark for drop. - changed = true - } - } - - if !hasVisibleContent { - // The line holds only thinking (reasoning-only deltas and/or content that - // emptied out after stripping) — nothing worth keeping. - return nil, true - } - if !changed { - return bytes.Clone(line), false - } - return append([]byte("data: "), payload...), false -} - -// stripThinkingMarkersFromContent removes thinking wrapper markers from a content -// string. It is intentionally conservative: only text bounded by an explicitly -// listed marker pair is stripped, so normal image-description text is never -// touched. -// -// The marker table is currently empty. hy3-preview reports thinking via the -// dedicated reasoning_content field, so no content-level marker has been -// confirmed; guessing would risk stripping legitimate answer text. Add a marker -// pair here once a real wrapper is captured in the vision-raw-sse diagnostic -// dump. -func stripThinkingMarkersFromContent(content string) string { - type markerPair struct{ open, close string } - - // Known thinking wrappers observed across reasoning-capable backends. A pair - // with an empty close acts as a sentinel prefix: everything up to and - // including the sentinel is dropped, the remainder kept. - markers := []markerPair{ - // Example once confirmed (do not enable without a real dump): - // {"", ""}, - } - - result := content - for _, m := range markers { - if m.close == "" { - if idx := strings.Index(result, m.open); idx >= 0 { - result = result[idx+len(m.open):] - } - continue - } - for { - start := strings.Index(result, m.open) - if start < 0 { - break - } - end := strings.Index(result[start+len(m.open):], m.close) - if end < 0 { - break - } - end += start + len(m.open) + len(m.close) - result = result[:start] + result[end:] - } - } - return result -} - -// inspectCodebuddyImage asks the vision model a specific question about a single -// image, returning the model's answer text. -func (e *CodebuddyExecutor) inspectCodebuddyImage( - ctx context.Context, - auth *cliproxyauth.Auth, - creds codebuddy.Creds, - baseURL string, - imagePart []byte, - question string, - visionModel string, - baseModel string, -) (string, usage.Detail, error) { - return e.inspectCodebuddyImageWithEmit(ctx, auth, creds, baseURL, imagePart, question, visionModel, baseModel, nil) -} - -// inspectCodebuddyImageWithEmit is inspectCodebuddyImage with an optional -// streaming sink that receives the vision model's content deltas as they arrive. -// baseModel is used to rewrite the forwarded chunk's model field. -func (e *CodebuddyExecutor) inspectCodebuddyImageWithEmit( - ctx context.Context, - auth *cliproxyauth.Auth, - creds codebuddy.Creds, - baseURL string, - imagePart []byte, - question string, - visionModel string, - baseModel string, - emit func([]byte) bool, -) (string, usage.Detail, error) { - userContent := []any{ - json.RawMessage(imagePart), - map[string]any{"type": "text", "text": question}, - } - body := map[string]any{ - "model": visionModel, - "messages": []any{ - map[string]any{"role": "user", "content": userContent}, - }, - } - // Disable reasoning on the vision sub-request so the model returns only the - // image description text. Reasoning-capable vision models (e.g. hy3-preview) - // otherwise emit thinking content that leaks into the forwarded delta and, - // worse, gets stored as the tool result the text-only model consumes — which - // makes the text-only model believe the image was never actually described. - body["reasoning_effort"] = "none" - bodyJSON, err := json.Marshal(body) - if err != nil { - return "", usage.Detail{}, err - } - - // Inject the system constraint so the vision model only extracts objective - // image details and never proposes solutions/actions. This mirrors the - // preprocess path's defaultCodebuddyVisionPrompt and keeps both paths - // consistent about "describe, don't advise". - bodyJSON, err = prependCodebuddySystemMessage(bodyJSON, codebuddyInspectImageSystemPrompt) - if err != nil { - return "", usage.Detail{}, err - } - - aggregated, _, err := e.doCodebuddyChatRequestWithEmit(ctx, auth, creds, baseURL, bodyJSON, baseModel, emit) - if err != nil { - return "", usage.Detail{}, err - } - // Take only the visible answer text. reasoning_content (thinking) is stored - // separately by collectChatCompletion and must never become the tool result: - // if a reasoning-capable vision model returns only thinking and no answer, we - // treat it as an empty answer rather than feeding the thinking back to the - // text-only model (which would make it conclude the image was not described). - content := strings.TrimSpace(gjson.GetBytes(aggregated, "choices.0.message.content").String()) - if content == "" { - return "", helps.ParseOpenAIUsage(aggregated), fmt.Errorf("vision model returned empty answer") - } - return content, helps.ParseOpenAIUsage(aggregated), nil -} - -// runAgenticLoop runs the server-side tool-calling loop: repeatedly send the -// accumulated messages to the text-only model, intercept inspect_image tool -// calls, answer them via the vision model, and continue until the model stops -// calling tools or the round limit is reached. It returns the final aggregated -// ChatCompletion JSON and the last upstream response headers. -func (e *CodebuddyExecutor) runAgenticLoop( - ctx context.Context, - auth *cliproxyauth.Auth, - creds codebuddy.Creds, - baseURL string, - body []byte, - images []codebuddyAgenticImageRef, - visionModel string, - baseModel string, - maxRounds int, - reporter *helps.UsageReporter, - heartbeat func(), - emit func([]byte) bool, -) ([]byte, http.Header, error) { - var lastAggregated []byte - var lastHeaders http.Header - var mainUsage usage.Detail - var visionUsage usage.Detail - - // beat emits a downstream keep-alive if the caller supplied one. It is nil - // safe so the non-streaming path can pass nil. - beat := func() { - if heartbeat != nil { - heartbeat() - } - } - - // Publish the usage of the agentic loop exactly once per model, regardless - // of which return path terminates the loop. The text-only model's usage is - // published as the main record, and the vision sub-model's usage as a - // separate additional-model record so both credit/token sets are visible. - defer func() { - if reporter != nil { - helps.DumpCodebuddyDebugBody("vision-reporter-publish", - []byte(fmt.Sprintf("model=%s inputTokens=%d outputTokens=%d credit=%v visionInputTokens=%d visionOutputTokens=%d visionCredit=%v", - reporter.Model(), mainUsage.InputTokens, mainUsage.OutputTokens, mainUsage.Credit, - visionUsage.InputTokens, visionUsage.OutputTokens, visionUsage.Credit))) - reporter.Publish(ctx, mainUsage) - reporter.PublishAdditionalModelAlways(ctx, visionModel, visionUsage) - } - }() - - for round := 0; round < maxRounds; round++ { - beat() - aggregated, headers, err := e.doCodebuddyChatRequest(ctx, auth, creds, baseURL, body) - beat() - if err != nil { - if lastAggregated != nil { - return lastAggregated, lastHeaders, nil - } - return nil, nil, err - } - lastAggregated = aggregated - lastHeaders = headers - - // Accumulate the text-only model's usage into the main record. - addCodebuddyAgenticUsage(&mainUsage, helps.ParseOpenAIUsage(aggregated)) - - toolCalls := gjson.GetBytes(aggregated, "choices.0.message.tool_calls") - if !toolCalls.IsArray() || len(toolCalls.Array()) == 0 { - // No tool calls: this is the final answer. - return aggregated, headers, nil - } - - // Append the assistant message (carrying tool_calls) first. - assistantRaw := gjson.GetBytes(aggregated, "choices.0.message").Raw - body, err = appendAgenticMessage(body, []byte(assistantRaw)) - if err != nil { - return aggregated, headers, nil - } - - handled := false - for _, tc := range toolCalls.Array() { - fn := tc.Get("function") - if fn.Get("name").String() != inspectImageToolName { - continue - } - handled = true - toolCallID := tc.Get("id").String() - argsStr := fn.Get("arguments").String() - argsParsed := gjson.Parse(argsStr) - imageID := int(argsParsed.Get("image_id").Int()) - question := argsParsed.Get("question").String() - - var answer string - if imageID >= 1 && imageID <= len(images) { - beat() - a, inspectUsage, inspectErr := e.inspectCodebuddyImageWithEmit( - ctx, auth, creds, baseURL, images[imageID-1].partJSON, question, visionModel, baseModel, emit, - ) - beat() - if inspectErr != nil { - answer = fmt.Sprintf("[图片查看失败: %v]", inspectErr) - log.Warnf("codebuddy vision agentic: inspect_image(%d) failed: %v", imageID, inspectErr) - } else { - answer = a - // Accumulate the vision sub-model usage into its own record. - addCodebuddyAgenticUsage(&visionUsage, inspectUsage) - } - } else { - answer = fmt.Sprintf("[无效的图片编号 %d，可用范围 1-%d]", imageID, len(images)) - } - - toolMsgJSON, err := json.Marshal(map[string]any{ - "role": "tool", - "tool_call_id": toolCallID, - "content": answer, - }) - if err != nil { - continue - } - body, err = appendAgenticMessage(body, toolMsgJSON) - if err != nil { - return aggregated, headers, nil - } - } - - if !handled { - // Tool calls present but none is inspect_image: return as-is. - return aggregated, headers, nil - } - } - - // Round limit reached: return the last accumulated result. - if lastAggregated != nil { - return lastAggregated, lastHeaders, nil - } - return nil, nil, fmt.Errorf("codebuddy vision agentic: no result after %d rounds", maxRounds) -} - -// addCodebuddyAgenticUsage accumulates a single round's usage into the running -// total for the vision sub-agent loop. -func addCodebuddyAgenticUsage(total *usage.Detail, add usage.Detail) { - if total == nil { - return - } - total.InputTokens += add.InputTokens - total.OutputTokens += add.OutputTokens - total.ReasoningTokens += add.ReasoningTokens - total.CachedTokens += add.CachedTokens - total.CacheReadTokens += add.CacheReadTokens - total.CacheCreationTokens += add.CacheCreationTokens - total.TotalTokens += add.TotalTokens - total.Credit += add.Credit - total.TokenBreakdown.TotalTokens += add.TokenBreakdown.TotalTokens - total.TokenBreakdown.Input.TotalTokens += add.TokenBreakdown.Input.TotalTokens - total.TokenBreakdown.Input.UncachedTokens += add.TokenBreakdown.Input.UncachedTokens - total.TokenBreakdown.Input.CacheReadTokens += add.TokenBreakdown.Input.CacheReadTokens - total.TokenBreakdown.Input.CacheWriteTokens += add.TokenBreakdown.Input.CacheWriteTokens - total.TokenBreakdown.Output.TotalTokens += add.TokenBreakdown.Output.TotalTokens - total.TokenBreakdown.Output.NonReasoningTokens += add.TokenBreakdown.Output.NonReasoningTokens - total.TokenBreakdown.Output.ReasoningTokens += add.TokenBreakdown.Output.ReasoningTokens - total.TokenBreakdown.UnclassifiedTokens += add.TokenBreakdown.UnclassifiedTokens -} - -// executeCodebuddyVisionAgentic runs the agentic loop for non-streaming Execute -// and translates the final aggregated payload back to the client format. -func (e *CodebuddyExecutor) executeCodebuddyVisionAgentic( - ctx context.Context, - auth *cliproxyauth.Auth, - req cliproxyexecutor.Request, - opts cliproxyexecutor.Options, - body []byte, - baseModel string, - creds codebuddy.Creds, - baseURL string, - reporter *helps.UsageReporter, -) (cliproxyexecutor.Response, error) { - visionCfg := e.cfg.CodebuddyVision - visionModel := visionCfg.VisionModel() - maxRounds := visionCfg.MaxVisionToolRounds() - - body, images, err := extractCodebuddyImagesForAgentic(body) - if err != nil { - return cliproxyexecutor.Response{}, err - } - if len(images) == 0 { - return cliproxyexecutor.Response{}, fmt.Errorf("codebuddy vision agentic: no images to inspect") - } - body = injectCodebuddyInspectTool(body, len(images)) - - log.Infof("codebuddy vision agentic: %d images, model=%s, vision=%s, maxRounds=%d", - len(images), baseModel, visionModel, maxRounds) - - aggregated, headers, err := e.runAgenticLoop(ctx, auth, creds, baseURL, body, images, visionModel, baseModel, maxRounds, reporter, nil, nil) - if err != nil { - return cliproxyexecutor.Response{}, err - } - - respFrom := sdktranslator.FromString("openai") - respTo := opts.SourceFormat - var param any - out := sdktranslator.TranslateNonStream(ctx, respFrom, respTo, req.Model, opts.OriginalRequest, body, aggregated, ¶m) - return cliproxyexecutor.Response{Payload: out, Headers: headers}, nil -} - -// executeCodebuddyVisionAgenticStream runs the agentic loop in a background -// goroutine and returns immediately, emitting an initial chunk first so the -// relay's stream-open watchdog (default 10s) is satisfied even though the loop -// itself may take 10-30s (multiple deepseek + hy3 round-trips). -func (e *CodebuddyExecutor) executeCodebuddyVisionAgenticStream( - ctx context.Context, - auth *cliproxyauth.Auth, - req cliproxyexecutor.Request, - opts cliproxyexecutor.Options, - body []byte, - baseModel string, - creds codebuddy.Creds, - baseURL string, - reporter *helps.UsageReporter, -) (*cliproxyexecutor.StreamResult, error) { - visionCfg := e.cfg.CodebuddyVision - visionModel := visionCfg.VisionModel() - maxRounds := visionCfg.MaxVisionToolRounds() - - // Extract images + inject tool synchronously (fast, <1s). - body, images, err := extractCodebuddyImagesForAgentic(body) - if err != nil { - return nil, err - } - if len(images) == 0 { - return nil, fmt.Errorf("codebuddy vision agentic: no images to inspect") - } - body = injectCodebuddyInspectTool(body, len(images)) - - log.Infof("codebuddy vision agentic (stream): %d images, model=%s, vision=%s, maxRounds=%d", - len(images), baseModel, visionModel, maxRounds) - - respFrom := sdktranslator.FromString("openai") - respTo := opts.SourceFormat - - out := make(chan cliproxyexecutor.StreamChunk) - go func() { - defer close(out) - var param any - emit := func(line []byte) bool { - chunks := sdktranslator.TranslateStream(ctx, respFrom, respTo, req.Model, opts.OriginalRequest, body, line, ¶m) - for _, c := range chunks { - select { - case out <- cliproxyexecutor.StreamChunk{Payload: c}: - case <-ctx.Done(): - return false - } - } - return true - } - - // 1. Open the stream immediately (role delta) so the relay watchdog - // does not time out while the loop runs. - initChunk := buildCodebuddyVisionChunk("", baseModel, 0, nil, "assistant") - if initChunk != nil && !emit(initChunk) { - return - } - - // heartbeat keeps the downstream SSE connection alive while the agentic - // loop performs its multi-round (10-30s) upstream calls without emitting - // any visible content. It sends an SSE comment keep-alive so the relay's - // stream idle watchdog does not trip mid-loop. - heartbeat := func() { - select { - case out <- cliproxyexecutor.StreamChunk{Payload: []byte(": keep-alive\n\n")}: - case <-ctx.Done(): - } - } - - // 2. Run the loop (blocking; may take 10-30s). The vision sub-agent's - // inspect_image answers are streamed to the client in real time via - // inspectEmit, replacing the old post-loop pseudo-replay. - inspectEmit := func(line []byte) bool { - // Forward the vision model's raw content delta through the same - // translator/emit path so the client sees it as assistant content. - return emit(line) - } - aggregated, _, loopErr := e.runAgenticLoop(ctx, auth, creds, baseURL, body, images, visionModel, baseModel, maxRounds, reporter, heartbeat, inspectEmit) - if loopErr != nil || aggregated == nil { - // Surface the failure as visible assistant content instead of a - // silent empty response, and log the full error for diagnosis. - errText := "codebuddy 视觉代理失败，未能获取图片描述" - if loopErr != nil { - errText = fmt.Sprintf("codebuddy 视觉代理失败: %s", summarize([]byte(loopErr.Error()))) - log.Errorf("codebuddy vision agentic (stream): loop failed: %v", loopErr) - } else { - log.Errorf("codebuddy vision agentic (stream): loop returned no result") - } - errChunk, errJSON := json.Marshal(map[string]any{ - "id": "", "object": "chat.completion.chunk", "created": 0, "model": baseModel, - "choices": []any{ - map[string]any{"index": 0, "delta": map[string]any{"content": errText}, "finish_reason": nil}, - }, - }) - if errJSON == nil { - if !emit(append([]byte("data: "), errChunk...)) { - return - } - } - emit([]byte("data: [DONE]")) - return - } - - // 3. Replay the final content pseudo-streamed. - id := gjson.GetBytes(aggregated, "id").String() - model := gjson.GetBytes(aggregated, "model").String() - created := gjson.GetBytes(aggregated, "created").Int() - content := gjson.GetBytes(aggregated, "choices.0.message.content").String() - - runes := []rune(content) - const chunkSize = 8 - for i := 0; i < len(runes); i += chunkSize { - end := i + chunkSize - if end > len(runes) { - end = len(runes) - } - delta := string(runes[i:end]) - chunkJSON, err := json.Marshal(map[string]any{ - "id": id, "object": "chat.completion.chunk", "created": created, "model": model, - "choices": []any{ - map[string]any{"index": 0, "delta": map[string]any{"content": delta}, "finish_reason": nil}, - }, - }) - if err != nil { - continue - } - if !emit(append([]byte("data: "), chunkJSON...)) { - return - } - } - - // Terminal finish chunk. - finishJSON, _ := json.Marshal(map[string]any{ - "id": id, "object": "chat.completion.chunk", "created": created, "model": model, - "choices": []any{ - map[string]any{"index": 0, "delta": map[string]any{}, "finish_reason": "stop"}, - }, - }) - emit(append([]byte("data: "), finishJSON...)) - emit([]byte("data: [DONE]")) - }() - return &cliproxyexecutor.StreamResult{Chunks: out}, nil -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_agentic_test.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_agentic_test.go deleted file mode 100644 index afd5790..0000000 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_agentic_test.go +++ /dev/null @@ -1,212 +0,0 @@ -package executor - -import ( - "encoding/json" - "strings" - "testing" - - "github.com/tidwall/gjson" -) - -// TestCollectChatCompletionSeparatesReasoningFromContent guards the regression -// where a reasoning-capable vision model's thinking leaked into the answer text -// that gets fed back to the text-only model. collectChatCompletion must keep -// delta.reasoning_content out of choices[0].message.content so that -// inspectCodebuddyImageWithEmit returns only the visible answer text. -func TestCollectChatCompletionSeparatesReasoningFromContent(t *testing.T) { - lines := [][]byte{ - []byte(`data: {"id":"cmpl-1","object":"chat.completion.chunk","created":1,"model":"hy3-preview","choices":[{"index":0,"delta":{"role":"assistant","reasoning_content":"让我看看这张图"}}]}`), - []byte(`data: {"id":"cmpl-1","object":"chat.completion.chunk","created":1,"model":"hy3-preview","choices":[{"index":0,"delta":{"reasoning_content":"这是一张"}}]}`), - []byte(`data: {"id":"cmpl-1","object":"chat.completion.chunk","created":1,"model":"hy3-preview","choices":[{"index":0,"delta":{"content":"红色方块"}}]}`), - []byte(`data: {"id":"cmpl-1","object":"chat.completion.chunk","created":1,"model":"hy3-preview","choices":[{"index":0,"delta":{"content":"，背景是蓝色"},"finish_reason":"stop"}]}`), - } - - out := collectChatCompletion(lines) - - if got := gjson.GetBytes(out, "choices.0.message.content").String(); got != "红色方块，背景是蓝色" { - t.Fatalf("content = %q, want only the visible answer text", got) - } - // The thinking must be preserved separately, not merged into content. - if got := gjson.GetBytes(out, "choices.0.message.reasoning_content").String(); got != "让我看看这张图这是一张" { - t.Fatalf("reasoning_content = %q, want thinking text kept separate", got) - } -} - -// TestCollectChatCompletionReasoningOnlyYieldsEmptyContent ensures that when a -// vision model emits only thinking and no answer, the resulting content is empty -// (so inspectCodebuddyImageWithEmit reports an empty answer instead of feeding -// thinking back to the text-only model). -func TestCollectChatCompletionReasoningOnlyYieldsEmptyContent(t *testing.T) { - lines := [][]byte{ - []byte(`data: {"id":"cmpl-2","object":"chat.completion.chunk","created":1,"model":"hy3-preview","choices":[{"index":0,"delta":{"reasoning_content":"无法解析图片"}}]}`), - } - - out := collectChatCompletion(lines) - - if got := gjson.GetBytes(out, "choices.0.message.content").String(); got != "" { - t.Fatalf("content = %q, want empty when only reasoning is present", got) - } - if got := gjson.GetBytes(out, "choices.0.message.reasoning_content").String(); got != "无法解析图片" { - t.Fatalf("reasoning_content = %q, want thinking preserved", got) - } -} - -// TestInspectCodebuddyImageDisablesReasoning guards the regression where the -// vision sub-request did not disable reasoning, causing hy3-preview to emit -// thinking that polluted the forwarded delta and the tool result. -func TestInspectCodebuddyImageDisablesReasoning(t *testing.T) { - // Build the request the same way inspectCodebuddyImageWithEmit does, then - // verify the reasoning field is present and disabled. This mirrors the inline - // body construction so a future removal of the guard is caught. - body := map[string]any{ - "model": "hy3-preview", - "messages": []any{ - map[string]any{"role": "user", "content": []any{ - map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/png;base64,AAAA"}}, - map[string]any{"type": "text", "text": "描述这张图"}, - }}, - }, - } - body["reasoning_effort"] = "none" - - b, err := json.Marshal(body) - if err != nil { - t.Fatalf("marshal: %v", err) - } - if got := gjson.GetBytes(b, "reasoning_effort").String(); got != "none" { - t.Fatalf("reasoning_effort = %q, want %q", got, "none") - } - if strings.Contains(string(b), "hy3-preview") == false { - t.Fatalf("body should reference the vision model") - } -} - -// TestStripVisionThinkingLineReasoningOnlyDrops guards rule A: a delta carrying -// only reasoning_content (no content) is dropped outright. -func TestStripVisionThinkingLineReasoningOnlyDrops(t *testing.T) { - line := []byte(`data: {"id":"cmpl-1","model":"hy3-preview","choices":[{"index":0,"delta":{"reasoning_content":"让我看看这张图"}}]}`) - cleaned, drop := stripVisionThinkingLine(line) - if !drop { - t.Fatalf("expected reasoning-only line to be dropped") - } - if cleaned != nil { - t.Fatalf("expected nil cleaned for a dropped line, got %q", cleaned) - } -} - -// TestStripVisionThinkingLineVisibleContentKept guards that a normal content -// delta (no thinking) is preserved unchanged. -func TestStripVisionThinkingLineVisibleContentKept(t *testing.T) { - line := []byte(`data: {"id":"cmpl-1","model":"hy3-preview","choices":[{"index":0,"delta":{"content":"红色方块"}}]}`) - cleaned, drop := stripVisionThinkingLine(line) - if drop { - t.Fatalf("expected visible content line to be kept") - } - // The content must survive stripping. - if got := gjson.GetBytes(cleaned, "choices.0.delta.content").String(); got != "红色方块" { - t.Fatalf("content = %q, want %q", got, "红色方块") - } -} - -// TestStripVisionThinkingLineMixedReasoningAndContentKeepsContent guards that a -// chunk carrying both reasoning_content and a visible content delta is kept, with -// the visible content intact. -func TestStripVisionThinkingLineMixedReasoningAndContentKeepsContent(t *testing.T) { - line := []byte(`data: {"id":"cmpl-1","model":"hy3-preview","choices":[{"index":0,"delta":{"reasoning_content":"思考中","content":"这是一张图"}}]}`) - cleaned, drop := stripVisionThinkingLine(line) - if drop { - t.Fatalf("expected line with visible content to be kept") - } - if got := gjson.GetBytes(cleaned, "choices.0.delta.content").String(); got != "这是一张图" { - t.Fatalf("content = %q, want %q", got, "这是一张图") - } -} - -// TestStripVisionThinkingLineNonDataLinePassThrough guards that non-SSE lines -// (e.g. comments) are returned unchanged and not dropped. -func TestStripVisionThinkingLineNonDataLinePassThrough(t *testing.T) { - line := []byte(": keep-alive") - cleaned, drop := stripVisionThinkingLine(line) - if drop { - t.Fatalf("expected non-data line to not be dropped") - } - if string(cleaned) != string(line) { - t.Fatalf("cleaned = %q, want %q", cleaned, line) - } -} - -// TestStripVisionThinkingLineDonePassThrough guards that [DONE] sentinels are -// passed through unchanged. -func TestStripVisionThinkingLineDonePassThrough(t *testing.T) { - line := []byte("data: [DONE]") - cleaned, drop := stripVisionThinkingLine(line) - if drop { - t.Fatalf("expected [DONE] to not be dropped") - } - if string(cleaned) != string(line) { - t.Fatalf("cleaned = %q, want %q", cleaned, line) - } -} - -// TestStripThinkingMarkersFromContentEmptyTableIsIdentity guards the conservative -// default: with an empty marker table, content text is returned verbatim so -// legitimate image descriptions are never stripped. -func TestStripThinkingMarkersFromContentEmptyTableIsIdentity(t *testing.T) { - text := "图片右上角的文字是「出口」，背景是蓝色。" - if got := stripThinkingMarkersFromContent(text); got != text { - t.Fatalf("got %q, want identity %q", got, text) - } -} - -// TestInspectImageSystemPromptForbidsAdvice guards the agentic-path system -// constraint: the vision sub-request's system message must require objective -// detail extraction only and forbid solutions/suggestions. -func TestInspectImageSystemPromptForbidsAdvice(t *testing.T) { - for _, keyword := range []string{"解决方案", "修改建议", "操作步骤", "分析判断", "只客观"} { - if !strings.Contains(codebuddyInspectImageSystemPrompt, keyword) { - t.Fatalf("codebuddyInspectImageSystemPrompt must forbid %q, got: %s", keyword, codebuddyInspectImageSystemPrompt) - } - } -} - -// TestInspectCodebuddyImagePrependsSystemMessage guards that the vision -// sub-request built by inspectCodebuddyImageWithEmit carries a system message -// with the detail-extraction constraint before the user message. It mirrors the -// body construction order (marshal → prepend system) so a future removal of the -// prepend is caught. -func TestInspectCodebuddyImagePrependsSystemMessage(t *testing.T) { - body := map[string]any{ - "model": "hy3-preview", - "messages": []any{ - map[string]any{"role": "user", "content": []any{ - map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/png;base64,AAAA"}}, - map[string]any{"type": "text", "text": "图片里的文字是什么"}, - }}, - }, - } - body["reasoning_effort"] = "none" - - b, err := json.Marshal(body) - if err != nil { - t.Fatalf("marshal: %v", err) - } - b, err = prependCodebuddySystemMessage(b, codebuddyInspectImageSystemPrompt) - if err != nil { - t.Fatalf("prepend: %v", err) - } - - msgs := gjson.GetBytes(b, "messages") - if !msgs.IsArray() || len(msgs.Array()) != 2 { - t.Fatalf("expected 2 messages after prepend, got %s", msgs.Raw) - } - if gjson.GetBytes(b, "messages.0.role").String() != "system" { - t.Fatalf("messages.0 must be system, got %s", gjson.GetBytes(b, "messages.0.role").String()) - } - if got := gjson.GetBytes(b, "messages.0.content").String(); got != codebuddyInspectImageSystemPrompt { - t.Fatalf("system content = %q, want %q", got, codebuddyInspectImageSystemPrompt) - } - if gjson.GetBytes(b, "messages.1.role").String() != "user" { - t.Fatalf("messages.1 must be the user message, got %s", gjson.GetBytes(b, "messages.1.role").String()) - } -} - diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_test.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_test.go deleted file mode 100644 index 2b8e943..0000000 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/codebuddy_executor_vision_test.go +++ /dev/null @@ -1,682 +0,0 @@ -package executor - -import ( - "strings" - "testing" - - "github.com/router-for-me/CLIProxyAPI/v7/internal/config" - "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/usage" - "github.com/tidwall/gjson" -) - -// --- image input detection ------------------------------------------------- - -func TestCodebuddyChatHasImageInput(t *testing.T) { - tests := []struct { - name string - in string - want bool - }{ - { - // payload 必须超过 codebuddyImageStubMaxPayloadChars(512)， - // 否则会被 codebuddyImagePartIsStub 判定为截断残片（stub）。 - name: "image_url part detected", - in: `{"messages":[{"role":"user","content":[{"type":"text","text":"hi"},{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 600) + `"}}]}]}`, - want: true, - }, - { - name: "input_image part detected", - in: `{"messages":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,` + strings.Repeat("B", 600) + `"}]}]}`, - want: true, - }, - { - name: "text only", - in: `{"messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}`, - want: false, - }, - { - name: "string content", - in: `{"messages":[{"role":"user","content":"hello"}]}`, - want: false, - }, - { - name: "no messages", - in: `{"model":"auto"}`, - want: false, - }, - { - name: "invalid json", - in: `not-json`, - want: false, - }, - { - name: "historical image ignored when last user message is text-only", - in: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]},{"role":"assistant","content":"ok"},{"role":"user","content":[{"type":"text","text":"继续"}]}]}`, - want: false, - }, - { - name: "image in last user message detected despite text history", - in: `{"messages":[{"role":"user","content":[{"type":"text","text":"之前"}]},{"role":"assistant","content":"ok"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 600) + `"}}]}]}`, - want: true, - }, - { - name: "image in tool message detected (Read tool result)", - in: `{"messages":[{"role":"user","content":[{"type":"text","text":"读一下这张图"}]},{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"Read","arguments":"{\"file_path\":\"a.png\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 600) + `"}}]}]}`, - want: true, - }, - { - name: "historical tool image ignored when last user message is text-only", - in: `{"messages":[{"role":"user","content":[{"type":"text","text":"读图"}]},{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"Read","arguments":"{\"file_path\":\"a.png\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,HIST"}}]},{"role":"assistant","content":"看完了"},{"role":"user","content":[{"type":"text","text":"继续"}]}]}`, - want: false, - }, - { - name: "no user message", - in: `{"messages":[{"role":"assistant","content":"ok"}]}`, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := codebuddyChatHasImageInput([]byte(tt.in)); got != tt.want { - t.Fatalf("codebuddyChatHasImageInput() = %v, want %v", got, tt.want) - } - }) - } -} - -// --- model rewrite (routing mode) ------------------------------------------ - -func TestRewriteCodebuddyModel(t *testing.T) { - tests := []struct { - name string - in string - model string - want string - }{ - { - name: "existing model replaced", - in: `{"model":"deepseek-v4-flash","messages":[]}`, - model: "hy3-preview", - want: "hy3-preview", - }, - { - name: "missing model added", - in: `{"messages":[]}`, - model: "hy3-preview", - want: "hy3-preview", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := rewriteCodebuddyModel([]byte(tt.in), tt.model) - got := gjson.GetBytes(out, "model").String() - if got != tt.want { - t.Fatalf("model = %q, want %q; out=%s", got, tt.want, out) - } - // messages must survive untouched - if !gjson.GetBytes(out, "messages").Exists() { - t.Fatalf("messages lost; out=%s", out) - } - }) - } -} - -// --- image -> text replacement (preprocess mode) --------------------------- - -func TestReplaceCodebuddyImagesWithText(t *testing.T) { - in := `{"messages":[{"role":"user","content":[{"type":"text","text":"这是什么？"},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]}]}` - out := replaceCodebuddyImagesWithText([]byte(in), "一张红色方块") - - parts := gjson.GetBytes(out, "messages.0.content").Array() - if len(parts) != 2 { - t.Fatalf("expected 2 parts, got %d; out=%s", len(parts), out) - } - if parts[0].Get("type").String() != "text" || parts[0].Get("text").String() != "这是什么？" { - t.Fatalf("text part corrupted: %s", parts[0].Raw) - } - if parts[1].Get("type").String() != "text" { - t.Fatalf("image part not replaced by text: %s", parts[1].Raw) - } - if got := parts[1].Get("text").String(); got != "一张红色方块" { - t.Fatalf("replacement text = %q, want %q", got, "一张红色方块") - } -} - -func TestReplaceCodebuddyImagesWithText_NoImages(t *testing.T) { - in := `{"messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}` - out := replaceCodebuddyImagesWithText([]byte(in), "ignored") - if string(out) != in { - t.Fatalf("expected unchanged, got %s", out) - } -} - -// --- vision proxy plan (decision) ------------------------------------------- - -func TestCodebuddyVisionPlan(t *testing.T) { - tests := []struct { - name string - mode string - visionModel string - currentModel string - hasImage bool - supportsImg bool - want codebuddyVisionAction - }{ - { - name: "off always passes through", - mode: config.CodebuddyVisionModeOff, - currentModel: "deepseek-v4-flash", - hasImage: true, - want: codebuddyVisionPassThrough, - }, - { - name: "no image passes through", - mode: config.CodebuddyVisionModeRouting, - currentModel: "deepseek-v4-flash", - hasImage: false, - want: codebuddyVisionPassThrough, - }, - { - name: "vision model itself passes through (no recursion)", - mode: config.CodebuddyVisionModeRouting, - visionModel: "hy3-preview", - currentModel: "hy3-preview", - hasImage: true, - want: codebuddyVisionPassThrough, - }, - { - name: "native vision model passes through", - mode: config.CodebuddyVisionModeRouting, - visionModel: "hy3-preview", - currentModel: "glm-4.6v", - hasImage: true, - supportsImg: true, - want: codebuddyVisionPassThrough, - }, - { - name: "routing swaps text-only model", - mode: config.CodebuddyVisionModeRouting, - visionModel: "hy3-preview", - currentModel: "deepseek-v4-flash", - hasImage: true, - want: codebuddyVisionRoute, - }, - { - name: "preprocess describes then keeps model", - mode: config.CodebuddyVisionModePreprocess, - visionModel: "hy3-preview", - currentModel: "deepseek-v4-flash", - hasImage: true, - want: codebuddyVisionPreprocess, - }, - { - name: "unknown mode falls back to pass-through", - mode: "bogus", - visionModel: "hy3-preview", - currentModel: "deepseek-v4-flash", - hasImage: true, - want: codebuddyVisionPassThrough, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := codebuddyVisionPlan(tt.mode, tt.visionModel, tt.currentModel, tt.hasImage, tt.supportsImg) - if got != tt.want { - t.Fatalf("codebuddyVisionPlan() = %v, want %v", got, tt.want) - } - }) - } -} - -// --- agentic vision: image extraction & tool injection --------------------- - -func TestExtractCodebuddyImagesForAgentic(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","messages":[ - {"role":"user","content":[ - {"type":"text","text":"看这两张图"}, - {"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}, - {"type":"image_url","image_url":{"url":"data:image/png;base64,BBBB"}} - ]} - ]}`) - - out, images, err := extractCodebuddyImagesForAgentic(body) - if err != nil { - t.Fatalf("extractCodebuddyImagesForAgentic() error: %v", err) - } - if len(images) != 2 { - t.Fatalf("expected 2 images, got %d", len(images)) - } - if images[0].id != 1 || images[1].id != 2 { - t.Fatalf("expected sequential ids 1,2, got %d,%d", images[0].id, images[1].id) - } - // First image part must be replaced by a text hint, second likewise. - if gjson.GetBytes(out, "messages.0.content.1.type").String() != "text" { - t.Fatalf("image part 1 should be replaced with text, got %s", gjson.GetBytes(out, "messages.0.content.1").Raw) - } - if !strings.Contains(gjson.GetBytes(out, "messages.0.content.1.text").String(), "inspect_image") { - t.Fatalf("replacement should reference inspect_image tool") - } - // Text part untouched. - if gjson.GetBytes(out, "messages.0.content.0.text").String() != "看这两张图" { - t.Fatalf("text part should remain untouched") - } -} - -func TestExtractCodebuddyImagesForAgentic_OnlyLastUserMessage(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","messages":[ - {"role":"user","content":[ - {"type":"image_url","image_url":{"url":"data:image/png;base64,HIST"}} - ]}, - {"role":"assistant","content":"ok"}, - {"role":"user","content":[ - {"type":"text","text":"再看这张"}, - {"type":"image_url","image_url":{"url":"data:image/png;base64,NEW"}} - ]} - ]}`) - - out, images, err := extractCodebuddyImagesForAgentic(body) - if err != nil { - t.Fatalf("extractCodebuddyImagesForAgentic() error: %v", err) - } - if len(images) != 1 { - t.Fatalf("expected 1 image (only last user message), got %d", len(images)) - } - // The historical image (messages.0.content.0) must be replaced with a - // placeholder so it never reaches the text-only model. - if gjson.GetBytes(out, "messages.0.content.0.type").String() != "text" { - t.Fatalf("historical image should be replaced with text, got %s", gjson.GetBytes(out, "messages.0.content.0").Raw) - } - // The last user message's image (messages.2.content.1) must be replaced. - if gjson.GetBytes(out, "messages.2.content.1.type").String() != "text" { - t.Fatalf("last user image should be replaced with text, got %s", gjson.GetBytes(out, "messages.2.content.1").Raw) - } - if !strings.Contains(gjson.GetBytes(out, "messages.2.content.1.text").String(), "inspect_image") { - t.Fatalf("replacement should reference inspect_image tool") - } -} - -func TestExtractCodebuddyImagesForAgentic_HistoricalImageIgnored(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","messages":[ - {"role":"user","content":[ - {"type":"image_url","image_url":{"url":"data:image/png;base64,HIST"}} - ]}, - {"role":"assistant","content":"ok"}, - {"role":"user","content":[ - {"type":"text","text":"继续"} - ]} - ]}`) - - out, images, err := extractCodebuddyImagesForAgentic(body) - if err != nil { - t.Fatalf("extractCodebuddyImagesForAgentic() error: %v", err) - } - if len(images) != 0 { - t.Fatalf("expected 0 images (last user message is text-only), got %d", len(images)) - } - // The historical image must be replaced with a placeholder so it never - // reaches the text-only model. - if gjson.GetBytes(out, "messages.0.content.0.type").String() != "text" { - t.Fatalf("historical image should be replaced with text, got %s", gjson.GetBytes(out, "messages.0.content.0").Raw) - } -} - -func TestExtractCodebuddyImagesForAgentic_ToolMessageImage(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","messages":[ - {"role":"user","content":[{"type":"text","text":"读一下这张图"}]}, - {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"Read","arguments":"{\"file_path\":\"a.png\"}"}}]}, - {"role":"tool","tool_call_id":"call_1","content":[ - {"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}} - ]} - ]}`) - - out, images, err := extractCodebuddyImagesForAgentic(body) - if err != nil { - t.Fatalf("extractCodebuddyImagesForAgentic() error: %v", err) - } - if len(images) != 1 { - t.Fatalf("expected 1 image from tool message, got %d", len(images)) - } - if images[0].id != 1 { - t.Fatalf("expected image id 1, got %d", images[0].id) - } - // The tool message's image must be replaced with a text hint referencing - // inspect_image, so the text-only model can query it via the vision model. - if gjson.GetBytes(out, "messages.2.content.0.type").String() != "text" { - t.Fatalf("tool image should be replaced with text, got %s", gjson.GetBytes(out, "messages.2.content.0").Raw) - } - if !strings.Contains(gjson.GetBytes(out, "messages.2.content.0.text").String(), "inspect_image") { - t.Fatalf("replacement should reference inspect_image tool") - } -} - -func TestInjectCodebuddyInspectTool(t *testing.T) { - body := []byte(`{"model":"deepseek-v4-pro","messages":[{"role":"user","content":"hi"}]}`) - out := injectCodebuddyInspectTool(body, 1) - - // tools array injected. - tools := gjson.GetBytes(out, "tools") - if !tools.IsArray() || len(tools.Array()) != 1 { - t.Fatalf("expected 1 tool injected, got %s", tools.Raw) - } - if tools.Get("0.function.name").String() != inspectImageToolName { - t.Fatalf("expected inspect_image tool, got %s", tools.Get("0.function.name").String()) - } - - // System message prepended. - sysContent := gjson.GetBytes(out, "messages.0.content").String() - if gjson.GetBytes(out, "messages.0.role").String() != "system" || !strings.Contains(sysContent, "inspect_image") { - t.Fatalf("expected system guidance message, got role=%s content=%s", - gjson.GetBytes(out, "messages.0.role").String(), sysContent) - } -} - -func TestAppendAgenticMessage(t *testing.T) { - body := []byte(`{"messages":[{"role":"user","content":"hi"}]}`) - out, err := appendAgenticMessage(body, []byte(`{"role":"assistant","content":"ok"}`)) - if err != nil { - t.Fatalf("appendAgenticMessage() error: %v", err) - } - arr := gjson.GetBytes(out, "messages") - if !arr.IsArray() || len(arr.Array()) != 2 { - t.Fatalf("expected 2 messages, got %s", arr.Raw) - } - if gjson.GetBytes(out, "messages.1.role").String() != "assistant" { - t.Fatalf("expected assistant message appended") - } -} - -// TestAddCodebuddyAgenticUsageAccumulatesTokenBreakdown guards the regression -// where addCodebuddyAgenticUsage only summed TokenBreakdown.TotalTokens, leaving -// the Input/Output sub-fields at zero and making the aggregated breakdown -// invalid (which in turn caused the request log's input/output columns to show 0). -func TestAddCodebuddyAgenticUsageAccumulatesTokenBreakdown(t *testing.T) { - total := usage.Detail{} - add1 := usage.Detail{ - InputTokens: 100, - OutputTokens: 50, - TokenBreakdown: usage.TokenBreakdown{ - TotalTokens: 150, - Input: usage.TokenInputBreakdown{ - TotalTokens: 100, - UncachedTokens: 80, - CacheReadTokens: 15, - CacheWriteTokens: 5, - }, - Output: usage.TokenOutputBreakdown{ - TotalTokens: 50, - NonReasoningTokens: 40, - ReasoningTokens: 10, - }, - }, - } - add2 := usage.Detail{ - InputTokens: 20, - OutputTokens: 30, - TokenBreakdown: usage.TokenBreakdown{ - TotalTokens: 50, - Input: usage.TokenInputBreakdown{ - TotalTokens: 20, - UncachedTokens: 12, - CacheReadTokens: 8, - CacheWriteTokens: 0, - }, - Output: usage.TokenOutputBreakdown{ - TotalTokens: 30, - NonReasoningTokens: 30, - ReasoningTokens: 0, - }, - }, - } - - addCodebuddyAgenticUsage(&total, add1) - addCodebuddyAgenticUsage(&total, add2) - - if total.InputTokens != 120 || total.OutputTokens != 80 { - t.Fatalf("top-level tokens = %d/%d, want 120/80", total.InputTokens, total.OutputTokens) - } - if total.TokenBreakdown.TotalTokens != 200 { - t.Fatalf("breakdown total = %d, want 200", total.TokenBreakdown.TotalTokens) - } - if total.TokenBreakdown.Input.TotalTokens != 120 || - total.TokenBreakdown.Input.UncachedTokens != 92 || - total.TokenBreakdown.Input.CacheReadTokens != 23 || - total.TokenBreakdown.Input.CacheWriteTokens != 5 { - t.Fatalf("breakdown input = %+v", total.TokenBreakdown.Input) - } - if total.TokenBreakdown.Output.TotalTokens != 80 || - total.TokenBreakdown.Output.NonReasoningTokens != 70 || - total.TokenBreakdown.Output.ReasoningTokens != 10 { - t.Fatalf("breakdown output = %+v", total.TokenBreakdown.Output) - } -} - -func TestCodebuddyVisionAgenticEnabled(t *testing.T) { - off := &CodebuddyExecutor{cfg: &config.Config{SDKConfig: config.SDKConfig{CodebuddyVision: config.CodebuddyVisionConfig{Mode: "off"}}}} - if off.codebuddyVisionAgenticEnabled() { - t.Fatal("off mode should not report agentic enabled") - } - agentic := &CodebuddyExecutor{cfg: &config.Config{SDKConfig: config.SDKConfig{CodebuddyVision: config.CodebuddyVisionConfig{Mode: "agentic"}}}} - if !agentic.codebuddyVisionAgenticEnabled() { - t.Fatal("agentic mode should report enabled") - } -} - -// TestDefaultCodebuddyVisionPromptForbidsAdvice guards the regression where the -// vision model proposed solutions/actions instead of only describing the image. -// The default preprocess prompt must explicitly forbid solutions/suggestions. -func TestDefaultCodebuddyVisionPromptForbidsAdvice(t *testing.T) { - for _, keyword := range []string{"解决方案", "修改建议", "操作步骤", "分析判断"} { - if !strings.Contains(defaultCodebuddyVisionPrompt, keyword) { - t.Fatalf("defaultCodebuddyVisionPrompt must forbid %q, got: %s", keyword, defaultCodebuddyVisionPrompt) - } - } - if !strings.Contains(defaultCodebuddyVisionPrompt, "只客观陈述") { - t.Fatalf("defaultCodebuddyVisionPrompt must require objective description only") - } -} - -// TestBuildCodebuddyVisionPromptFocusedForbidsAdvice guards that the focused -// (user-question) branch also forbids solutions/suggestions. -func TestBuildCodebuddyVisionPromptFocusedForbidsAdvice(t *testing.T) { - got := buildCodebuddyVisionPrompt("", "图片里的报错是什么") - for _, keyword := range []string{"解决方案", "修改建议", "操作步骤", "分析判断", "只客观陈述"} { - if !strings.Contains(got, keyword) { - t.Fatalf("focused prompt must forbid %q, got: %s", keyword, got) - } - } -} - -// TestBuildCodebuddyVisionPromptCustomTakesPriority guards that a user-supplied -// PreprocessPrompt is returned verbatim (no injected constraint), preserving the -// explicit-override contract. -func TestBuildCodebuddyVisionPromptCustomTakesPriority(t *testing.T) { - custom := "自定义描述 prompt" - got := buildCodebuddyVisionPrompt(custom, "任意问题") - if got != custom { - t.Fatalf("custom prompt must be returned verbatim, got %q", got) - } -} - -// TestIsCodebuddyReadToolName guards the read-tool name matching used by the -// problem-two diagnostic. -func TestIsCodebuddyReadToolName(t *testing.T) { - for _, in := range []string{"read", "Read", "read_file", "READ_FILE", "readfile", "read-file", " Read "} { - if !isCodebuddyReadToolName(in) { - t.Fatalf("isCodebuddyReadToolName(%q) = false, want true", in) - } - } - for _, in := range []string{"bash", "write", "write_file", "ReadFilex", "globs"} { - if isCodebuddyReadToolName(in) { - t.Fatalf("isCodebuddyReadToolName(%q) = true, want false", in) - } - } -} - -// TestCodebuddyBodyMentionsReadTool guards the diagnostic gate detection across -// tool declarations, assistant tool_calls, and role=tool messages. -func TestCodebuddyBodyMentionsReadTool(t *testing.T) { - tests := []struct { - name string - in string - want bool - }{ - { - name: "tool declaration read", - in: `{"tools":[{"type":"function","function":{"name":"read"}}],"messages":[]}`, - want: true, - }, - { - name: "assistant tool_calls read", - in: `{"messages":[{"role":"assistant","tool_calls":[{"id":"c1","type":"function","function":{"name":"Read","arguments":"{\"file_path\":\"a.png\"}"}}]}]}`, - want: true, - }, - { - name: "role tool message", - in: `{"messages":[{"role":"tool","tool_call_id":"c1","content":"text"}]}`, - want: true, - }, - { - name: "no read tool", - in: `{"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, - want: false, - }, - { - name: "invalid json", - in: `not-json`, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := codebuddyBodyMentionsReadTool([]byte(tt.in)); got != tt.want { - t.Fatalf("codebuddyBodyMentionsReadTool() = %v, want %v", got, tt.want) - } - }) - } -} - -// --- historical image rewrite ---------------------------------------------- - -func TestReplaceCodebuddyHistoricalImagesWithText(t *testing.T) { - marker := "[历史图片]" - tests := []struct { - name string - in string - want func(t *testing.T, out string) - }{ - { - name: "historical stub replaced, current-turn image kept", - in: `{"messages":[` + - `{"role":"user","content":[{"type":"text","text":"turn1"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,/9j/4AAQSkZJRgABA"}}]},` + - `{"role":"assistant","content":"描述..."},` + - `{"role":"user","content":[{"type":"text","text":"turn2"},{"type":"image_url","image_url":{"url":"data:image/png;base64,REALIMAGE"}}]}]}`, - want: func(t *testing.T, out string) { - t.Helper() - first := gjson.Get(out, "messages.0.content.1") - if first.Get("type").String() != "text" || first.Get("text").String() != marker { - t.Fatalf("historical image should become text marker, got %s", first.Raw) - } - current := gjson.Get(out, "messages.2.content.1") - if current.Get("type").String() != "image_url" { - t.Fatalf("current-turn image must stay untouched, got %s", current.Raw) - } - }, - }, - { - name: "text-only follow-up: stub replaced so model relies on description", - in: `{"messages":[` + - `{"role":"user","content":[{"type":"text","text":"turn1"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,/9j/4AAQSkZJRgABA"}}]},` + - `{"role":"assistant","content":"这是一张猫的图片"},` + - `{"role":"user","content":"它是什么颜色?"}]}`, - want: func(t *testing.T, out string) { - t.Helper() - first := gjson.Get(out, "messages.0.content.1") - if first.Get("type").String() != "text" || first.Get("text").String() != marker { - t.Fatalf("historical stub should become text marker, got %s", first.Raw) - } - if gjson.Get(out, "messages.2.content").String() != "它是什么颜色?" { - t.Fatalf("text-only current turn must stay untouched") - } - }, - }, - { - name: "no historical messages: no-op", - in: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]}]}`, - want: func(t *testing.T, out string) { - t.Helper() - if gjson.Get(out, "messages.0.content.0.type").String() != "image_url" { - t.Fatalf("single-turn image must stay untouched") - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := replaceCodebuddyHistoricalImagesWithText([]byte(tt.in), marker) - tt.want(t, string(out)) - }) - } -} - -// --- stub (truncated image) handling ---------------------------------------- - -const stubURL = "data:image/jpeg;base64,/9j/4AAQSkZJRgABA" // 40-char payload, the real-world 80-char stub - -func TestCodebuddyImagePartIsStub(t *testing.T) { - tests := []struct { - name string - raw string - want bool - }{ - {"80-char truncated stub", `{"type":"image_url","image_url":{"url":"` + stubURL + `"}}`, true}, - {"real data url", `{"type":"image_url","image_url":{"url":"data:image/png;base64,` + string(make([]byte, 600)) + `"}}`, false}, - {"remote url is never stub", `{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}`, false}, - {"empty url", `{"type":"image_url","image_url":{"url":""}}`, true}, - {"input_image stub form", `{"type":"input_image","image_url":"` + stubURL + `"}`, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := codebuddyImagePartIsStub([]byte(tt.raw)); got != tt.want { - t.Fatalf("codebuddyImagePartIsStub() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestCodebuddyChatHasImageInputIgnoresStubs(t *testing.T) { - // Tool-continuation turn: user message carries a truncated stub, followed by - // tool results. Must NOT count as image input (no vision call on stubs). - body := `{"messages":[` + - `{"role":"user","content":[{"type":"text","text":"描述"},{"type":"image_url","image_url":{"url":"` + stubURL + `"}}]},` + - `{"role":"assistant","tool_calls":[{"id":"c1","function":{"name":"read_file","arguments":"{\"path\":\"h:////home.png/"}"}}]},` + - `{"role":"tool","tool_call_id":"c1","content":"Read image file: h://home.png"}]}` - if codebuddyChatHasImageInput([]byte(body)) { - t.Fatal("truncated stub must not count as image input") - } - if len(extractCodebuddyCurrentImages([]byte(body))) != 0 { - t.Fatal("extractCodebuddyCurrentImages must skip stubs") - } -} - -func TestReplaceCodebuddyCurrentTurnStubsWithText(t *testing.T) { - marker := "[历史图片]" - realImg := "data:image/png;base64," + string(make([]byte, 600)) - body := `{"messages":[` + - `{"role":"user","content":[{"type":"text","text":"再看这张"},{"type":"image_url","image_url":{"url":"` + stubURL + `"}},{"type":"image_url","image_url":{"url":"` + realImg + `"}}]}]}` - out := string(replaceCodebuddyCurrentTurnStubsWithText([]byte(body), marker)) - if gjson.Get(out, "messages.0.content.1.type").String() != "text" || gjson.Get(out, "messages.0.content.1.text").String() != marker { - t.Fatalf("stub should become marker text, got %s", gjson.Get(out, "messages.0.content.1").Raw) - } - if gjson.Get(out, "messages.0.content.2.type").String() != "image_url" { - t.Fatalf("real image must stay untouched, got %s", gjson.Get(out, "messages.0.content.2.type").String()) - } -} - -func TestCursorReadFileV2PlaceholderRecognized(t *testing.T) { - if !isCodebuddyImagePlaceholder("Read image file: h://Ai 自测空间文档\\home.png") { - t.Fatal("Cursor Read File V2 confirmation must be recognized as an image placeholder") - } -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers.go index a352989..b01b881 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers.go @@ -42,7 +42,6 @@ type UsageReporter struct { ttftStart time.Time ttftSet bool once sync.Once - visionSubagent bool } // Model returns the model label this reporter will publish. @@ -53,14 +52,6 @@ func (r *UsageReporter) Model() string { return r.model } -// MarkVisionSubagent flags this reporter's record as handled by the pure-text -// vision sub-agent loop, so the UI can render a "视" badge next to the model. -func (r *UsageReporter) MarkVisionSubagent() { - if r != nil { - r.visionSubagent = true - } -} - type usageExecutor interface { Identifier() string } @@ -143,37 +134,6 @@ func (r *UsageReporter) PublishAdditionalModel(ctx context.Context, model string r.publishRecord(ctx, record) } -// PublishAdditionalModelAlways is like PublishAdditionalModel but also publishes -// a record when the detail has all-zero token usage. It is used for the vision -// sub-model path: the sub-model's usage may parse to all-zero (e.g. when the -// upstream omits or renames token fields), but the request itself still happened -// and must remain visible in the request log as a distinct additional-model -// record alongside the base model. -func (r *UsageReporter) PublishAdditionalModelAlways(ctx context.Context, model string, detail usage.Detail) { - record, ok := r.buildAdditionalModelRecordAlways(model, detail) - if !ok { - return - } - r.publishRecord(ctx, record) -} - -// buildAdditionalModelRecordAlways is buildAdditionalModelRecord with an -// all-zero-token fallback: it returns a record even when the detail carries no -// token figures, so the additional-model request is never silently dropped. -func (r *UsageReporter) buildAdditionalModelRecordAlways(model string, detail usage.Detail) (usage.Record, bool) { - record, ok := r.buildAdditionalModelRecord(model, detail) - if ok { - return record, true - } - if r == nil || strings.TrimSpace(model) == "" { - return usage.Record{}, false - } - // Fall back to an all-zero record so the vision sub-model's request is - // still recorded even without token figures. - var noFailure usage.Failure - return r.buildRecordForModel(model, normalizeUsageDetailTotal(usage.Detail{}, r.provider, r.executorType), false, noFailure), true -} - func (r *UsageReporter) SetTranslatedReasoningEffort(payload []byte, format string) { if r == nil { return @@ -344,7 +304,6 @@ func (r *UsageReporter) buildRecordForModel(model string, detail usage.Detail, f ServiceTier: r.serviceTier, ResponseServiceTier: strings.TrimSpace(detail.ResponseServiceTier), Generate: usage.GenerateFlag(r.generate), - VisionSubagent: r.visionSubagent, RequestedAt: r.requestedAt, Latency: r.latency(), TTFT: r.ttftDuration(), diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers_test.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers_test.go index 3278a97..5296dbd 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers_test.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/helps/usage_helpers_test.go @@ -256,33 +256,3 @@ func TestUsageReporterBuildAdditionalModelRecordSkipsZeroTokens(t *testing.T) { } } -func TestUsageReporterBuildAdditionalModelRecordAlwaysKeepsZeroTokens(t *testing.T) { - reporter := &UsageReporter{ - provider: "codebuddy", - model: "deepseek-v4-pro", - requestedAt: time.Now(), - } - - // All-zero token usage must still produce a record for the vision sub-model - // path, so the request remains visible in the request log. - record, ok := reporter.buildAdditionalModelRecordAlways("hy3-preview", usage.Detail{}) - if !ok { - t.Fatalf("expected all-zero token usage to still produce a record") - } - if record.Model != "hy3-preview" { - t.Fatalf("record model = %q, want %q", record.Model, "hy3-preview") - } - if record.Provider != "codebuddy" { - t.Fatalf("record provider = %q, want %q", record.Provider, "codebuddy") - } - - // Non-zero usage still records normally. - if _, ok := reporter.buildAdditionalModelRecordAlways("hy3-preview", usage.Detail{InputTokens: 2}); !ok { - t.Fatalf("expected non-zero token usage to be recorded") - } - - // Empty model still dropped (nothing meaningful to record). - if _, ok := reporter.buildAdditionalModelRecordAlways(" ", usage.Detail{InputTokens: 2}); ok { - t.Fatalf("expected empty model to be skipped") - } -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/openai_compat_executor.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/openai_compat_executor.go index 55cfe57..e92d79d 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/openai_compat_executor.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/openai_compat_executor.go @@ -135,12 +135,6 @@ func (e *OpenAICompatExecutor) Execute(ctx context.Context, auth *cliproxyauth.A return resp, err } } - // Historical image rewrite: replace truncated historical image stubs with a - // text marker so text-only models do not choke on them. - translated = e.rewriteOpenAICompatHistoricalImages(translated, baseModel) - // Vision proxy: transparently handle image input for non-vision models on the - // OpenAI-compatible (third-party relay) path, mirroring the CodeBuddy executor. - translated, _ = e.applyOpenAICompatVisionProxy(ctx, auth, translated, baseModel, nil, reporter) if opts.Alt == "responses/compact" { if updated, errDelete := sjson.DeleteBytes(translated, "stream"); errDelete == nil { translated = updated @@ -361,17 +355,6 @@ func (e *OpenAICompatExecutor) ExecuteStream(ctx context.Context, auth *cliproxy translated = helps.SetBoolIfDifferent(translated, "stream_options.include_usage", true) reporter.SetTranslatedReasoningEffort(translated, to.String()) - // Historical image rewrite: replace truncated historical image stubs with a - // text marker so text-only models do not choke on them. - translated = e.rewriteOpenAICompatHistoricalImages(translated, baseModel) - - // Vision proxy: detect whether this request needs preprocess (describe images - // via the vision model first, then continue with the text-only model). When - // needed, we keep the image-bearing body intact here and describe it inside - // the stream goroutine so the description can be forwarded to the client in - // real time without tripping the relay stream-open watchdog. - needsPreprocess := e.openAICompatVisionNeedsPreprocess(translated, baseModel) - url := strings.TrimSuffix(baseURL, "/") + "/chat/completions" out := make(chan cliproxyexecutor.StreamChunk) go func() { @@ -385,50 +368,6 @@ func (e *OpenAICompatExecutor) ExecuteStream(ctx context.Context, auth *cliproxy } }() - // Preprocess streaming: describe the images via the vision model first, - // forwarding each description delta to the client in real time, then - // rewrite the image parts into text and continue with the text-only - // model. The initial role chunk opens the stream immediately so the - // relay's stream-open watchdog does not trip during the multi-second - // vision call. - if needsPreprocess { - visionModel := e.cfg.CodebuddyVision.VisionModel() - emit := func(line []byte) bool { - select { - case out <- cliproxyexecutor.StreamChunk{Payload: line}: - return true - case <-ctx.Done(): - return false - } - } - initChunk := buildCodebuddyVisionChunk("", baseModel, 0, nil, "assistant") - if initChunk != nil && !emit(initChunk) { - return - } - descriptions, visionUsage, descErr := e.describeOpenAICompatImages(ctx, auth, translated, visionModel, e.cfg.CodebuddyVision.PreprocessPrompt, baseModel, emit) - if descErr != nil { - log.Warnf("openai compat vision proxy: preprocess stream failed for %s (vision=%s): %v; omitting images", baseModel, visionModel, descErr) - translated = replaceCodebuddyImagesWithText(translated, codebuddyOmittedImageText) - } else { - log.Infof("openai compat vision proxy: preprocessed %d image(s) for %s via %s", len(descriptions), baseModel, visionModel) - translated = replaceCodebuddyImagesWithDescriptions(translated, descriptions, codebuddyOmittedImageText) - reporter.PublishAdditionalModelAlways(ctx, visionModel, visionUsage) - } - // Re-apply stream forcing on the (rewritten) text-only body. - var errSet error - translated, errSet = sjson.SetBytes(translated, "stream", true) - if errSet == nil { - translated, errSet = sjson.SetBytes(translated, "stream_options.include_usage", true) - } - if errSet != nil { - select { - case out <- cliproxyexecutor.StreamChunk{Err: errSet}: - case <-ctx.Done(): - } - return - } - } - httpReq, errReq := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(translated)) if errReq != nil { helps.RecordAPIResponseError(ctx, e.cfg, errReq) diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/openai_compat_vision.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/openai_compat_vision.go deleted file mode 100644 index 5a51b7e..0000000 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/internal/runtime/executor/openai_compat_vision.go +++ /dev/null @@ -1,314 +0,0 @@ -package executor - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - - "github.com/router-for-me/CLIProxyAPI/v7/internal/config" - "github.com/router-for-me/CLIProxyAPI/v7/internal/runtime/executor/helps" - cliproxyauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth" - "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/usage" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - log "github.com/sirupsen/logrus" -) - -// openAICompatVisionNeedsPreprocess reports whether an OpenAI-compatible request -// should be handled by the preprocess strategy (describe images first via the -// vision model, then continue with the original text-only model). It mirrors the -// routing decision of openAICompatVisionPlan without performing any I/O, so the -// streaming path can defer the (blocking, ~seconds) vision call into the stream -// goroutine. It is the OpenAI-compat counterpart of codebuddyVisionNeedsPreprocess. -func (e *OpenAICompatExecutor) openAICompatVisionNeedsPreprocess(body []byte, baseModel string) bool { - visionCfg := e.cfg.CodebuddyVision - mode := visionCfg.NormalizedVisionMode() - if mode != config.CodebuddyVisionModePreprocess { - return false - } - if !codebuddyChatHasImageInput(body) { - return false - } - visionModel := visionCfg.VisionModel() - currentModel := strings.TrimSpace(gjson.GetBytes(body, "model").String()) - if currentModel == "" { - currentModel = strings.TrimSpace(baseModel) - } - // Never re-route the vision engine itself (avoids recursion). - if strings.EqualFold(strings.TrimSpace(currentModel), strings.TrimSpace(visionModel)) { - return false - } - return true -} - -// rewriteOpenAICompatHistoricalImages replaces image parts in historical -// messages (before the current turn) with a text marker, mirroring -// rewriteCodebuddyHistoricalImagesForTextModel. OpenAI-compatible providers -// have no native-vision capability registry, so every non-vision-engine model -// is treated as text-only (consistent with openAICompatVisionPlan being -// invoked with currentSupportsImages=false). -func (e *OpenAICompatExecutor) rewriteOpenAICompatHistoricalImages(body []byte, baseModel string) []byte { - visionCfg := e.cfg.CodebuddyVision - mode := visionCfg.NormalizedVisionMode() - if mode != config.CodebuddyVisionModePreprocess && mode != config.CodebuddyVisionModeRouting { - return body - } - currentModel := strings.TrimSpace(gjson.GetBytes(body, "model").String()) - if currentModel == "" { - currentModel = strings.TrimSpace(baseModel) - } - if strings.EqualFold(currentModel, strings.TrimSpace(visionCfg.VisionModel())) { - return body - } - body = replaceCodebuddyHistoricalImagesWithText(body, codebuddyHistoricalImageText) - // Truncated stubs inside the current turn get the same treatment (mirrors - // rewriteCodebuddyHistoricalImagesForTextModel). - body = replaceCodebuddyCurrentTurnStubsWithText(body, codebuddyHistoricalImageText) - return body -} - -// openAICompatVisionPlan is a pure function deciding how the vision-proxy layer -// should handle an OpenAI-compatible request. It mirrors codebuddyVisionPlan. -func openAICompatVisionPlan(mode, visionModel, currentModel string, hasImage, currentSupportsImages bool) codebuddyVisionAction { - if mode != config.CodebuddyVisionModeRouting && mode != config.CodebuddyVisionModePreprocess { - return codebuddyVisionPassThrough - } - if !hasImage { - return codebuddyVisionPassThrough - } - if strings.EqualFold(strings.TrimSpace(currentModel), strings.TrimSpace(visionModel)) { - return codebuddyVisionPassThrough - } - if currentSupportsImages { - return codebuddyVisionPassThrough - } - if mode == config.CodebuddyVisionModePreprocess { - return codebuddyVisionPreprocess - } - return codebuddyVisionRoute -} - -// applyOpenAICompatVisionProxy is the OpenAI-compat counterpart of -// applyCodebuddyVisionProxy. It rewrites image input for non-vision models. In -// routing mode it swaps the model; in preprocess mode it calls the vision model -// (hy4-preview by default) via the same OpenAI-compatible upstream credentials, -// then swaps image parts for the returned text descriptions. On failure it -// degrades to omitting images rather than failing the whole request. -// -// When emit is non-nil (streaming path), each vision description delta is -// forwarded through it so the user can watch the image being described in real -// time; otherwise deltas are only aggregated. -func (e *OpenAICompatExecutor) applyOpenAICompatVisionProxy(ctx context.Context, auth *cliproxyauth.Auth, body []byte, baseModel string, emit func([]byte) bool, reporter *helps.UsageReporter) ([]byte, bool) { - visionCfg := e.cfg.CodebuddyVision - mode := visionCfg.NormalizedVisionMode() - if mode == config.CodebuddyVisionModeOff { - return body, false - } - if !codebuddyChatHasImageInput(body) { - return body, false - } - - visionModel := visionCfg.VisionModel() - currentModel := strings.TrimSpace(gjson.GetBytes(body, "model").String()) - if currentModel == "" { - currentModel = strings.TrimSpace(baseModel) - } - - action := openAICompatVisionPlan(mode, visionModel, currentModel, true, false) - switch action { - case codebuddyVisionRoute: - log.Infof("openai compat vision proxy: routing %s -> %s", currentModel, visionModel) - return rewriteCodebuddyModel(body, visionModel), true - - case codebuddyVisionPreprocess: - descriptions, visionUsage, err := e.describeOpenAICompatImages(ctx, auth, body, visionModel, visionCfg.PreprocessPrompt, baseModel, emit) - if err != nil { - log.Warnf("openai compat vision proxy: preprocess failed for %s (vision=%s): %v; omitting images", currentModel, visionModel, err) - return replaceCodebuddyImagesWithText(body, codebuddyOmittedImageText), true - } - log.Infof("openai compat vision proxy: preprocessed %d image(s) for %s via %s", len(descriptions), currentModel, visionModel) - if reporter != nil { - reporter.PublishAdditionalModelAlways(ctx, visionModel, visionUsage) - } - return replaceCodebuddyImagesWithDescriptions(body, descriptions, codebuddyOmittedImageText), true - - default: - return body, false - } -} - -// describeOpenAICompatImages describes every image in the current turn, one at a -// time (serial), via the vision model using the same OpenAI-compatible upstream -// credentials as the main request. It returns one description per image, in body -// order, plus the accumulated vision-model usage across all images. It is the -// OpenAI-compat counterpart of describeImagesWithVisionModel. -func (e *OpenAICompatExecutor) describeOpenAICompatImages( - ctx context.Context, - auth *cliproxyauth.Auth, - body []byte, - visionModel, prompt string, - baseModel string, - emit func([]byte) bool, -) ([]string, usage.Detail, error) { - question := codebuddyUserQuestion(body) - fullPrompt := buildCodebuddyVisionPrompt(prompt, question) - - images := extractCodebuddyCurrentImages(body) - if len(images) == 0 { - return nil, usage.Detail{}, fmt.Errorf("vision model called with no images") - } - - var totalUsage usage.Detail - descriptions := make([]string, 0, len(images)) - for i, img := range images { - desc, u, err := e.describeOpenAICompatSingleImage(ctx, auth, body, visionModel, fullPrompt, baseModel, img, emit) - if err != nil { - log.Warnf("openai compat vision proxy: image %d/%d description failed (vision=%s): %v", i+1, len(images), visionModel, err) - descriptions = append(descriptions, "") - continue - } - addCodebuddyVisionUsage(&totalUsage, u) - descriptions = append(descriptions, desc) - } - return descriptions, totalUsage, nil -} - -// describeOpenAICompatSingleImage sends a single image to the vision model via -// the OpenAI-compatible upstream and returns its text description plus the -// vision-model usage for that call. The request body is rebuilt so that only the -// target image (plus the injected user question, if any) is sent. -func (e *OpenAICompatExecutor) describeOpenAICompatSingleImage( - ctx context.Context, - auth *cliproxyauth.Auth, - body []byte, - visionModel, prompt string, - baseModel string, - img codebuddyImagePart, - emit func([]byte) bool, -) (string, usage.Detail, error) { - baseURL, apiKey := e.resolveCredentials(auth) - if baseURL == "" { - return "", usage.Detail{}, fmt.Errorf("missing provider baseURL for vision model") - } - - question := codebuddyUserQuestion(body) - userContent := []any{json.RawMessage(img.raw)} - if question != "" { - userContent = append(userContent, map[string]any{"type": "text", "text": question}) - } - reqBody := map[string]any{ - "model": visionModel, - "messages": []any{ - map[string]any{"role": "user", "content": userContent}, - }, - } - descBody, err := json.Marshal(reqBody) - if err != nil { - return "", usage.Detail{}, err - } - descBody, err = prependCodebuddySystemMessage(descBody, prompt) - if err != nil { - return "", usage.Detail{}, err - } - descBody, err = sjson.SetBytes(descBody, "stream", true) - if err != nil { - return "", usage.Detail{}, err - } - descBody, err = sjson.SetBytes(descBody, "stream_options.include_usage", true) - if err != nil { - return "", usage.Detail{}, err - } - - url := strings.TrimSuffix(baseURL, "/") + "/chat/completions" - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(descBody)) - if err != nil { - return "", usage.Detail{}, err - } - httpReq.Header.Set("Content-Type", "application/json") - if apiKey != "" { - httpReq.Header.Set("Authorization", "Bearer "+apiKey) - } - httpReq.Header.Set("User-Agent", "cli-proxy-openai-compat-vision") - httpReq.Header.Set("Accept", "text/event-stream") - - httpClient := helps.NewProxyAwareHTTPClient(ctx, e.cfg, auth, 0) - httpResp, err := httpClient.Do(httpReq) - if err != nil { - return "", usage.Detail{}, err - } - defer func() { _ = httpResp.Body.Close() }() - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - b, _ := io.ReadAll(httpResp.Body) - return "", usage.Detail{}, statusErr{code: httpResp.StatusCode, msg: string(b)} - } - - var ( - id string - created int64 - content strings.Builder - firstEmitDone bool - usageDetail usage.Detail - ) - // The vision chunk's model field is always rewritten to baseModel so the - // client sees a single consistent model from start to finish. - model := baseModel - scanner := bufio.NewScanner(httpResp.Body) - scanner.Buffer(nil, 52_428_800) - for scanner.Scan() { - line := bytes.TrimSpace(scanner.Bytes()) - if len(line) == 0 || !bytes.HasPrefix(line, []byte("data:")) { - continue - } - payload := bytes.TrimSpace(line[len("data:"):]) - if len(payload) == 0 || bytes.Equal(payload, []byte("[DONE]")) { - continue - } - if !gjson.ValidBytes(payload) { - continue - } - if u, ok := helps.ParseOpenAIStreamUsage(line); ok { - addCodebuddyVisionUsage(&usageDetail, u) - } - res := gjson.ParseBytes(payload) - if id == "" { - id = res.Get("id").String() - } - if created == 0 { - created = res.Get("created").Int() - } - for _, ch := range res.Get("choices").Array() { - delta := ch.Get("delta") - if c := delta.Get("content"); c.Exists() && c.Type == gjson.String && c.String() != "" { - content.WriteString(c.String()) - if emit != nil { - if !firstEmitDone { - firstEmitDone = true - roleChunk := buildCodebuddyVisionChunk(id, model, created, nil, "assistant") - if !emit(roleChunk) { - return "", usage.Detail{}, ctx.Err() - } - } - contentStr := c.String() - contentChunk := buildCodebuddyVisionChunk(id, model, created, &contentStr, "") - if !emit(contentChunk) { - return "", usage.Detail{}, ctx.Err() - } - } - } - } - } - if errScan := scanner.Err(); errScan != nil { - return "", usage.Detail{}, errScan - } - - desc := strings.TrimSpace(content.String()) - if desc == "" { - return "", usage.Detail{}, fmt.Errorf("vision model returned empty description") - } - return desc, usageDetail, nil -} diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/api/handlers/handlers.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/api/handlers/handlers.go index 87c21b9..cf31c5e 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/api/handlers/handlers.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/api/handlers/handlers.go @@ -439,7 +439,6 @@ func (h *BaseAPIHandler) GetContextWithCancel(handler interfaces.APIHandler, c * } newCtx = logging.WithResponseStatusHolder(newCtx) newCtx = logging.WithResponseHeadersHolder(newCtx) - newCtx = logging.WithVisionSubagentHolder(newCtx) cancelCtx := newCtx if requestCtx != nil && requestCtx != parentCtx { diff --git a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/cliproxy/usage/manager.go b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/cliproxy/usage/manager.go index 83a9df0..c05fc6b 100644 --- a/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/cliproxy/usage/manager.go +++ b/sidecars/coderelay-proxy/third_party/CLIProxyAPI/sdk/cliproxy/usage/manager.go @@ -44,10 +44,7 @@ type Record struct { // Generate reports whether the client requested actual generation. // nil or true means generation is enabled; only an explicit false disables generation. // Use GenerateFlag to set the value and GenerateEnabled to read it with the default. - Generate *bool - // VisionSubagent marks requests handled by the pure-text vision sub-agent - // loop (text-only model + vision model), so the UI can render a badge. - VisionSubagent bool + Generate *bool RequestedAt time.Time Latency time.Duration TTFT time.Duration diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 0220d20..45817dc 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "coderelay" -version = "0.1.13" +version = "0.2.0" dependencies = [ "base64 0.22.1", "chrono", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 556a8b9..5df1e6d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "coderelay" -version = "0.1.13" +version = "0.2.0" description = "CodeRelay desktop manager for CodeBuddy local proxy" authors = ["CodeRelay contributors"] edition = "2021" diff --git a/src-tauri/src/gateway.rs b/src-tauri/src/gateway.rs index 6df79a6..4b4ad55 100644 --- a/src-tauri/src/gateway.rs +++ b/src-tauri/src/gateway.rs @@ -458,20 +458,6 @@ fn prepare_runtime_files( if api_keys.is_empty() { return Err("没有启用的 API Key，请先创建以 sk- 开头的 Key".to_string()); } - let vision_mode = if state.config.vision_tool_enabled { - let mode = state.config.vision_mode.trim(); - match mode { - "routing" | "preprocess" | "agentic" => mode, - _ => "preprocess", - } - } else { - "off" - }; - let vision_model = if state.config.vision_model.trim().is_empty() { - "hy4-preview".to_string() - } else { - state.config.vision_model.trim().to_string() - }; let config = json!({ "host": state.config.bind_host, "port": state.config.port, @@ -493,7 +479,6 @@ fn prepare_runtime_files( }, "image-generation-mode": state.config.image_generation_mode, "max-concurrent-image-requests": 1, - "codebuddy-vision": { "mode": vision_mode, "model": vision_model, "max-tool-rounds": 3 }, }); let manifest_keys: Vec = state .keys @@ -525,8 +510,6 @@ fn prepare_runtime_files( "debugLogs": state.config.debug_logs, "imageGenerationMode": state.config.image_generation_mode, "imageModels": ["codebuddy-image-1"], - "visionMode": vision_mode, - "visionModel": vision_model, }); atomic_write( &files.config_path, @@ -921,6 +904,15 @@ fn ingest_event(app: &AppHandle, inner: &Arc, value: &Value) { if let Some(success) = value.get("success").and_then(Value::as_bool) { log.success = success; } + let usage_error = value + .get("errorMessage") + .and_then(Value::as_str) + .map(str::trim) + .filter(|message| !message.is_empty()) + .map(str::to_string); + if usage_error.is_some() { + log.error = usage_error; + } } let updated = state.logs[index].clone(); state.record_usage(&updated, &prev); @@ -1144,11 +1136,16 @@ fn start_service_locked(app: &AppHandle, inner: &Arc) -> Result{state.running ? `运行中 · ${state.actualPort ?? draft.port}` : '已停止'}} /> {state.lastError &&
{state.lastError}
} -

网络

服务默认只绑定本机，局域网访问需要显式开启。

change('port', Number(e.target.value))} />
{draft.scope === 'lan' ? '局域网访问已开启' : '仅允许本机访问'}{draft.scope === 'lan' ? '同一网络中的设备可以连接此服务，请确认网络可信。' : '外部设备无法访问此服务，适合单机开发。'}

请求处理

配置超时、重试和账号选择行为。

change('requestTimeoutMs', Number(e.target.value) * 1000)} />秒
change('sessionAffinity', value)} />

协议兼容

保持 OpenAI Chat Completions 请求格式，同时支持视觉代理。

change('visionToolEnabled', value)} />{draft.visionToolEnabled && <> change('visionModel', e.target.value)} placeholder="hy3-preview" />} change('imageGenerationMode', value ? 'enabled' : 'disabled')} /> change('debugLogs', value)} />
保存配置不会自动重启服务。
连接信息

本地接口

OpenAI 兼容
Base URLhttp://localhost:{draft.port}/v1 { void copyText(`http://localhost:${draft.port}/v1`).then(() => notify('Base URL 已复制')); }}>
{draft.scope === 'lan' &&
LAN URLhttp://局域网地址:{draft.port}/v1 notify('请将“局域网地址”替换为本机实际 IPv4 地址')}>
}
API Key 鉴权POST /v1/chat/completions

接入客户端

将 Base URL 设置为上方地址，并使用 CodeRelay API Key 作为 Bearer Token。

安全提示

API Key 只保存在本机配置目录。日志和错误消息不会记录上游 Token。

; +

网络

服务默认只绑定本机，局域网访问需要显式开启。

change('port', Number(e.target.value))} />
{draft.scope === 'lan' ? '局域网访问已开启' : '仅允许本机访问'}{draft.scope === 'lan' ? '同一网络中的设备可以连接此服务，请确认网络可信。' : '外部设备无法访问此服务，适合单机开发。'}

请求处理

配置超时、重试和账号选择行为。

change('requestTimeoutMs', Number(e.target.value) * 1000)} />秒
change('sessionAffinity', value)} />

协议兼容

保持 OpenAI Chat Completions 请求格式。

change('imageGenerationMode', value ? 'enabled' : 'disabled')} /> change('debugLogs', value)} />
保存配置不会自动重启服务。
连接信息

本地接口

OpenAI 兼容
Base URLhttp://localhost:{draft.port}/v1 { void copyText(`http://localhost:${draft.port}/v1`).then(() => notify('Base URL 已复制')); }}>
{draft.scope === 'lan' &&
LAN URLhttp://局域网地址:{draft.port}/v1 notify('请将“局域网地址”替换为本机实际 IPv4 地址')}>
}
API Key 鉴权POST /v1/chat/completions

接入客户端

将 Base URL 设置为上方地址，并使用 CodeRelay API Key 作为 Bearer Token。

安全提示

API Key 只保存在本机配置目录。日志和错误消息不会记录上游 Token。

; } function Field({ label, hint, children, wide = false }: { label: string; hint?: string; children: ReactNode; wide?: boolean }) { return ; } @@ -650,7 +650,7 @@ function ModelsPage({ state, notify }: { state: AppState; notify: NoticeHandler } finally { setSyncing(false); } }; const filtered = useMemo(() => models.filter((model) => !query || model.id.toLowerCase().includes(query.toLowerCase())), [models, query]); - return <>{lastSync ? `上次同步：${formatDate(lastSync)}` : '尚未同步'}
} />
模型目录来自 CodeBuddy CN 后端没有运行服务或有效 API Key 时，不会显示伪造的模型列表。
{models.length ? '已同步' : '等待同步'}
模型目录{filtered.length} 个模型
setQuery(e.target.value)} placeholder="搜索模型" />
{filtered.length ?
模型能力可用状态来源别名
{filtered.map((model) => { const capabilities = ['文本', ...(model.supportsImages || model.inputModalities?.includes('image') ? ['视觉'] : []), ...(model.supportsToolCall ? ['工具'] : [])]; return
{model.id}{model.ownedBy ?? 'codebuddy'}
{capabilities.map((capability) => {capability})}
可用CodeBuddy CN notify(`${model.id}：上下文 ${model.contextLength ?? '未知'}`)}>
; })}
: { void sync(); }} disabled={syncing}>同步模型} />}
视觉能力由后端模型目录和账号池探测结果决定。
; + return <>{lastSync ? `上次同步：${formatDate(lastSync)}` : '尚未同步'}
} />
模型目录来自 CodeBuddy CN 后端没有运行服务或有效 API Key 时，不会显示伪造的模型列表。
{models.length ? '已同步' : '等待同步'}
模型目录{filtered.length} 个模型
setQuery(e.target.value)} placeholder="搜索模型" />
{filtered.length ?
模型能力可用状态来源别名
{filtered.map((model) => { const capabilities = ['文本', ...(model.supportsImages || model.inputModalities?.includes('image') ? ['视觉'] : []), ...(model.supportsToolCall ? ['工具'] : [])]; return
{model.id}{model.ownedBy ?? 'codebuddy'}
{capabilities.map((capability) => {capability})}
可用CodeBuddy CN notify(`${model.id}：上下文 ${model.contextLength ?? '未知'}`)}>
; })}
: { void sync(); }} disabled={syncing}>同步模型} />}
视觉能力由在线模型目录与实测校正表决定。
; } function SettingsPage({ onReset, notify }: { onReset: () => void; notify: NoticeHandler }) { diff --git a/src/types.ts b/src/types.ts index 9c8c66c..90da83d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,7 +2,6 @@ export type PageId = 'overview' | 'service' | 'keys' | 'logs' | 'accounts' | 'mo export type ThemeMode = 'light' | 'dark' | 'system'; export type ServiceScope = 'localhost' | 'lan'; export type RoutingStrategy = 'auto' | 'random' | 'single_account' | 'quota_high_first' | 'custom'; -export type VisionMode = 'off' | 'routing' | 'preprocess' | 'agentic'; export type AccountStatus = 'available' | 'needs_auth' | 'cooling' | 'restricted' | 'disabled'; export interface Account { @@ -68,9 +67,6 @@ export interface ServiceConfig { maxRetries: number; routingStrategy: RoutingStrategy; sessionAffinity: boolean; - visionToolEnabled: boolean; - visionMode: VisionMode; - visionModel: string; imageGenerationMode: 'enabled' | 'images_only' | 'disabled'; debugLogs: boolean; } @@ -175,9 +171,6 @@ export const defaultConfig: ServiceConfig = { maxRetries: 2, routingStrategy: 'auto', sessionAffinity: true, - visionToolEnabled: true, - visionMode: 'preprocess', - visionModel: 'hy4-preview', imageGenerationMode: 'enabled', debugLogs: false, };