fix(chatgpt): infer image input from OpenAI catalogue

docs/configuration/models/index.md

@@ -90,7 +90,9 @@ For custom OpenAI-compatible providers, local models (Ollama, DMR), and any
 model the built-in catalogue does not describe, Docker Agent cannot
 auto-detect whether the endpoint accepts image, PDF, audio, or video
 attachments. When the model is absent from the catalogue, Docker Agent logs a
-diagnostic and falls back to text-only, silently dropping attachments.
+diagnostic and falls back to text-only, silently dropping attachments. For the
+`chatgpt` provider, a missing entry first falls back to the matching `openai`
+entry for **image input only**; PDF, audio, and video are not inferred.
 
 Declare `capabilities` to make the model's attachment support authoritative
 and skip the catalogue lookup entirely:
@@ -130,8 +132,8 @@ models:
 
 The flags must match what the endpoint actually accepts. Claiming a modality
 that the endpoint does not support leads to a provider-side API error. When
-`capabilities` is omitted the behaviour is unchanged (catalogue lookup then
-conservative text-only fallback).
+`capabilities` is omitted, input support comes from the catalogue, the scoped
+ChatGPT image fallback above, or conservative text-only fallback.
 
 ### Unsupported media is stripped before the call
 

docs/providers/chatgpt/index.md

@@ -101,6 +101,33 @@ The effort picker exposes Low/Medium/High/XHigh/Max on the GPT-5.6 family
   there. Client-side sampling parameters (`temperature`, `top_p`,
   `max_tokens`) are not supported by the backend and are dropped.
 
+## Image Input
+
+For image-capable models such as `gpt-6.1-sol`, you can attach an image:
+
+```bash
+docker agent run examples/chatgpt-images.yaml --exec \
+  --attach examples/chatgpt-images.png "Describe the shapes and colors."
+```
+
+The sample includes a red square centered on a blue background. Running it
+requires ChatGPT sign-in and access to the selected model. Image input for
+`gpt-6.1-sol` is inferred from the matching OpenAI catalogue entry; no
+`capabilities` override is needed.
+
+When models.dev has no direct `chatgpt/<model>` entry, Docker Agent uses the
+matching `openai/<model>` entry for **image input only**. This applies both to
+`--attach` images and image content returned by MCP tools (for example,
+screenshots). Tool images are sent in a follow-up user message alongside the
+text function-call output. PDF, audio, and video support, prices, context limits,
+tool support, output capabilities, and `max_tokens` are not inherited.
+A direct ChatGPT catalogue entry takes precedence; unknown models remain
+text-only.
+
+To intentionally override catalogue capabilities or declare support for an
+unknown model, see [Attachment Capability Overrides](../../configuration/models/index.md#attachment-capability-overrides).
+Image input support does not enable image generation or editing.
+
 ## Setting the Token Explicitly
 
 `CHATGPT_OAUTH_TOKEN` can also be set like any other credential (shell

examples/README.md

@@ -210,6 +210,7 @@ remote MCP endpoints.
 | [`cloudflare-workers-ai.yaml`](cloudflare-workers-ai.yaml) | Cloudflare Workers AI edge-hosted open models. |
 | [`cloudflare-ai-gateway.yaml`](cloudflare-ai-gateway.yaml) | Cloudflare AI Gateway multi-provider router. |
 | [`grok.yaml`](grok.yaml) | xAI Grok model. |
+| [`chatgpt-images.yaml`](chatgpt-images.yaml) + [`chatgpt-images.png`](chatgpt-images.png) | ChatGPT image input with an attached sample image and automatic capability detection. |
 | [`github-copilot.yaml`](github-copilot.yaml) | GitHub Copilot models via OAuth device-flow. |
 | [`fallback_models.yaml`](fallback_models.yaml) | Automatic fallback to a secondary model when the primary fails. |
 | [`model_picker.yaml`](model_picker.yaml) | Lets the agent itself swap to a stronger model mid-conversation. |

examples/chatgpt-images.yaml

@@ -0,0 +1,20 @@
+# ChatGPT image input with automatic capability detection.
+# Requires ChatGPT sign-in and access to the selected model.
+# Image input for gpt-6.1-sol is inferred from the matching OpenAI catalogue entry.
+#
+# From the repository root:
+#   docker agent run examples/chatgpt-images.yaml --exec \
+#     --attach examples/chatgpt-images.png "Describe the shapes and colors."
+# The sample image is a red square centered on a blue background.
+# With a source build, use ./bin/docker-agent instead of docker agent.
+
+models:
+  vision:
+    provider: chatgpt
+    model: gpt-6.1-sol
+
+agents:
+  root:
+    model: vision
+    description: ChatGPT image-input example
+    instruction: Describe the attached image accurately and concisely.

pkg/model/provider/capability_override_test.go

@@ -74,3 +74,19 @@ func TestCapabilityOverride_SurvivesProviderConstruction(t *testing.T) {
 		assert.Equal(t, &modelinfo.CapsOverride{Image: true, PDF: true, Audio: true, Video: true}, got)
 	})
 }
+
+func TestCapabilityOverride_ChatGPT(t *testing.T) {
+	t.Parallel()
+
+	env := environment.NewMapEnvProvider(map[string]string{"CHATGPT_OAUTH_TOKEN": "test-token"})
+	for _, image := range []bool{false, true} {
+		cfg := &latest.ModelConfig{
+			Provider: "chatgpt", Model: "gpt-6.1-sol",
+			Capabilities: &latest.CapabilitiesConfig{Image: image},
+		}
+		p, err := fullTestRegistry().New(t.Context(), cfg, env)
+		require.NoError(t, err)
+		bc := p.BaseConfig()
+		assert.Equal(t, &modelinfo.CapsOverride{Image: image}, bc.CapsOverride())
+	}
+}

pkg/model/provider/openai/chatgpt_test.go

@@ -19,6 +19,8 @@ import (
 	"github.com/docker/docker-agent/pkg/chatgpt"
 	"github.com/docker/docker-agent/pkg/config/latest"
 	"github.com/docker/docker-agent/pkg/environment"
+	"github.com/docker/docker-agent/pkg/model/provider/options"
+	"github.com/docker/docker-agent/pkg/modelsdev"
 	"github.com/docker/docker-agent/pkg/tools"
 )
 
@@ -260,3 +262,94 @@ func TestChatGPTFallsBackToStoredLogin(t *testing.T) {
 	assert.Equal(t, "Bearer "+token, got.header.Get("Authorization"))
 	assert.Equal(t, "acc_stored", got.header.Get("chatgpt-account-id"))
 }
+
+func TestChatGPTImageInputFromOpenAICatalog(t *testing.T) {
+	t.Parallel()
+
+	store := modelsdev.NewDatabaseStore(&modelsdev.Database{Providers: map[string]modelsdev.Provider{
+		"openai": {Models: map[string]modelsdev.Model{
+			"gpt-6.1-sol": {Modalities: modelsdev.Modalities{Input: []string{"text", "image", "pdf", "audio", "video"}}},
+		}},
+	}})
+	for _, tc := range []struct {
+		name      string
+		override  *latest.CapabilitiesConfig
+		wantImage bool
+	}{
+		{name: "no override", wantImage: true},
+		{name: "explicit image true", override: &latest.CapabilitiesConfig{Image: true}, wantImage: true},
+		{name: "explicit image false", override: &latest.CapabilitiesConfig{}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			server, captured := startFakeCodexBackend(t)
+			cfg := &latest.ModelConfig{
+				Provider: "chatgpt", Model: "gpt-6.1-sol", BaseURL: server.URL,
+				TokenKey: chatgpt.TokenEnvVar, Capabilities: tc.override,
+			}
+			env := environment.NewMapEnvProvider(map[string]string{
+				chatgpt.TokenEnvVar: chatgptTestToken(t, "acc_images"),
+			})
+			client, err := NewClient(t.Context(), cfg, env, options.WithModelsDevStore(store))
+			require.NoError(t, err)
+
+			image := chat.MessagePart{Type: chat.MessagePartTypeDocument, Document: &chat.Document{
+				Name: "screenshot.png", MimeType: "image/png", Source: chat.DocumentSource{InlineData: []byte{1, 2, 3}},
+			}}
+			pdf := chat.MessagePart{Type: chat.MessagePartTypeDocument, Document: &chat.Document{
+				Name: "report.pdf", MimeType: "application/pdf", Source: chat.DocumentSource{InlineData: []byte("%PDF")},
+			}}
+			drainChatStream(t, client, []chat.Message{
+				{Role: chat.MessageRoleUser, MultiContent: []chat.MessagePart{
+					{Type: chat.MessagePartTypeText, Text: "describe this image"}, image, pdf,
+				}},
+				{Role: chat.MessageRoleAssistant, ToolCalls: []tools.ToolCall{{
+					ID: "call_image", Type: "function", Function: tools.FunctionCall{Name: "screenshot", Arguments: `{}`},
+				}}},
+				{Role: chat.MessageRoleTool, ToolCallID: "call_image", Content: "screenshot captured", MultiContent: []chat.MessagePart{image, pdf}},
+			})
+
+			input, ok := captured().body["input"].([]any)
+			require.True(t, ok)
+			wantLen := 3
+			if tc.wantImage {
+				wantLen++
+			}
+			require.Len(t, input, wantLen)
+			user := input[0].(map[string]any)
+			assert.Equal(t, "user", user["role"])
+			toolCall := input[1].(map[string]any)
+			assert.Equal(t, "function_call", toolCall["type"])
+			toolOutput := input[2].(map[string]any)
+			assert.Equal(t, "function_call_output", toolOutput["type"])
+			assert.Equal(t, "call_image", toolOutput["call_id"])
+			assert.Equal(t, "screenshot captured", toolOutput["output"])
+
+			var images int
+			for _, item := range input {
+				msg := item.(map[string]any)
+				content, _ := msg["content"].([]any)
+				for _, part := range content {
+					p := part.(map[string]any)
+					assert.NotEqual(t, "input_file", p["type"], "OpenAI PDF input must not be inherited")
+					if p["type"] == "input_image" {
+						assert.Equal(t, "user", msg["role"])
+						assert.Equal(t, "data:image/png;base64,AQID", p["image_url"])
+						images++
+					}
+				}
+			}
+			if tc.wantImage {
+				assert.Equal(t, 2, images, "attachment and tool image must both reach the backend")
+				followUp := input[3].(map[string]any)
+				assert.Equal(t, "user", followUp["role"])
+				content := followUp["content"].([]any)
+				require.Len(t, content, 2)
+				assert.Equal(t, "Attached content from tool result:", content[0].(map[string]any)["text"])
+				assert.Equal(t, "input_image", content[1].(map[string]any)["type"])
+			} else {
+				assert.Zero(t, images)
+			}
+		})
+	}
+}

pkg/modelinfo/catalog_caps.go

@@ -0,0 +1,27 @@
+package modelinfo
+
+import (
+	"context"
+
+	"github.com/docker/docker-agent/pkg/modelsdev"
+)
+
+// ModelLookup is the catalogue lookup needed to resolve input capabilities.
+type ModelLookup interface {
+	GetModel(ctx context.Context, id modelsdev.ID) (*modelsdev.Model, error)
+}
+
+// AliasedCatalogCaps resolves image input support for ChatGPT from the matching
+// OpenAI entry after a direct catalogue miss. Other capabilities, limits, and
+// pricing are not inherited.
+func AliasedCatalogCaps(ctx context.Context, store ModelLookup, id modelsdev.ID) (ModelCapabilities, bool) {
+	if store == nil || !id.IsValid() || id.Provider != "chatgpt" || ctx.Err() != nil {
+		return ModelCapabilities{}, false
+	}
+	model, err := store.GetModel(ctx, modelsdev.NewID("openai", id.Model))
+	if err != nil || model == nil || ctx.Err() != nil {
+		return ModelCapabilities{}, false
+	}
+	caps := capsFromModalities(model.Modalities.Input)
+	return CapsWith(caps.SupportsImage(), false, false, false), true
+}

pkg/modelinfo/modelinfo.go

@@ -704,9 +704,9 @@ func warnCapsLookupMiss(ctx context.Context, id modelsdev.ID, cause error) {
 // model ID using the provided store.
 //
 // When the store is nil or the model is not found, LoadCaps returns a
-// conservative capability set that only allows text MIME types. A models.dev
-// miss is logged once per model via [warnCapsLookupMiss] so the degraded
-// behaviour is diagnosable rather than silent.
+// conservative capability set that only allows text MIME types. On a direct
+// catalogue miss, [AliasedCatalogCaps] may supply scoped input capabilities.
+// An unresolved models.dev miss is logged once per model via [warnCapsLookupMiss].
 //
 // The supplied ctx is wrapped with loadCapsTimeout so the lookup stays
 // cancellable with the caller and the underlying models.dev load is bounded.
@@ -722,6 +722,9 @@ func LoadCaps(ctx context.Context, store *modelsdev.Store, id modelsdev.ID) Mode
 
 	model, err := store.GetModel(ctx, id)
 	if err != nil {
+		if caps, ok := AliasedCatalogCaps(ctx, store, id); ok {
+			return caps
+		}
 		if ctx.Err() != nil {
 			slog.WarnContext(ctx, "modelinfo: models.dev lookup timed out, using conservative caps",
 				"model", id.String(), "timeout", loadCapsTimeout)

pkg/modelinfo/resolve_caps_test.go

@@ -2,6 +2,7 @@ package modelinfo
 
 import (
 	"bytes"
+	"context"
 	"log/slog"
 	"strings"
 	"testing"
@@ -154,3 +155,71 @@ func TestResolveCapsFromModel(t *testing.T) {
 		})
 	}
 }
+
+func TestResolveCaps_ChatGPTCatalogFallback(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name     string
+		model    string
+		direct   []string
+		override *CapsOverride
+		want     ModelCapabilities
+	}{
+		{name: "image input only", model: "gpt-6.1-sol", want: CapsWith(true, false, false, false)},
+		{name: "unknown model", model: "unknown", want: CapsWith(false, false, false, false)},
+		{name: "direct entry wins", model: "gpt-6.1-sol", direct: []string{"text", "audio"}, want: CapsWith(false, false, true, false)},
+		{name: "explicit false wins", model: "gpt-6.1-sol", override: &CapsOverride{}, want: CapsWith(false, false, false, false)},
+		{name: "explicit true wins even for unknown", model: "unknown", override: &CapsOverride{Image: true, PDF: true}, want: CapsWith(true, true, false, false)},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			providers := map[string]modelsdev.Provider{
+				"openai": {Models: map[string]modelsdev.Model{
+					"gpt-6.1-sol": {Modalities: modelsdev.Modalities{Input: []string{"text", "image", "pdf", "audio", "video"}}},
+				}},
+			}
+			if tc.direct != nil {
+				providers["chatgpt"] = modelsdev.Provider{Models: map[string]modelsdev.Model{
+					tc.model: {Modalities: modelsdev.Modalities{Input: tc.direct}},
+				}}
+			}
+			store := modelsdev.NewDatabaseStore(&modelsdev.Database{Providers: providers})
+			got := ResolveCaps(t.Context(), store, modelsdev.NewID("chatgpt", tc.model), tc.override)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestAliasedCatalogCaps_Misses(t *testing.T) {
+	t.Parallel()
+	store := modelsdev.NewDatabaseStore(&modelsdev.Database{Providers: map[string]modelsdev.Provider{
+		"openai": {Models: map[string]modelsdev.Model{
+			"vision":    {Modalities: modelsdev.Modalities{Input: []string{"image", "pdf"}}},
+			"text-only": {Modalities: modelsdev.Modalities{Input: []string{"text"}, Output: []string{"image"}}},
+		}},
+	}})
+	for _, id := range []modelsdev.ID{
+		modelsdev.NewID("chatgpt", "missing"),
+		modelsdev.NewID("other", "vision"),
+		modelsdev.NewID("", "vision"),
+		modelsdev.NewID("chatgpt", ""),
+	} {
+		caps, ok := AliasedCatalogCaps(t.Context(), store, id)
+		assert.False(t, ok, id.String())
+		assert.Equal(t, ModelCapabilities{}, caps)
+	}
+	caps, ok := AliasedCatalogCaps(t.Context(), store, modelsdev.NewID("chatgpt", "text-only"))
+	assert.True(t, ok)
+	assert.Equal(t, ModelCapabilities{}, caps, "output images must not imply image input")
+
+	caps, ok = AliasedCatalogCaps(t.Context(), nil, modelsdev.NewID("chatgpt", "vision"))
+	assert.False(t, ok)
+	assert.Equal(t, ModelCapabilities{}, caps)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	caps, ok = AliasedCatalogCaps(ctx, store, modelsdev.NewID("chatgpt", "vision"))
+	assert.False(t, ok)
+	assert.Equal(t, ModelCapabilities{}, caps)
+}

pkg/runtime/transforms.go

@@ -99,7 +99,11 @@ func (r *LocalRuntime) prepareMessagesForModel(
 	cfg := model.BaseConfig()
 	caps := modelinfo.ResolveCapsFromModel(catalogModel, cfg.CapsOverride())
 	if catalogModel == nil && cfg.CapsOverride() == nil {
-		caps = providerFallbackCaps(ctx, cfg.ModelConfig, modelID)
+		if aliased, ok := modelinfo.AliasedCatalogCaps(ctx, r.modelsStore, modelID); ok {
+			caps = aliased
+		} else {
+			caps = providerFallbackCaps(ctx, cfg.ModelConfig, modelID)
+		}
 	}
 	return r.applyBeforeLLMCallTransforms(ctx, sess, a, modelID.String(), &caps, msgs)
 }