From a8772f4dffedd34fac9e815e7b1fb30639d0db2c Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 16 Sep 2026 21:01:09 -0400 Subject: [PATCH] Add provider-aware image generation and Custom endpoints Separate image capabilities by provider and endpoint cloud, add MAI Image and FLUX adapters, and make image editing capability-aware while preserving shared connection policies. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_ai_connection_migration.py | 33 +- .../single_app/functions_ai_connections.py | 87 +-- application/single_app/functions_documents.py | 7 +- .../single_app/functions_image_adapters.py | 125 ++++ .../single_app/functions_image_api_route.py | 152 +++-- .../functions_image_capabilities.py | 324 ++++++++++ .../single_app/functions_image_edit.py | 356 ++++------- .../single_app/functions_image_generation.py | 224 +++++-- application/single_app/functions_keyvault.py | 5 +- .../functions_model_capabilities.py | 26 +- .../functions_model_endpoint_auth.py | 188 ++++++ .../functions_model_endpoint_diagnostics.py | 104 ++++ .../functions_model_endpoint_providers.py | 161 +++++ .../functions_model_endpoint_runtime.py | 174 +++++- .../functions_model_endpoint_types.py | 63 ++ .../functions_model_endpoint_validation.py | 258 ++++++++ .../functions_orchestration_models.py | 7 +- application/single_app/functions_settings.py | 69 ++- .../single_app/functions_workflow_runner.py | 15 +- .../single_app/model_endpoint_clients.py | 536 ++++++++++++++++- .../single_app/route_backend_agents.py | 3 + application/single_app/route_backend_chats.py | 22 +- .../single_app/route_backend_collaboration.py | 3 +- .../route_backend_conversation_export.py | 16 +- .../single_app/route_backend_models.py | 75 ++- .../single_app/route_backend_settings.py | 9 +- application/single_app/route_backend_v2.py | 37 +- .../route_frontend_admin_settings.py | 20 +- .../single_app/route_frontend_chats.py | 5 +- .../single_app/semantic_kernel_loader.py | 36 +- .../static/js/admin/admin_model_endpoints.js | 291 ++++++++- .../static/json/model_capabilities.json | 378 +++++++++++- .../schemas/model_capabilities.schema.json | 73 +++ .../admin/_panes/model-endpoints.html | 82 ++- .../admin/CapabilityModelPicker.tsx | 55 +- .../admin/CustomConnectionFields.tsx | 130 ++++ .../admin/CustomNetworkPolicyEditor.tsx | 61 ++ .../admin/ModelConnectionsManager.tsx | 144 ++++- .../v2_ui/src/components/chat/ImageEditor.tsx | 309 ++++++---- application/v2_ui/src/lib/capabilityModels.ts | 72 ++- .../v2_ui/src/lib/customModelConnections.ts | 142 +++++ application/v2_ui/src/lib/endpoints.ts | 6 +- application/v2_ui/src/lib/imageRevisions.ts | 178 ++++-- application/v2_ui/src/lib/modelConnections.ts | 66 ++- application/v2_ui/src/lib/types.ts | 25 +- application/v2_ui/src/stores/chatStore.ts | 4 + docs/admin/ai-models.md | 100 +++- .../features/AI_CONNECTIONS_FRAMEWORK.md | 51 +- .../CUSTOM_MODEL_ENDPOINT_PROVIDER.md | 142 +++++ .../IMAGE_GENERATION_RESPONSES_MODELS.md | 117 ++-- .../features/MODEL_CAPABILITY_CATALOG.md | 132 +++++ .../GPT_CHAT_MODEL_IMAGE_GENERATION_FIX.md | 5 + .../fixes/IMAGE_PROVIDER_CAPABILITIES_FIX.md | 130 ++++ docs/guides/configure-ai-connections.md | 60 +- docs/guides/generate-images.md | 43 +- docs/reference/chat-controls.md | 19 +- ..._admin_multi_endpoint_persistence_guard.py | 39 +- .../test_ai_connection_defaults_api.py | 38 +- .../test_ai_connection_image_migration.py | 164 +++++- .../test_ai_connection_image_runtime.py | 325 ++++++++-- .../test_ai_connections_capabilities.py | 554 +++++++++++++++--- ...rsation_summary_model_endpoint_protocol.py | 5 +- .../test_custom_model_connections_ui_logic.js | 78 +++ .../test_custom_model_endpoint_foundation.py | 543 +++++++++++++++++ .../test_image_chat_route_integration.py | 49 +- .../test_image_custom_output_transport.py | 135 +++++ .../test_image_edit_provider_operations.py | 237 ++++++++ .../test_image_generation_responses_route.py | 301 +++++++++- .../test_image_generation_sdk_http.py | 326 +++++++++-- .../test_image_provider_capabilities.py | 245 ++++++++ .../test_image_provider_sdk_http.py | 214 +++++++ ...l_endpoint_management_cloud_environment.py | 7 +- ..._model_endpoint_payload_auth_type_order.py | 6 +- ...odel_endpoints_key_vault_secret_storage.py | 46 +- .../test_new_foundry_streaming_runtime.py | 6 +- .../test_v2_admin_model_endpoints_api.py | 3 +- .../test_v2_image_capability_logic.mjs | 346 +++++++++++ functional_tests/test_v2_image_editor.py | 11 +- .../test_workflow_model_core_capabilities.py | 3 +- ui_tests/fixtures/ai_connections_admin.py | 47 +- .../fixtures/classic_ai_connections_admin.py | 6 +- ui_tests/fixtures/custom_connections_admin.py | 140 +++++ ui_tests/fixtures/image_editor.py | 343 +++++++++++ ui_tests/test_admin_custom_connections.py | 150 +++++ ui_tests/test_admin_shared_ai_connections.py | 48 +- ...est_admin_shared_ai_connections_classic.py | 50 +- ui_tests/test_image_editor_capabilities.py | 236 ++++++++ 88 files changed, 9440 insertions(+), 1218 deletions(-) create mode 100644 application/single_app/functions_image_adapters.py create mode 100644 application/single_app/functions_image_capabilities.py create mode 100644 application/single_app/functions_model_endpoint_auth.py create mode 100644 application/single_app/functions_model_endpoint_diagnostics.py create mode 100644 application/single_app/functions_model_endpoint_providers.py create mode 100644 application/single_app/functions_model_endpoint_types.py create mode 100644 application/single_app/functions_model_endpoint_validation.py create mode 100644 application/single_app/static/json/schemas/model_capabilities.schema.json create mode 100644 application/v2_ui/src/components/admin/CustomConnectionFields.tsx create mode 100644 application/v2_ui/src/components/admin/CustomNetworkPolicyEditor.tsx create mode 100644 application/v2_ui/src/lib/customModelConnections.ts create mode 100644 docs/explanation/features/CUSTOM_MODEL_ENDPOINT_PROVIDER.md create mode 100644 docs/explanation/features/MODEL_CAPABILITY_CATALOG.md create mode 100644 docs/explanation/fixes/IMAGE_PROVIDER_CAPABILITIES_FIX.md create mode 100644 functional_tests/test_custom_model_connections_ui_logic.js create mode 100644 functional_tests/test_custom_model_endpoint_foundation.py create mode 100644 functional_tests/test_image_custom_output_transport.py create mode 100644 functional_tests/test_image_edit_provider_operations.py create mode 100644 functional_tests/test_image_provider_capabilities.py create mode 100644 functional_tests/test_image_provider_sdk_http.py create mode 100644 functional_tests/test_v2_image_capability_logic.mjs create mode 100644 ui_tests/fixtures/custom_connections_admin.py create mode 100644 ui_tests/fixtures/image_editor.py create mode 100644 ui_tests/test_admin_custom_connections.py create mode 100644 ui_tests/test_image_editor_capabilities.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 32f7b3c4a..a5e984980 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.106" +VERSION = "0.261.107" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_ai_connection_migration.py b/application/single_app/functions_ai_connection_migration.py index 057c3ab05..d532e01e7 100644 --- a/application/single_app/functions_ai_connection_migration.py +++ b/application/single_app/functions_ai_connection_migration.py @@ -98,13 +98,14 @@ def _legacy_image_connection(settings, apim): if deployment == active_name: support = resolve_model_capability(model, IMAGE_GENERATION_CAPABILITY, "aoai") model_name = str(model.get("modelName") or "").lower() - legacy_route = ( - "responses" - if not apim and model_name and not any(marker in model_name for marker in ("image", "dall-e", "dalle")) - else "images" - ) - model["supportsImageGeneration"] = True - model["image_generation_api"] = support["api"] or legacy_route + if model.get("supportsImageGeneration") is not False: + if support["supported"]: + model["supportsImageGeneration"] = True + model["image_generation_api"] = support["api"] + elif not model_name: + # An unrecorded legacy Images deployment remains recoverable, not a GPT tool guess. + model["supportsImageGeneration"] = True + model["image_generation_api"] = "images" models.append(model) auth_type = "api_key" if apim else str(settings.get("azure_openai_image_gen_authentication_type") or "key") @@ -177,7 +178,9 @@ def _compatible_connection(existing, imported): (model for model in existing.get("models") or [] if model.get("deploymentName") == incoming["deploymentName"]), None, ) - if match and incoming["enabled"] and not supports_model_capability(match, IMAGE_GENERATION_CAPABILITY, existing.get("provider")): + if match and incoming["enabled"] and not supports_model_capability( + match, IMAGE_GENERATION_CAPABILITY, existing.get("provider"), endpoint=existing + ): return False return bool(existing.get("id")) @@ -196,6 +199,7 @@ def build_image_connection_migration(settings, normalize_endpoint=None): endpoints = copy.deepcopy(current) selected = dict(EMPTY_MODEL_SELECTION) imported_count = 0 + default_warning = "" active_apim = bool(settings.get("enable_image_gen_apim")) for apim in (False, True): imported, active_name = _legacy_image_connection(settings, apim) @@ -228,7 +232,12 @@ def build_image_connection_migration(settings, normalize_endpoint=None): "model_id": str(model["id"]), "provider": str(existing["provider"]).lower(), } - return { + if not supports_model_capability(model, IMAGE_GENERATION_CAPABILITY, existing["provider"], endpoint=existing): + default_warning = ( + "The imported image default is not supported on its provider. " + "Select a dedicated image model in AI Connections; the original settings have been retained." + ) + patch = { "model_endpoints": endpoints, IMAGE_SELECTION_KEY: selected, IMAGE_MIGRATION_VERSION_KEY: IMAGE_MIGRATION_VERSION, @@ -238,6 +247,12 @@ def build_image_connection_migration(settings, normalize_endpoint=None): "message": "Existing image configuration is now managed through AI Connections.", }, } + if default_warning: + patch["ai_connection_default_notices"] = { + **(settings.get("ai_connection_default_notices") or {}), + IMAGE_GENERATION_CAPABILITY: default_warning, + } + return patch def migrate_image_connections( diff --git a/application/single_app/functions_ai_connections.py b/application/single_app/functions_ai_connections.py index c27d97671..9c885b7cc 100644 --- a/application/single_app/functions_ai_connections.py +++ b/application/single_app/functions_ai_connections.py @@ -15,6 +15,11 @@ get_model_catalog_capabilities, resolve_model_vision_support, ) +from functions_image_capabilities import ( + IMAGE_APIS, + IMAGE_PROVIDERS, + resolve_image_model_capability, +) CHAT_CAPABILITY = "chat" @@ -23,7 +28,6 @@ IMAGE_MIGRATION_VERSION_KEY = "ai_connections_image_migration_version" IMAGE_MIGRATION_VERSION = 1 EMPTY_MODEL_SELECTION = {"endpoint_id": "", "model_id": "", "provider": ""} -IMAGE_PROVIDERS = ("aoai", "aifoundry", "new_foundry") _DIRECT_IMAGE_PATTERN = re.compile(r"^(?:gpt-image(?:-|$)|dall-e(?:-|$)|dalle(?:-|$))") _NON_CHAT_PATTERN = re.compile( r"embedding|^whisper|(?:^|-)(?:tts|transcribe|realtime)(?:-|$)" @@ -55,7 +59,7 @@ class CapabilityDefinition: IMAGE_SELECTION_KEY, "generatesImages", IMAGE_PROVIDERS, - feature_flag="enable_image_generation", api_routes=("images", "responses"), + feature_flag="enable_image_generation", api_routes=IMAGE_APIS, ), } _CLIENT_FACTORIES = {} @@ -172,7 +176,7 @@ def _support(supported, source, reason="", api=""): } -def resolve_model_capability(model, capability, provider="aoai"): +def resolve_model_capability(model, capability, provider="aoai", *, endpoint=None): """Describe technical support; availability and transient service health are separate.""" definition = get_capability_definition(capability) provider = str(provider or "aoai").lower() @@ -200,7 +204,8 @@ def resolve_model_capability(model, capability, provider="aoai"): for field_name in ("modelName", "behavior_name") ) ) - if direct_image or (underlying_is_named and _NON_CHAT_PATTERN.search(name)): + image_only = catalog and catalog.get("generatesImages") is True and catalog.get("generatesText") is False + if direct_image or image_only or (underlying_is_named and _NON_CHAT_PATTERN.search(name)): return _support(False, "model", "This model is not supported by the text-chat adapter.") declared = _declared_flag(model, "supportsChat", "supports_chat") if declared is not None: @@ -212,23 +217,8 @@ def resolve_model_capability(model, capability, provider="aoai"): return _support(bool(name), "legacy", "" if name else "A deployment name is required.", "chat") if capability == IMAGE_GENERATION_CAPABILITY: - declared = _declared_flag(model, "supportsImageGeneration", "supports_image_generation") - route = str(model.get("image_generation_api") or "") if isinstance(model, Mapping) else "" - if declared is False: - return _support(False, "declared", "Image generation is not supported by this model.") - if direct_image: - return _support(True, "model", api="images") - if declared is True: - return _support(True, "declared", api=route if route in ("images", "responses") else "responses") - if catalog and catalog.get("imageGenerationTool") is True: - return _support(True, "catalog", api="responses") - if catalog and catalog.get("generatesImages") is True: - return _support( - False, - "provider", - "This image model requires a provider-specific image adapter or an explicitly declared compatible image API.", - ) - return _support(False, "unknown", "Image generation support has not been established for this model.") + support = resolve_image_model_capability(model, endpoint, provider) + return _support(support["supported"], support["source"], support["reason"], support["api"]) supported = bool(catalog and catalog.get(definition.catalog_flag)) return _support( @@ -238,7 +228,7 @@ def resolve_model_capability(model, capability, provider="aoai"): ) -def supports_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai"): +def supports_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai", *, endpoint=None): """Return whether a model is technically suitable and published for this operation.""" if isinstance(model, Mapping): if model.get("enabled") is False: @@ -246,12 +236,12 @@ def supports_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai" enabled_capabilities = model.get("enabled_capabilities") if isinstance(enabled_capabilities, list) and capability not in enabled_capabilities: return False - return resolve_model_capability(model, capability, provider)["supported"] + return resolve_model_capability(model, capability, provider, endpoint=endpoint)["supported"] -def require_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai"): +def require_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai", *, endpoint=None): """Reject incompatible saved/free-form bindings at an inference boundary.""" - if not supports_model_capability(model, capability, provider): + if not supports_model_capability(model, capability, provider, endpoint=endpoint): definition = get_capability_definition(capability) raise AIConnectionError( f"The selected model is not available for {definition.label.lower()}. Choose a compatible model.", @@ -264,7 +254,7 @@ def normalize_model_capability_fields(model): """Validate new metadata without discarding unrelated, existing model properties.""" normalized = dict(model) normalized.pop("capability_status", None) - for field in ("supportsChat", "supportsImageGeneration"): + for field in ("supportsChat", "supportsImageGeneration", "supportsImageEditing", "supportsImageMasking"): if field in normalized and not isinstance(normalized[field], bool): raise AIConnectionError(f"{field} must be true or false.") if "enabled_capabilities" in normalized: @@ -274,8 +264,10 @@ def normalize_model_capability_fields(model): ): raise AIConnectionError("Model availability must name implemented AI capabilities.") normalized["enabled_capabilities"] = list(dict.fromkeys(values)) - if normalized.get("image_generation_api") not in (None, "", "images", "responses"): - raise AIConnectionError("The image-generation API must be Images or Responses.") + if normalized.get("image_generation_api") not in (None, "", *IMAGE_APIS): + raise AIConnectionError("The image-generation API must name an implemented image operation.") + if normalized.get("supportsImageMasking") is True and normalized.get("supportsImageEditing") is False: + raise AIConnectionError("Masked image editing requires image-editing support.") return normalized @@ -291,7 +283,7 @@ def filter_model_endpoints_by_capability(endpoints, capability=CHAT_CAPABILITY, for model in endpoint.get("models") or [] if isinstance(model, Mapping) and endpoint.get("enabled") is not False - and supports_model_capability(model, capability, endpoint.get("provider")) + and supports_model_capability(model, capability, endpoint.get("provider"), endpoint=endpoint) ] if models or preserve_empty: projected = copy.deepcopy(endpoint) @@ -326,6 +318,11 @@ def resolve_capability_model_selection(selection, endpoints, capability=CHAT_CAP ) if endpoint is None or endpoint.get("enabled") is False: return dict(EMPTY_MODEL_SELECTION), f"{definition.label} connection is unavailable. Select another connection." + if ( + capability == IMAGE_GENERATION_CAPABILITY and normalized["provider"] + and normalized["provider"] != str(endpoint.get("provider") or "aoai").lower() + ): + return dict(EMPTY_MODEL_SELECTION), "The selected image model does not belong to that provider." model = next( ( item for item in endpoint.get("models") or [] @@ -334,7 +331,7 @@ def resolve_capability_model_selection(selection, endpoints, capability=CHAT_CAP ), None, ) - if model is None or not supports_model_capability(model, capability, endpoint.get("provider")): + if model is None or not supports_model_capability(model, capability, endpoint.get("provider"), endpoint=endpoint): return dict(EMPTY_MODEL_SELECTION), f"{definition.label} model is unavailable or incompatible. Select a compatible model." normalized["provider"] = str(endpoint.get("provider") or "aoai").lower() return normalized, None @@ -383,29 +380,47 @@ def build_capability_model_catalog(endpoints, capability=CHAT_CAPABILITY): model_id = str(model.get("id") or model.get("deploymentName") or "") if not model_id: continue + request_model = str(model.get("deploymentName") or model.get("deployment") or model_id) + if endpoint.get("provider") == "custom": + # The optional Custom registry owns model-name versus deployment-name semantics. + from functions_model_endpoint_types import resolve_model_endpoint_request_model + + request_model = resolve_model_endpoint_request_model(endpoint, model) + support = ( + resolve_image_model_capability(model, endpoint, endpoint.get("provider")) + if capability == IMAGE_GENERATION_CAPABILITY + else resolve_model_capability(model, capability, endpoint.get("provider"), endpoint=endpoint) + ) + support.pop("connection_provider", None) choices.append({ "endpoint_id": endpoint_id, "model_id": model_id, "provider": str(endpoint.get("provider") or "aoai"), "connection_name": str(endpoint.get("name") or "Connection"), "label": str(model.get("displayName") or model.get("modelName") or model.get("deploymentName") or model_id), - "deployment_name": str(model.get("deploymentName") or model.get("deployment") or model_id), - "capability": resolve_model_capability(model, capability, endpoint.get("provider")), + "deployment_name": request_model, + "capability": support, }) return sorted(choices, key=lambda item: (item["connection_name"].lower(), item["label"].lower(), item["endpoint_id"], item["model_id"])) -def describe_model_capabilities(model, provider="aoai"): +def describe_model_capabilities(model, provider="aoai", *, endpoint=None): """Public technical support and publication metadata, never connection secrets.""" result = {} for key in CAPABILITY_DEFINITIONS: + support = ( + resolve_image_model_capability(model, endpoint, provider) + if key == IMAGE_GENERATION_CAPABILITY + else resolve_model_capability(model, key, provider, endpoint=endpoint) + ) + support.pop("connection_provider", None) result[key] = { - **resolve_model_capability(model, key, provider), - "available": supports_model_capability(model, key, provider), + **support, + "available": supports_model_capability(model, key, provider, endpoint=endpoint), } supports_vision, source = resolve_model_vision_support(model) result["vision"] = { - "supported": supports_vision and resolve_model_capability(model, CHAT_CAPABILITY, provider)["supported"], + "supported": supports_vision and resolve_model_capability(model, CHAT_CAPABILITY, provider, endpoint=endpoint)["supported"], "source": source, } return result diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index a01e89a96..2c38707d7 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -38,6 +38,8 @@ from functions_keyvault import SecretReturnType, keyvault_model_endpoint_get_helper from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers from functions_model_endpoint_runtime import MODEL_ENDPOINT_PROVIDER_ALLOWLIST, build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model +from model_endpoint_clients import MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, infer_model_endpoint_protocol import azure.cognitiveservices.speech as speechsdk _AUDIO_RUNTIME_CAPABILITIES_CACHE = None @@ -186,13 +188,14 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): connection = endpoint_cfg.get("connection", {}) or {} auth_settings = endpoint_cfg.get("auth", {}) or {} - deployment = str(model_cfg.get("deploymentName") or model_cfg.get("deployment") or "").strip() + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) endpoint = str(connection.get("endpoint") or "").strip() api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip() if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST: raise ValueError(f"Selected metadata extraction provider '{provider}' is not supported.") - if not endpoint or not api_version or not deployment: + protocol = infer_model_endpoint_protocol(provider, endpoint, deployment, get_model_endpoint_api_type(endpoint_cfg)) + if not endpoint or not deployment or (protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version): raise ValueError("Selected metadata extraction endpoint is missing endpoint, API version, or deployment configuration.") return _build_model_endpoint_client( diff --git a/application/single_app/functions_image_adapters.py b/application/single_app/functions_image_adapters.py new file mode 100644 index 000000000..ce242d55c --- /dev/null +++ b/application/single_app/functions_image_adapters.py @@ -0,0 +1,125 @@ +# functions_image_adapters.py +"""Image wire operations over an already authorized, provider-authenticated client.""" + +import base64 + +from functions_image_api_route import build_image_generation_tool +from functions_image_capabilities import validate_image_options + + +def _native_dimensions(capability, size): + sizes = capability.get("sizes") or [] + if not size and not sizes: + raise ValueError("The image model has no configured output dimensions.") + resolved_size = size or sizes[0] + validate_image_options(capability, size=resolved_size) + return tuple(int(part) for part in resolved_size.split("x")) + + +def _source_data_url(source): + return f'data:{source["mime_type"]};base64,{base64.b64encode(source["bytes"]).decode("ascii")}' + + +def _image_options(size, quality, background): + arguments = {"n": 1} + if size: + arguments["size"] = size + optional = {name: value for name, value in {"quality": quality, "background": background}.items() if value} + if optional: + arguments["extra_body"] = optional + return arguments + + +def generate_image(client, model, capability, prompt, size="", quality="", background=""): + """Generate one image without imposing an OpenAI schema on MAI or FLUX.""" + validate_image_options(capability, size, quality, background) + api = capability["api"] + if api == "responses": + return client.responses.create( + model=model, + input=prompt, + tools=[build_image_generation_tool(size, quality, background)], + tool_choice={"type": "image_generation"}, + ) + if capability.get("transport") == "openai_images": + size = size or capability["sizes"][0] + return client.images.generate(model=model, prompt=prompt, **_image_options(size, quality, background)) + if api == "images": + return client.images.generate(model=model, prompt=prompt, **_image_options(size, quality, background)) + width, height = _native_dimensions(capability, size) + body = {"model": model, "prompt": prompt, "width": width, "height": height} + if api == "mai": + return client.post("images/generations", cast_to=dict[str, object], body=body) + if api == "flux": + body["output_format"] = "png" + if capability["model_path"] == "flux-2-pro": + body["num_images"] = 1 + elif capability["model_path"] == "flux-pro-1.1": + body["n"] = 1 + return client.post(capability["model_path"], cast_to=dict[str, object], body=body) + raise ValueError("The image operation is unsupported.") + + +def edit_image(client, model, capability, prompt, source, mask=None, size="", quality="", background=""): + """Edit actual source bytes; a mask is never silently degraded into a prompt-only request.""" + validate_image_options(capability, size, quality, background) + if not capability.get("editing"): + raise ValueError("The selected image model does not support source-image editing.") + if mask and not capability.get("masking"): + raise ValueError("The selected image model does not support uploaded masks.") + if not source or not isinstance(source.get("bytes"), bytes) or not source["bytes"]: + raise ValueError("A source image is required for editing.") + if source.get("mime_type") not in capability.get("input_formats", []): + raise ValueError("The source image format is unsupported by this model.") + image_file = (source["file_name"], source["bytes"], source["mime_type"]) + api = capability["api"] + if api == "responses": + tool = build_image_generation_tool(size, quality, background) + tool["action"] = "edit" + if mask: + tool["input_image_mask"] = { + "image_url": f'data:image/png;base64,{base64.b64encode(mask["bytes"]).decode("ascii")}', + } + return client.responses.create( + model=model, + input=[{ + "role": "user", + "content": [ + {"type": "input_text", "text": prompt}, + {"type": "input_image", "image_url": _source_data_url(source)}, + ], + }], + tools=[tool], + tool_choice={"type": "image_generation"}, + ) + if api == "images" or capability.get("transport") == "openai_images": + if capability.get("transport") == "openai_images": + size = size or capability["sizes"][0] + arguments = _image_options(size, quality, background) + if mask: + arguments["mask"] = ("mask.png", mask["bytes"], "image/png") + return client.images.edit(model=model, prompt=prompt, image=image_file, **arguments) + if api == "mai": + # MAI edits accept the reference image and prompt, not generation-only dimensions. + if size: + raise ValueError("MAI source edits do not accept an output-size override; regenerate to change dimensions.") + return client.post( + "images/edits", + cast_to=dict[str, object], + body={"model": model, "prompt": prompt}, + files={"image": image_file}, + options={"headers": {"Content-Type": "multipart/form-data"}}, + ) + if api == "flux": + if size: + raise ValueError("Native FLUX reference edits do not expose a size override here; regenerate to change dimensions.") + return client.post( + capability["model_path"], + cast_to=dict[str, object], + body={ + "model": model, "prompt": prompt, + "output_format": "png", + "input_image": base64.b64encode(source["bytes"]).decode("ascii"), + }, + ) + raise ValueError("The image edit operation is unsupported.") diff --git a/application/single_app/functions_image_api_route.py b/application/single_app/functions_image_api_route.py index cfc944948..69a1be24f 100644 --- a/application/single_app/functions_image_api_route.py +++ b/application/single_app/functions_image_api_route.py @@ -2,11 +2,9 @@ """Pure image route, shared-binding, and response validation helpers. -Dedicated image models use Images; compatible GPT deployments use the v1 Responses -image tool. Technical support comes from the shared capability catalog or explicit -model metadata, not from generic tool or vision support. Neither proves live service -availability. Unmigrated legacy image settings retain their existing Images route -when the underlying model was never recorded. +Azure image selections use dedicated image APIs. Compatible direct OpenAI Custom +models may also use the Responses image tool. Provider-qualified catalog metadata +decides the operation; neither vision nor a GPT name proves image support. """ import base64 @@ -25,20 +23,19 @@ resolve_model_capability, supports_model_capability, ) +from functions_image_capabilities import ( + IMAGE_APIS, + resolve_image_endpoint_context, + resolve_image_model_capability, +) # The two ways an image is produced. Named rather than expressed as a boolean because # "not the images endpoint" is not a description of anything. IMAGE_API_ROUTE_IMAGES = 'images' IMAGE_API_ROUTE_RESPONSES = 'responses' - -# Models that serve /images/generations. Matched as substrings because a deployment -# reports a model name like ``gpt-image-1.5`` or ``dall-e-3`` and the family is what -# decides the route, not the version. ``image`` is deliberately the broad form rather than -# ``gpt-image``: it is what the discovery filter has always matched on, so narrowing it -# here would drop a deployment that was previously selectable and send it to a route it -# cannot serve. -IMAGES_ENDPOINT_MODEL_MARKERS = ('image', 'dall-e', 'dalle') +IMAGE_API_ROUTE_MAI = 'mai' +IMAGE_API_ROUTE_FLUX = 'flux' RESPONSES_IMAGE_API_VERSION = 'v1' # New shared connections support both Images generations and edits without inheriting @@ -69,13 +66,19 @@ def __init__(self, message, code='image_generation_failed', status_code=502, con def resolve_image_binding_deployment(binding): - """Use the deployment, never a stable registry ID, as the provider's model value.""" - deployment = str( - binding.model.get('deploymentName') or binding.model.get('deployment') or '' - ).strip() + """Use the provider's wire model identity, never a stable registry ID.""" + if binding.endpoint.get('provider') == 'custom': + # Keep the optional Custom registry out of the legacy bootstrap import path. + from functions_model_endpoint_types import resolve_model_endpoint_request_model + + deployment = resolve_model_endpoint_request_model(binding.endpoint, binding.model) + else: + deployment = str( + binding.model.get('deploymentName') or binding.model.get('deployment') or '' + ).strip() if not deployment: raise AIConnectionError( - 'The selected image model has no deployment configured.', + 'The selected image model has no model or deployment identifier configured.', 'model_configuration_unavailable', ) return deployment @@ -95,23 +98,25 @@ def resolve_shared_image_binding(settings): def resolve_image_binding_api(binding): """Resolve only an established, enabled image capability on the selected connection.""" provider = binding.endpoint.get('provider') or 'aoai' - support = resolve_model_capability(binding.model, IMAGE_GENERATION_CAPABILITY, provider) + support = resolve_model_capability( + binding.model, IMAGE_GENERATION_CAPABILITY, provider, endpoint=binding.endpoint + ) if ( binding.capability != IMAGE_GENERATION_CAPABILITY or binding.endpoint.get('enabled') is False - or not supports_model_capability(binding.model, IMAGE_GENERATION_CAPABILITY, provider) + or not supports_model_capability(binding.model, IMAGE_GENERATION_CAPABILITY, provider, endpoint=binding.endpoint) ): raise AIConnectionError( support.get('reason') or 'The selected model is not available for image generation.', 'model_capability_unavailable', ) profile = get_connection_operation_settings(binding.endpoint, IMAGE_GENERATION_CAPABILITY) - if profile.get('api') not in (None, '', IMAGE_API_ROUTE_IMAGES, IMAGE_API_ROUTE_RESPONSES): + if profile.get('api') not in (None, '', *IMAGE_APIS): raise AIConnectionError('The image connection has an unsupported API route.') # A connection can host both kinds of model. A migrated operation profile must # not force a newly selected GPT deployment onto its previous Images route. route = support.get('api') or profile.get('api') - if route not in (IMAGE_API_ROUTE_IMAGES, IMAGE_API_ROUTE_RESPONSES): + if route not in IMAGE_APIS: raise AIConnectionError('The selected model has no compatible image API.', 'unsupported_capability') return route @@ -167,9 +172,8 @@ def resolve_selected_image_deployment_name(settings): def resolve_image_api_route(settings): """Return which API the selected image deployment is reached through. - Defaults to the images endpoint. Every deployment selectable before the Responses - route existed answers to it, so an unrecognised or unrecorded model name leaves an - existing configuration exactly where it was. + Only an unrecorded legacy model retains its existing Images route. Named models + must qualify for the configured provider; an unknown name is not an image-tool guess. """ if image_settings_use_connections(settings): return resolve_image_binding_api(resolve_shared_image_binding(settings)) @@ -178,9 +182,6 @@ def resolve_image_api_route(settings): if not model_name: return IMAGE_API_ROUTE_IMAGES - if any(marker in model_name for marker in IMAGES_ENDPOINT_MODEL_MARKERS): - return IMAGE_API_ROUTE_IMAGES - support = resolve_model_capability( {'modelName': model_name}, IMAGE_GENERATION_CAPABILITY, 'aoai' ) @@ -192,10 +193,10 @@ def resolve_image_api_route(settings): return support['api'] -def is_image_capable_model_name(model_name): +def is_image_capable_model_name(model_name, provider='aoai', endpoint=None): """Compatibility entry point for discovery, using the shared technical catalog.""" return resolve_model_capability( - {'modelName': str(model_name or '').strip()}, IMAGE_GENERATION_CAPABILITY, 'aoai' + {'modelName': str(model_name or '').strip()}, IMAGE_GENERATION_CAPABILITY, provider, endpoint=endpoint )['supported'] @@ -204,14 +205,28 @@ def resolve_responses_image_api_version(settings): return RESPONSES_IMAGE_API_VERSION +def resolve_image_binding_api_version(binding): + route = resolve_image_binding_api(binding) + if route in (IMAGE_API_ROUTE_RESPONSES, IMAGE_API_ROUTE_MAI): + return 'v1' + if route == IMAGE_API_ROUTE_FLUX: + return 'preview' + context = resolve_image_endpoint_context(binding.endpoint) + if context['provider'] in ('openai', 'custom'): + return 'v1' + profile = get_connection_operation_settings(binding.endpoint, IMAGE_GENERATION_CAPABILITY) + return str(profile.get('api_version') or DEFAULT_IMAGES_API_VERSION).strip() + + def resolve_image_generation_api_version(settings): """Read image-specific transport options without changing the connection's chat version.""" - if resolve_image_api_route(settings) == IMAGE_API_ROUTE_RESPONSES: + route = resolve_image_api_route(settings) + if route in (IMAGE_API_ROUTE_RESPONSES, IMAGE_API_ROUTE_MAI): return RESPONSES_IMAGE_API_VERSION + if route == IMAGE_API_ROUTE_FLUX: + return 'preview' if image_settings_use_connections(settings): - binding = resolve_shared_image_binding(settings) - profile = get_connection_operation_settings(binding.endpoint, IMAGE_GENERATION_CAPABILITY) - return str(profile.get('api_version') or DEFAULT_IMAGES_API_VERSION).strip() + return resolve_image_binding_api_version(resolve_shared_image_binding(settings)) key = ( 'azure_apim_image_gen_api_version' if settings.get('enable_image_gen_apim') @@ -220,6 +235,32 @@ def resolve_image_generation_api_version(settings): return str(settings.get(key) or LEGACY_DEFAULT_IMAGES_API_VERSION).strip() +def resolve_selected_image_capability(settings): + """Project the selected operation without resolving credentials or creating clients.""" + if image_settings_use_connections(settings): + binding = resolve_shared_image_binding(settings) + result = resolve_image_model_capability(binding.model, binding.endpoint) + else: + apim = bool(settings.get('enable_image_gen_apim')) + prefix = 'azure_apim_image_gen' if apim else 'azure_openai_image_gen' + selected = (settings.get('image_gen_model') or {}).get('selected') or [] + model = dict(selected[0]) if selected and isinstance(selected[0], dict) and not apim else {} + model['deploymentName'] = resolve_selected_image_deployment_name(settings) + if not model['deploymentName']: + raise AIConnectionError('No image generation deployment is selected.', 'model_configuration_unavailable') + model.setdefault('supportsImageGeneration', True) + model.setdefault('image_generation_api', 'images') + endpoint = { + 'provider': 'aoai', + 'migration_source': 'legacy_image_apim' if apim else 'legacy_image_direct', + 'connection': {'endpoint': settings.get(f'{prefix}_endpoint')}, + } + result = resolve_image_model_capability(model, endpoint) + if not result['supported']: + raise AIConnectionError(result['reason'], 'model_capability_unavailable') + return result + + def build_image_api_base_url(endpoint, route, deployment='', api_version=''): """Normalize an API suffix, preserving the origin and every gateway/project prefix.""" try: @@ -232,7 +273,16 @@ def build_image_api_base_url(endpoint, route, deployment='', api_version=''): except ValueError as exc: raise AIConnectionError('The image connection endpoint is invalid.') from exc + if route not in IMAGE_APIS: + raise AIConnectionError('The image operation is unsupported.') path = parsed.path.rstrip('/') + hostname = parsed.hostname.lower() + foundry_resource = any( + hostname.endswith(f'.{suffix}') for suffix in ('services.ai.azure.com', 'services.ai.azure.us') + ) + if foundry_resource and re.fullmatch(r'/api/projects/[^/]+', path, flags=re.IGNORECASE): + # Resource image APIs are siblings of the documented project API, on the same origin. + path = '' # Only strip a recognizable API suffix. A gateway prefix such as /models/team # is part of the configured route, not a reason to redirect to the resource root. suffix = re.search( @@ -243,6 +293,14 @@ def build_image_api_base_url(endpoint, route, deployment='', api_version=''): ) if suffix: path = path[:suffix.start()] + if route == IMAGE_API_ROUTE_MAI: + path = re.sub(r'/mai/v1(?:/images/(?:generations|edits))?$', '', path, flags=re.IGNORECASE) + return urlunsplit((parsed.scheme, parsed.netloc, f'{path}/mai/v1/', '', '')) + if route == IMAGE_API_ROUTE_FLUX: + path = re.sub( + r'/providers/blackforestlabs/v1(?:/[-a-z0-9.]+)?$', '', path, flags=re.IGNORECASE + ) + return urlunsplit((parsed.scheme, parsed.netloc, f'{path}/providers/blackforestlabs/v1/', '', '')) if route == IMAGE_API_ROUTE_RESPONSES or api_version == 'v1': path = f'{path}/openai/v1/' else: @@ -252,32 +310,6 @@ def build_image_api_base_url(endpoint, route, deployment='', api_version=''): return urlunsplit((parsed.scheme, parsed.netloc, path, '', '')) -def resolve_responses_image_backend(binding): - """Use only an explicitly stored backend binding; otherwise leave routing to the provider.""" - profile = get_connection_operation_settings(binding.endpoint, IMAGE_GENERATION_CAPABILITY) - backend = profile.get('image_deployment') - if backend in (None, ''): - return '' - if not isinstance(backend, str) or not re.fullmatch(r'[A-Za-z0-9_.-]{1,128}', backend): - raise AIConnectionError('The stored image backend binding is invalid.') - if backend == resolve_image_binding_deployment(binding): - raise AIConnectionError('A GPT deployment cannot be used as its own image backend.') - for model in binding.endpoint.get('models') or []: - if not isinstance(model, dict): - continue - if (model.get('deploymentName') or model.get('deployment')) != backend: - continue - support = resolve_model_capability( - model, IMAGE_GENERATION_CAPABILITY, binding.endpoint.get('provider') - ) - if ( - not supports_model_capability(model, IMAGE_GENERATION_CAPABILITY, binding.endpoint.get('provider')) - or support.get('api') != IMAGE_API_ROUTE_IMAGES - ): - raise AIConnectionError('The stored image backend is unavailable or incompatible.') - return backend - - def build_image_generation_tool(size='', quality='', background=''): """Describe the hosted image_generation tool for a Responses request. diff --git a/application/single_app/functions_image_capabilities.py b/application/single_app/functions_image_capabilities.py new file mode 100644 index 000000000..97a1eee64 --- /dev/null +++ b/application/single_app/functions_image_capabilities.py @@ -0,0 +1,324 @@ +# functions_image_capabilities.py +"""Provider-qualified image operations, independent of application hosting and credentials.""" + +import re +from collections.abc import Mapping +from urllib.parse import urlsplit + +from functions_model_capabilities import get_image_operation_profile, get_model_catalog_capabilities + + +IMAGE_APIS = ("images", "responses", "mai", "flux") +IMAGE_PROVIDERS = ("aoai", "aifoundry", "new_foundry", "custom") +AZURE_IMAGE_POLICY_REASON = ( + "SimpleChat uses dedicated image models on Azure OpenAI and Foundry. " + "Choose a GPT Image, MAI Image, or supported Foundry FLUX deployment; " + "GPT chat models can use the image tool only through a compatible direct OpenAI Custom connection." +) +PROVIDER_LABELS = { + "openai": "OpenAI (direct)", + "azure_openai": "Azure OpenAI", + "foundry": "Microsoft Foundry", + "custom": "Custom", +} +CLOUD_LABELS = { + "commercial": "Commercial", + "government": "Azure Government", + "unknown": "Cloud not identified", +} +_CHAT_MODEL_PATTERN = re.compile(r"^(?:gpt[-_.]?\d|o\d(?:[-_.]|$))", re.IGNORECASE) + + +def _mapping(value): + return value if isinstance(value, Mapping) else {} + + +def _text(value): + return value.strip() if isinstance(value, str) else "" + + +def _flag(model, *names): + for name in names: + value = _mapping(model).get(name) + if isinstance(value, bool): + return value + return None + + +def image_model_name(model): + if isinstance(model, str): + return model.strip() + for name in ("modelName", "behavior_name", "deploymentName", "deployment", "name"): + value = _text(_mapping(model).get(name)) + if value: + return value + return "" + + +def _host_has_suffix(hostname, suffix): + return hostname == suffix or hostname.endswith(f".{suffix}") + + +def resolve_image_endpoint_context(endpoint=None, provider="aoai"): + """Identify service delivery, never using the app environment or token authority.""" + endpoint = _mapping(endpoint) + connection = _mapping(endpoint.get("connection")) + provider = _text(endpoint.get("provider") or provider).lower() + api_type = _text(endpoint.get("api_type")).lower().replace("-", "_") + try: + hostname = (urlsplit(_text(connection.get("endpoint"))).hostname or "").lower().rstrip(".") + except ValueError: + hostname = "" + azure_commercial = any(_host_has_suffix(hostname, suffix) for suffix in ( + "openai.azure.com", "services.ai.azure.com", "cognitiveservices.azure.com", + "api.cognitive.microsoft.com", + )) + azure_government = any(_host_has_suffix(hostname, suffix) for suffix in ( + "openai.azure.us", "services.ai.azure.us", "cognitiveservices.azure.us", + "api.cognitive.microsoft.us", + )) + cloud = "government" if azure_government else "commercial" if azure_commercial else "unknown" + explicit_cloud = _text(connection.get("image_cloud")).lower() + if cloud == "unknown" and explicit_cloud in ("commercial", "government"): + cloud = explicit_cloud + service = "" + if provider == "aoai": + service = "azure_openai" + elif provider in ("aifoundry", "new_foundry"): + service = "foundry" + elif provider == "custom": + if api_type == "azure_openai" or (api_type == "openai" and (azure_commercial or azure_government)): + service = "azure_openai" + elif api_type == "openai": + if hostname == "api.openai.com": + service, cloud = "openai", "commercial" + elif connection.get("image_provider") == "openai": + service = "openai" + else: + service = "custom" + if hostname == "api.openai.com" and not (provider == "custom" and api_type == "openai"): + service = "" + cloud = "commercial" + return { + "connection_provider": provider, + "provider": service, + "provider_label": PROVIDER_LABELS.get(service, "Unsupported image provider"), + "cloud": cloud, + "cloud_label": CLOUD_LABELS[cloud], + "api_type": api_type, + } + + +def _description(model, context, *, supported=False, source="unknown", reason="", api=""): + return { + "supported": supported, + "source": source, + "reason": reason, + "api": api, + "model_name": image_model_name(model), + "publisher": "", + "connection_provider": context["connection_provider"], + "provider": context["provider"], + "provider_label": context["provider_label"], + "cloud": context["cloud"], + "cloud_label": context["cloud_label"], + "availability": "unknown", + "availability_reason": "", + "lifecycle": "", + "editing": False, + "masking": False, + "mode": "unavailable", + "sizes": [], + "qualities": [], + "backgrounds": [], + "input_formats": [], + "output_formats": [], + "model_path": "", + "transport": "native", + "max_mask_bytes": 4 * 1024 * 1024, + "min_dimension": 0, + "max_pixels": 0, + } + + +def resolve_image_model_capability(model, endpoint=None, provider="aoai"): + """Intersect catalog facts, the implemented adapter, and explicit compatible metadata.""" + endpoint = _mapping(endpoint) + context = resolve_image_endpoint_context(endpoint, provider) + service = context["provider"] + result = _description(model, context) + if not service: + result.update(source="provider", reason="This connection type has no supported image adapter.") + return result + + declared = _flag(model, "supportsImageGeneration", "supports_image_generation") + if declared is False: + result.update(source="declared", reason="Image generation is not supported by this model.") + return result + catalog = get_model_catalog_capabilities(model) or {} + result["publisher"] = _text(catalog.get("publisher")) + profiles = _mapping(catalog.get("imageProfiles")) + profile_service = "azure_openai" if service == "foundry" and profiles.get("azure_openai") else service + lifecycle = _mapping(catalog.get("imageLifecycle")).get(profile_service, "") + result["lifecycle"] = lifecycle + if lifecycle == "retired": + result.update( + source="lifecycle", availability="unavailable", + reason="This image model is retired on the configured provider. Choose an available image model.", + ) + return result + + profile_id = profiles.get(profile_service) + profile = get_image_operation_profile(profile_id) if isinstance(profile_id, str) else None + route = _text(_mapping(model).get("image_generation_api")) + azure_delivery = service in ("azure_openai", "foundry") + chat_model = bool(_CHAT_MODEL_PATTERN.search(image_model_name(model))) or ( + catalog.get("generatesText") is True and catalog.get("generatesImages") is not True + ) + if azure_delivery and (chat_model or (route == "responses" and profile is None)): + result.update(source="policy", reason=AZURE_IMAGE_POLICY_REASON) + return result + if service == "azure_openai" and context["connection_provider"] == "custom" and context["api_type"] == "openai": + result.update( + source="provider", + reason="Use the Azure OpenAI API type for this Azure endpoint so its deployment identity and image route are preserved.", + ) + return result + if profile is None and profiles and service != "custom": + result.update(source="provider", reason="This image model has no documented image adapter on the configured provider.") + return result + + if profile is None and declared is True: + catalog_flag = {"images": "generatesImages", "responses": "imageGenerationTool"}.get(route) + if catalog_flag and catalog.get(catalog_flag) is False: + result.update( + source="catalog", + reason="The catalog does not support this model on the requested image API. A declaration cannot override a known incompatibility.", + ) + return result + legacy_images = ( + _text(endpoint.get("migration_source")).startswith("legacy_image_") + and route == "images" and not catalog + ) + custom_operation = context["connection_provider"] == "custom" and route in ("images", "responses") + if not (legacy_images or custom_operation): + result.update( + source="provider", + reason="Declare a compatible image API on a Custom connection, or select a cataloged dedicated image model.", + ) + return result + if azure_delivery and route != "images": + result.update(source="policy", reason=AZURE_IMAGE_POLICY_REASON) + return result + profile = { + "api": route, + "editing": _flag(model, "supportsImageEditing", "supports_image_editing") is True, + "masking": _flag(model, "supportsImageMasking", "supports_image_masking") is True, + "sizes": [], "qualities": [], "backgrounds": [], + "inputFormats": ["image/png", "image/jpeg"], + "outputFormats": ["image/png", "image/jpeg", "image/webp"], + "availability": {}, + } + if profile["masking"] and not profile["editing"]: + result.update(source="declared", reason="Masked image editing also requires declared image-editing support.") + return result + result["source"] = "legacy" if legacy_images else "declared" + elif profile is None: + result.update( + source="provider" if azure_delivery else "unknown", + reason=( + AZURE_IMAGE_POLICY_REASON if azure_delivery else + "Image generation has not been established for this model and endpoint. " + "An OpenAI-compatible API or a GPT model name alone is not sufficient." + ), + ) + return result + else: + result["source"] = "catalog" + + api = profile.get("api") + if api not in IMAGE_APIS or (azure_delivery and api == "responses"): + result.update(source="provider", reason="The model has no compatible image operation on this provider.") + return result + if api in ("mai", "flux") and service != "foundry": + result.update(source="provider", reason="This image model requires its Foundry provider API.") + return result + if not isinstance(profile.get("editing"), bool) or not isinstance(profile.get("masking"), bool): + result.update(source="catalog", reason="The image operation metadata is invalid.") + return result + if profile["masking"] and not profile["editing"]: + result.update(source="catalog", reason="The image operation has inconsistent editing metadata.") + return result + for name in ("sizes", "qualities", "backgrounds", "inputFormats", "outputFormats"): + value = profile.get(name, []) + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + result.update(source="catalog", reason="The image operation options are invalid.") + return result + for name in ("maxMaskBytes", "minDimension", "maxPixels"): + if name in profile and (type(profile[name]) is not int or profile[name] <= 0): + result.update(source="catalog", reason="The image operation limits are invalid.") + return result + model_path = profile.get("modelPath", "") + transport = profile.get("transport", "native") + if transport not in ("native", "openai_images") or ( + transport == "openai_images" and (api != "flux" or model_path != "flux-kontext-pro") + ): + result.update(source="catalog", reason="The image transport metadata is invalid.") + return result + if api == "flux" and model_path not in ("flux-2-pro", "flux-2-flex", "flux-kontext-pro", "flux-pro-1.1"): + result.update(source="catalog", reason="The Foundry FLUX operation path is invalid.") + return result + + availability_map = profile.get("availability") + if not isinstance(availability_map, Mapping) or any( + value not in ("documented", "unknown", "unavailable") for value in availability_map.values() + ): + result.update(source="catalog", reason="The image availability metadata is invalid.") + return result + availability = availability_map.get(context["cloud"], "unknown") + if availability == "unavailable": + result.update(availability=availability, source="provider", reason="This image model is unavailable on the endpoint's provider/cloud.") + return result + editing = profile["editing"] and _flag(model, "supportsImageEditing", "supports_image_editing") is not False + masking = profile["masking"] and editing and _flag(model, "supportsImageMasking", "supports_image_masking") is not False + result.update( + supported=True, api=api, editing=editing, masking=masking, + mode="masked" if masking else "edit" if editing else "regenerate", + availability=availability, + availability_reason=( + "Model availability in this endpoint cloud is not established by the reviewed provider documentation." + if availability == "unknown" else "" + ), + sizes=list(profile.get("sizes", [])), + qualities=list(profile.get("qualities", [])), + backgrounds=list(profile.get("backgrounds", [])), + input_formats=list(profile.get("inputFormats", [])), + output_formats=list(profile.get("outputFormats", [])), + model_path=model_path, + transport=transport, + max_mask_bytes=profile.get("maxMaskBytes", result["max_mask_bytes"]), + min_dimension=profile.get("minDimension", 0), + max_pixels=profile.get("maxPixels", 0), + ) + return result + + +def validate_image_options(capability, size="", quality="", background=""): + """Reject unsupported explicit options instead of quietly changing the requested image.""" + options = {"size": size, "quality": quality, "background": background} + for name, allowed_key in (("size", "sizes"), ("quality", "qualities"), ("background", "backgrounds")): + value = options[name] + if not isinstance(value, str): + raise ValueError(f"Image {name} must be text.") + if value and value not in capability.get(allowed_key, []): + raise ValueError(f"The selected image model does not support that {name}.") + if size: + match = re.fullmatch(r"(\d+)x(\d+)", size) + if not match: + raise ValueError("The image dimensions are invalid.") + width, height = (int(part) for part in match.groups()) + minimum = capability.get("min_dimension", 0) + maximum = capability.get("max_pixels", 0) + if width < minimum or height < minimum or (maximum and width * height > maximum): + raise ValueError("The image dimensions exceed the selected model's limits.") + return options diff --git a/application/single_app/functions_image_edit.py b/application/single_app/functions_image_edit.py index 4ddc6e84c..15baa4521 100644 --- a/application/single_app/functions_image_edit.py +++ b/application/single_app/functions_image_edit.py @@ -20,26 +20,23 @@ Two honest limitations are carried through to the interface rather than hidden: - A mask *guides* the model. It is not a pixel-level clamp, and areas outside it can still shift. -- Only ``gpt-image-*`` and legacy ``dall-e-2`` deployments expose ``/images/edits`` at all. - Everything else falls back to regenerating the whole image, which is a different operation and - is labelled as one. +- Provider-qualified capabilities distinguish masked edits, whole-image reference edits, + and prompt-only regeneration. Unsupported masks are rejected, not discarded. """ import base64 import io -import re from PIL import Image from functions_ai_connections import AIConnectionError from functions_image_api_route import ( - IMAGE_API_ROUTE_RESPONSES, ImageGenerationError, - resolve_image_api_route, resolve_image_generation_api_version, + resolve_selected_image_capability, resolve_selected_image_deployment_name, - resolve_selected_image_model_name, ) +from functions_image_capabilities import validate_image_options from functions_image_messages import decode_image_content, is_external_image_url # Config, the Azure clients and the generation helpers are imported inside the functions that @@ -63,8 +60,7 @@ # Formats the images API accepts as the image being edited. Anything else is converted to PNG. PASSTHROUGH_IMAGE_MIME_TYPES = ('image/png', 'image/jpeg', 'image/webp') -# Sizes the GPT image models emit. A value outside this set is rejected by the API, so the -# Controls tab is limited to these and anything else is simply not sent. +# Legacy GPT rendering labels; the selected operation profile determines accepted options. SUPPORTED_IMAGE_SIZES = ('1024x1024', '1024x1536', '1536x1024') SUPPORTED_IMAGE_QUALITIES = ('low', 'medium', 'high') SUPPORTED_IMAGE_BACKGROUNDS = ('transparent', 'opaque') @@ -73,30 +69,15 @@ # mask that has been resized can carry a few interpolated values at region edges. MASK_TRANSPARENT_THRESHOLD = 8 -# Errors naming a parameter the deployment does not know. The optional parameters below are -# newer than some deployments and than the pinned SDK, so one retry without them turns a hard -# failure into a slightly less capable success. -_UNSUPPORTED_PARAMETER_PATTERN = re.compile( - r'unsupported|unknown|unrecognized|not supported|invalid[_ ]?parameter|extra fields', - re.IGNORECASE, -) - - class ImageEditError(ImageGenerationError): """Safe image-operation failure, retaining the existing shared-editor exception contract.""" -# How an image may be changed once it exists. -# -# `masked` means the deployment exposes /images/edits, so a region can be selected and only that -# region asked to change. `regenerate` means it does not, and the only thing on offer is -# producing a new image from a revised prompt. +# Source edits, region edits, and prompt-only regeneration are distinct operations. IMAGE_EDIT_MODE_MASKED = 'masked' +IMAGE_EDIT_MODE_REFERENCE = 'edit' IMAGE_EDIT_MODE_REGENERATE = 'regenerate' - -# Models with an /images/edits endpoint. DALL-E 3 has none at all -- it can only generate -- so -# a deployment of it falls back to regeneration rather than failing at the point of use. -EDIT_CAPABLE_MODEL_MARKERS = ('gpt-image', 'dall-e-2', 'dalle-2') +IMAGE_EDIT_MODE_UNAVAILABLE = 'unavailable' # /images/edits is only routable on a recent preview API version. An older one fails in a way # that reads like a broken deployment, so it is detected up front and reported against the @@ -134,76 +115,41 @@ def resolve_image_edit_capability(settings): a cleared shared default into a bootstrap failure. """ settings = settings if isinstance(settings, dict) else {} - + unavailable = { + 'mode': IMAGE_EDIT_MODE_UNAVAILABLE, 'enabled': False, 'supported': False, + 'model_name': '', 'reason': '', 'api': '', 'editing': False, 'masking': False, + 'provider_label': '', 'cloud_label': '', 'availability': 'unknown', + 'availability_reason': '', 'sizes': [], 'qualities': [], 'backgrounds': [], + 'input_formats': [], 'output_formats': [], + } if not settings.get('enable_image_generation'): - return { - 'mode': IMAGE_EDIT_MODE_REGENERATE, - 'enabled': False, - 'model_name': '', - 'reason': '', - } - + return {**unavailable, 'reason': 'Image generation is not enabled.'} try: - model_name = resolve_selected_image_model_name(settings) - image_api_route = resolve_image_api_route(settings) + capability = resolve_selected_image_capability(settings) api_version = resolve_image_generation_api_version(settings) except AIConnectionError as exc: - return { - 'mode': IMAGE_EDIT_MODE_REGENERATE, - 'enabled': False, - 'model_name': '', - 'reason': exc.public_message, - } - normalized_model = model_name.lower() - - if not normalized_model: - return { - 'mode': IMAGE_EDIT_MODE_REGENERATE, - 'enabled': True, - 'model_name': model_name, - 'reason': ( - 'The image deployment does not report which model it runs, so changing part of ' - 'an image cannot be offered. Re-selecting the deployment in Admin Settings ' - 'records the model name.' - ), - } - - if not any(marker in normalized_model for marker in EDIT_CAPABLE_MODEL_MARKERS): - if image_api_route == IMAGE_API_ROUTE_RESPONSES: - reason = ( - f'{model_name} produces images through the Responses image tool, which has ' - 'no way to change part of one, so a change replaces the whole image.' - ) - else: - reason = ( - f'{model_name} can generate images but cannot edit them, so a change replaces ' - 'the whole image.' - ) - return { - 'mode': IMAGE_EDIT_MODE_REGENERATE, - 'enabled': True, - 'model_name': model_name, - 'reason': reason, - } - - if not image_api_version_supports_edit(api_version): - return { - 'mode': IMAGE_EDIT_MODE_REGENERATE, - 'enabled': True, - 'model_name': model_name, - 'reason': ( - f'Changing part of an image needs image generation API version ' - f'{MIN_IMAGE_EDIT_API_VERSION} or newer. It is currently set to ' - f'{api_version or "nothing"}.' - ), - } - - return { - 'mode': IMAGE_EDIT_MODE_MASKED, - 'enabled': True, - 'model_name': model_name, - 'reason': '', - } + return {**unavailable, 'reason': exc.public_message} + capability['enabled'] = True + if ( + capability['api'] == 'images' + and capability['provider'] in ('azure_openai', 'foundry') + and capability['editing'] + and not image_api_version_supports_edit(api_version) + ): + capability.update( + mode=IMAGE_EDIT_MODE_REGENERATE, editing=False, masking=False, + reason=f'Source-image editing requires Images API version {MIN_IMAGE_EDIT_API_VERSION} or newer.', + ) + elif capability['editing'] and not capability['masking']: + capability['reason'] = ( + 'This model supports whole-image reference edits, but an uploaded-mask operation ' + 'is not documented for this integration.' + ) + elif not capability['editing']: + capability['reason'] = ( + 'This model is configured for generation only. Regeneration creates a new whole image from the prompt.' + ) + return capability @@ -238,7 +184,7 @@ def _open_image(image_bytes, what): return image -def normalize_mask(mask_data_url, width, height, regions=0): +def normalize_mask(mask_data_url, width, height, regions=0, max_bytes=MAX_MASK_BYTES): """Return a mask PNG matching the source image, or None when nothing was selected. The returned bytes are RGBA PNG where transparent means "edit here", which is what the API @@ -259,6 +205,9 @@ def normalize_mask(mask_data_url, width, height, regions=0): normalized = str(mask_data_url).strip() if not normalized: return None + limit = min(MAX_MASK_BYTES, max_bytes) + if len(normalized) > ((limit + 2) // 3) * 4 + 128: + raise ImageEditError('The selected region is too large to send', 'invalid_image_mask', 400) if normalized.startswith('data:image/'): try: @@ -271,10 +220,15 @@ def normalize_mask(mask_data_url, width, height, regions=0): except Exception as exc: raise ImageEditError('The selected region could not be read') from exc - if len(mask_bytes) > MAX_MASK_BYTES: + if len(mask_bytes) >= limit: raise ImageEditError('The selected region is too large to send') - mask = _open_image(mask_bytes, 'selected region').convert('RGBA') + original_mask = _open_image(mask_bytes, 'selected region') + if original_mask.format != 'PNG' or ( + 'A' not in original_mask.getbands() and 'transparency' not in original_mask.info + ): + raise ImageEditError('The selected region must be a PNG mask with transparency.', 'invalid_image_mask', 400) + mask = original_mask.convert('RGBA') if mask.size != (int(width), int(height)): mask = mask.resize((int(width), int(height)), Image.NEAREST) @@ -291,7 +245,7 @@ def normalize_mask(mask_data_url, width, height, regions=0): buffer = io.BytesIO() mask.save(buffer, format='PNG') encoded = buffer.getvalue() - if len(encoded) > MAX_MASK_BYTES: + if len(encoded) >= limit: raise ImageEditError('The selected region is too large to send') bounded_regions = regions if isinstance(regions, int) and not isinstance(regions, bool) else 0 @@ -305,7 +259,7 @@ def normalize_mask(mask_data_url, width, height, regions=0): } -def prepare_source_image(mime_type, image_bytes): +def prepare_source_image(mime_type, image_bytes, allowed_formats=None): """Return the image to edit as bytes the API accepts, with its true dimensions. The dimensions matter more than the format: the mask has to match them exactly, and the only @@ -318,8 +272,9 @@ def prepare_source_image(mime_type, image_bytes): image = _open_image(image_bytes, 'image') width, height = image.size - normalized_mime = str(mime_type or '').strip().lower() - if normalized_mime in PASSTHROUGH_IMAGE_MIME_TYPES: + normalized_mime = Image.MIME.get(image.format) or str(mime_type or '').strip().lower() + allowed_formats = PASSTHROUGH_IMAGE_MIME_TYPES if allowed_formats is None else allowed_formats + if normalized_mime in allowed_formats: return { 'bytes': image_bytes, 'mime_type': normalized_mime, @@ -328,8 +283,12 @@ def prepare_source_image(mime_type, image_bytes): 'height': height, } + if 'image/png' not in allowed_formats: + raise ImageEditError('The source image format is unsupported by this model.', 'invalid_image_request', 400) buffer = io.BytesIO() image.convert('RGBA').save(buffer, format='PNG') + if len(buffer.getvalue()) > MAX_SOURCE_IMAGE_BYTES: + raise ImageEditError('This image is too large to edit after format conversion.', 'invalid_image_request', 400) return { 'bytes': buffer.getvalue(), 'mime_type': 'image/png', @@ -408,6 +367,7 @@ def revise_image_message( expected_revision_count=None, expected_current_revision_id='', reload_message=None, + operation='', ): """Produce a new version of an image message and store it, mutating ``message_doc``. @@ -444,25 +404,47 @@ def revise_image_message( # nothing pointing at them. assert_revision_expectations(message_doc, expected_revision_count, expected_current_revision_id) - source_mime, source_bytes = load_current_image_bytes(message_doc, complete_content) - source_image = prepare_source_image(source_mime, source_bytes) - capability = resolve_image_edit_capability(settings) + if not capability['enabled']: + raise AIConnectionError( + capability.get('reason') or 'The image model is unavailable.', + 'model_configuration_unavailable' if settings.get('enable_image_generation') else 'capability_disabled', + ) + if mask_data_url is not None and not isinstance(mask_data_url, str): + raise ImageEditError('The selected region must be an encoded PNG mask.', 'invalid_image_mask', 400) + if mask_regions and not mask_data_url: + raise ImageEditError('The selected region has no PNG mask. Clear or redraw the selection.', 'invalid_image_mask', 400) + if operation not in ('', 'edit', 'regenerate'): + raise ImageEditError('The image operation is invalid.', 'invalid_image_request', 400) + effective_operation = operation or ( + 'edit' if revision_origin == ORIGIN_AI and capability['editing'] else 'regenerate' + ) + if effective_operation == 'edit' and not capability['editing']: + raise ImageEditError('The selected model cannot edit a source image.', 'unsupported_image_operation', 400) + if mask_data_url and (not capability['masking'] or effective_operation != 'edit' or revision_origin != ORIGIN_AI): + raise ImageEditError( + 'The selected model or operation cannot use this mask. Refresh the image controls or remove the selection.', + 'unsupported_image_operation', 400, + ) + source_image = None + if effective_operation == 'edit': + source_mime, source_bytes = load_current_image_bytes(message_doc, complete_content) + source_image = prepare_source_image(source_mime, source_bytes, capability['input_formats']) mask = None if revision_origin == ORIGIN_AI: normalized_instruction = str(instruction or '').strip() if not normalized_instruction: raise ImageEditError('Describe the change you want') - # A region is only meaningful when the deployment can act on one. Sending a mask to a - # model without an edit endpoint would silently discard it, so it is dropped here and - # the interface says up front that the whole image will be replaced. - if capability['mode'] == IMAGE_EDIT_MODE_MASKED: + if mask_data_url: mask = normalize_mask( mask_data_url, source_image['width'], source_image['height'], regions=mask_regions, + max_bytes=capability.get('max_mask_bytes', MAX_MASK_BYTES), ) + if mask_regions and mask is None: + raise ImageEditError('The selected region contains no editable pixels.', 'invalid_image_mask', 400) effective_prompt = compose_edit_prompt(current_prompt, normalized_instruction, mask) elif revision_origin == ORIGIN_PROMPT: normalized_instruction = '' @@ -483,6 +465,7 @@ def revise_image_message( size=size, quality=quality, background=background, + operation=effective_operation, ) stored = store_revision_image( @@ -629,52 +612,6 @@ def compose_edit_prompt(current_prompt, instruction, mask=None): return '\n\n'.join(parts) -def _optional_parameters(quality='', background='', input_fidelity=''): - """Return the newer, optional API parameters as a body fragment. - - Sent through ``extra_body`` rather than as keyword arguments deliberately. ``quality``, - ``background`` and ``input_fidelity`` do not exist in every version of the SDK. - ``extra_body`` merges them into the same request body across the supported SDK versions - without requiring version-specific keyword arguments. - """ - optional = {} - if quality in SUPPORTED_IMAGE_QUALITIES: - optional['quality'] = quality - if background in SUPPORTED_IMAGE_BACKGROUNDS: - optional['background'] = background - if input_fidelity in ('high', 'low'): - optional['input_fidelity'] = input_fidelity - return optional - - -def _call_with_optional_parameters(operation, optional): - """Run an images call, retrying once without the optional parameters if they are refused. - - A deployment that predates ``input_fidelity`` rejects the whole request rather than ignoring - the field it does not know. Retrying without them produces a slightly less controlled image, - which is a much better outcome than refusing to edit at all on an older deployment. - """ - try: - return operation(optional) - except Exception as exc: - message = str(exc).lower() - if ( - not optional or getattr(exc, 'status_code', None) != 400 - or not _UNSUPPORTED_PARAMETER_PATTERN.search(message) - or not any(parameter in message for parameter in optional) - ): - raise - # Deferred with the rest of the runtime dependencies; pure mask helpers stay import-safe. - from functions_appinsights import log_event - from functions_image_generation import image_generation_error_log_context - - log_event( - '[IMAGE_EDIT] Retrying without optional image parameters', - extra={'parameters': sorted(optional), **image_generation_error_log_context(exc)}, - ) - return operation({}) - - def request_image_edit( settings, source_image, @@ -683,22 +620,16 @@ def request_image_edit( size='', quality='', background='', + operation='', ): """Ask the model for an edited image, returning its bytes and how it was produced. - Falls back to generating a new image when the deployment has no edit endpoint. The fallback - is reported in the result as ``method`` rather than being hidden, because "part of this - image changed" and "this is a different image" are different outcomes and the history has to - be able to say which happened. + An explicit edit must remain an edit. Regeneration is requested separately, or + selected for a generation-only model when a legacy caller omitted the operation. """ - from functions_appinsights import log_event from functions_image_generation import ( - close_image_generation_client, - extract_generated_image_source, - image_generation_error_log_context, - normalize_image_generation_error, + request_edited_image_source, request_generated_image_source, - resolve_image_generation_client, ) capability = resolve_image_edit_capability(settings) @@ -707,84 +638,39 @@ def request_image_edit( capability.get('reason') or 'Image generation is not enabled.', 'model_configuration_unavailable' if settings.get('enable_image_generation') else 'capability_disabled', ) - normalized_size = size if size in SUPPORTED_IMAGE_SIZES else '' - normalized_quality = quality if quality in SUPPORTED_IMAGE_QUALITIES else '' - normalized_background = background if background in SUPPORTED_IMAGE_BACKGROUNDS else '' - - if capability['mode'] != IMAGE_EDIT_MODE_MASKED: - # Every whole-image regeneration uses the same binding, request, and validation - # as chat and approvals. It must never try a second paid route after a failure. - deployment = resolve_selected_image_deployment_name(settings) - try: + if operation not in ('', 'edit', 'regenerate'): + raise ImageEditError('The image operation is invalid.', 'invalid_image_request', 400) + effective_operation = operation or ('edit' if capability['editing'] else 'regenerate') + if mask and (not capability['masking'] or effective_operation != 'edit'): + raise ImageEditError('The selected model or operation cannot use an uploaded mask.', 'unsupported_image_operation', 400) + if effective_operation == 'edit' and not capability['editing']: + raise ImageEditError('The selected model cannot edit a source image.', 'unsupported_image_operation', 400) + try: + validate_image_options(capability, size, quality, background) + except ValueError as exc: + raise ImageEditError( + 'The selected model does not support the requested image options.', 'invalid_image_request', 400 + ) from exc + deployment = resolve_selected_image_deployment_name(settings) + try: + if effective_operation == 'regenerate': generated_source = request_generated_image_source( - settings, - prompt, - size=normalized_size, - quality=normalized_quality, - background=normalized_background, + settings, prompt, size=size, quality=quality, background=background, ) - except ImageGenerationError as exc: - raise ImageEditError(exc.public_message, exc.code, exc.status_code, exc.context) from exc - - return _finish_image_edit( - generated_source, - deployment=deployment, - method='regenerate', - normalized_size=normalized_size, - quality=normalized_quality, - background=normalized_background, - ) - - client, deployment = resolve_image_generation_client(settings) - - optional = _optional_parameters( - quality=normalized_quality, - background=normalized_background, - # Only meaningful alongside a real edit, and only for the GPT image models. - input_fidelity='high' if capability['mode'] == IMAGE_EDIT_MODE_MASKED else '', - ) - - def edit(extra): - arguments = { - 'model': deployment, - 'prompt': prompt, - 'n': 1, - 'image': ( - source_image['file_name'], - source_image['bytes'], - source_image['mime_type'], - ), - } - if mask: - arguments['mask'] = ('mask.png', mask['bytes'], 'image/png') - if normalized_size: - arguments['size'] = normalized_size - if extra: - arguments['extra_body'] = extra - return client.images.edit(**arguments) - - try: - response = _call_with_optional_parameters(edit, optional) - generated_source = extract_generated_image_source(response) - except Exception as exc: - log_event( - '[IMAGE_EDIT] Edit request failed', - extra={'deployment': deployment, **image_generation_error_log_context(exc)}, - ) - error = normalize_image_generation_error(exc) - if isinstance(error, AIConnectionError): - raise - raise ImageEditError(error.public_message, error.code, error.status_code, error.context) from exc - finally: - close_image_generation_client(client) + else: + generated_source = request_edited_image_source( + settings, prompt, source_image, mask=mask, size=size, quality=quality, background=background, + ) + except ImageGenerationError as exc: + raise ImageEditError(exc.public_message, exc.code, exc.status_code, exc.context) from exc return _finish_image_edit( generated_source, deployment=deployment, - method='edit', - normalized_size=normalized_size, - quality=normalized_quality, - background=normalized_background, + method=effective_operation, + normalized_size=size, + quality=quality, + background=background, ) diff --git a/application/single_app/functions_image_generation.py b/application/single_app/functions_image_generation.py index 43d7bbb46..ee6c81890 100644 --- a/application/single_app/functions_image_generation.py +++ b/application/single_app/functions_image_generation.py @@ -1,6 +1,7 @@ # functions_image_generation.py """Shared helpers for opt-in chat image generation proposals.""" +import base64 import mimetypes import random import re @@ -10,6 +11,7 @@ from urllib.parse import urlparse import requests +import httpx from azure.identity import get_bearer_token_provider from openai import OpenAI @@ -22,19 +24,23 @@ register_capability_client_factory, ) from functions_appinsights import log_event +from functions_image_adapters import edit_image, generate_image +from functions_image_capabilities import resolve_image_endpoint_context, resolve_image_model_capability, validate_image_options from functions_image_api_route import ( DEFAULT_IMAGES_API_VERSION, IMAGE_API_ROUTE_RESPONSES, + IMAGE_API_ROUTE_MAI, + IMAGE_API_ROUTE_FLUX, ImageGenerationError, _as_response_dict, build_image_api_base_url, - build_image_generation_tool, extract_responses_image_source, resolve_image_api_route, resolve_image_binding_api, + resolve_image_binding_api_version, resolve_image_binding_deployment, resolve_image_generation_api_version, - resolve_responses_image_backend, + resolve_selected_image_capability, resolve_selected_image_deployment_name, resolve_shared_image_binding, ) @@ -273,24 +279,63 @@ def image_generation_error_response(exc): }, status -def _build_image_runtime_client(endpoint, settings, deployment, route, api_version, backend=''): +def _build_custom_image_runtime_client(endpoint, settings, deployment, route, api_version): + """Reuse Custom URL, credential and pinned-transport rules without chat inference.""" + # Runtime construction imports application configuration; keep leaf projections independent. + from functions_model_endpoint_runtime import build_custom_openai_client_kwargs + + api_type = endpoint.get('api_type') + if api_type not in ('openai', 'azure_openai') or route not in ('images', 'responses'): + raise AIConnectionError('This Custom API type has no compatible image adapter.', 'unsupported_capability') + if api_type == 'azure_openai' and route != 'images': + raise AIConnectionError('Azure image generation requires a dedicated image model.', 'unsupported_capability') + runtime_endpoint = deepcopy(endpoint) + if api_type == 'azure_openai': + runtime_endpoint['connection']['api_version'] = api_version + kwargs = build_custom_openai_client_kwargs( + runtime_endpoint, settings, request_model=deployment, + default_headers=build_model_endpoint_identity_headers(settings, endpoint_config=endpoint), + ) + if api_type == 'azure_openai' and api_version == 'v1': + if runtime_endpoint['connection'].get('url_mode') != 'exact': + kwargs['base_url'] = build_image_api_base_url( + runtime_endpoint['connection']['endpoint'], 'images', deployment=deployment, api_version='v1', + ) + kwargs['default_query'] = {} + try: + return _ImageOpenAIClient(image_auth_header='authorization', max_retries=0, **kwargs) + except (TypeError, ValueError): + kwargs['http_client'].close() + raise + + +def _build_image_runtime_client(endpoint, settings, deployment, route, api_version, image_capability=None): connection = endpoint.get('connection') or {} auth = endpoint.get('auth') or {} profile = get_connection_operation_settings(endpoint, IMAGE_GENERATION_CAPABILITY) provider = str(endpoint.get('provider') or 'aoai').strip().lower() + if provider == 'custom': + return _build_custom_image_runtime_client(endpoint, settings, deployment, route, api_version) if provider not in ('aoai', 'aifoundry', 'new_foundry'): raise AIConnectionError('This connection type has no compatible image adapter.', 'unsupported_capability') - if not isinstance(api_version, str) or not re.fullmatch(r'(?:v1|\d{4}-\d{2}-\d{2}(?:-preview)?)', api_version): + native_preview = route == IMAGE_API_ROUTE_FLUX and api_version == 'preview' + if not native_preview and ( + not isinstance(api_version, str) or not re.fullmatch(r'(?:v1|\d{4}-\d{2}-\d{2}(?:-preview)?)', api_version) + ): raise AIConnectionError('The image connection API version is invalid.') + compatible_images = (image_capability or {}).get('transport') == 'openai_images' base_url = build_image_api_base_url( - connection.get('endpoint'), route, deployment=deployment, api_version=api_version + connection.get('endpoint'), 'images' if compatible_images else route, + deployment=deployment, api_version='v1' if compatible_images else api_version, ) auth_type = str(auth.get('type') or 'managed_identity').strip().lower() - auth_header = str(profile.get('auth_header') or ( - 'api-key' if api_version != 'v1' or provider == 'aoai' or profile.get('is_apim') + default_header = ( + 'api-key' if compatible_images else 'authorization' if route == IMAGE_API_ROUTE_FLUX + else 'api-key' if route == IMAGE_API_ROUTE_MAI or api_version != 'v1' or provider == 'aoai' or profile.get('is_apim') else 'authorization' - )).lower() + ) + auth_header = str(profile.get('auth_header') or default_header).lower() if auth_header not in ('api-key', 'authorization', 'ocp-apim-subscription-key'): raise AIConnectionError('The image connection authentication header is unsupported.') token_provider = None @@ -309,16 +354,20 @@ def _build_image_runtime_client(endpoint, settings, deployment, route, api_versi ) credential = resolve_credential_for_model_endpoint_auth(auth) - scope = cognitive_services_scope - if provider in ('aifoundry', 'new_foundry'): + context = resolve_image_endpoint_context(endpoint) + scope = auth.get('foundry_scope') or '' + if not scope and context['cloud'] in ('commercial', 'government'): + domain = 'azure.us' if context['cloud'] == 'government' else 'azure.com' + scope = f'https://cognitiveservices.{domain}/.default' + elif not scope and provider in ('aifoundry', 'new_foundry'): scope = resolve_foundry_scope_for_endpoint_auth(auth, endpoint=connection.get('endpoint')) + elif not scope: + scope = cognitive_services_scope token_provider = get_bearer_token_provider(credential, scope) else: raise AIConnectionError('The image connection authentication type is unsupported.') headers = build_model_endpoint_identity_headers(settings, endpoint_config=endpoint) - if backend: - headers['x-ms-oai-image-generation-deployment'] = backend return _ImageOpenAIClient( api_key=api_key, base_url=base_url, @@ -338,12 +387,7 @@ def build_image_connection_client(binding, settings): if binding.selection.get('provider') not in (None, '', provider): raise AIConnectionError('The selected image model does not belong to that provider.') deployment = resolve_image_binding_deployment(binding) - profile = get_connection_operation_settings(binding.endpoint, IMAGE_GENERATION_CAPABILITY) - api_version = ( - 'v1' if route == IMAGE_API_ROUTE_RESPONSES - else str(profile.get('api_version') or DEFAULT_IMAGES_API_VERSION).strip() - ) - backend = resolve_responses_image_backend(binding) if route == IMAGE_API_ROUTE_RESPONSES else '' + api_version = resolve_image_binding_api_version(binding) # The model-endpoint helper enforces endpoint secret scope; a settings-secret # reference or browser-redacted credential must never be treated as the key. from functions_keyvault import SecretReturnType, keyvault_model_endpoint_get_helper @@ -351,7 +395,10 @@ def build_image_connection_client(binding, settings): endpoint = keyvault_model_endpoint_get_helper( deepcopy(binding.endpoint), binding.endpoint['id'], scope='global', return_type=SecretReturnType.VALUE ) - return _build_image_runtime_client(endpoint, settings, deployment, route, api_version, backend) + image_capability = resolve_image_model_capability(binding.model, binding.endpoint) + return _build_image_runtime_client( + endpoint, settings, deployment, route, api_version, image_capability=image_capability + ) except AIConnectionError: raise except Exception as exc: @@ -426,6 +473,18 @@ def request_generated_image_source(settings, prompt, size='', quality='', backgr test that exercised a different route from the real call would certify a path nobody uses. """ + return _request_image_source(settings, prompt, size=size, quality=quality, background=background) + + +def request_edited_image_source(settings, prompt, source_image, mask=None, size='', quality='', background=''): + """Use the same binding, errors and output validation for a real source-image edit.""" + return _request_image_source( + settings, prompt, source_image=source_image, mask=mask, + size=size, quality=quality, background=background, operation='edit', + ) + + +def _request_image_source(settings, prompt, *, source_image=None, mask=None, size='', quality='', background='', operation='generate'): client = None response = None route = '' @@ -434,30 +493,45 @@ def request_generated_image_source(settings, prompt, size='', quality='', backgr raise PermissionError('Image generation is not enabled') if not isinstance(prompt, str) or not prompt.strip(): raise ValueError('Image generation prompt is required') - route = resolve_image_api_route(settings) + if operation not in ('generate', 'edit'): + raise ValueError('The image operation is invalid') + capability = resolve_selected_image_capability(settings) + if operation == 'edit': + # Editing imports generation helpers lazily; keep the shared projection cycle-free. + from functions_image_edit import resolve_image_edit_capability + + capability = resolve_image_edit_capability(settings) + if not capability.get('enabled') or not capability.get('editing'): + raise AIConnectionError( + capability.get('reason') or 'The selected model cannot edit a source image.', + 'unsupported_image_operation', + ) + route = capability['api'] + validate_image_options(capability, size, quality, background) + if mask and not capability.get('masking'): + raise ImageGenerationError('The selected image operation does not support masks.', 'unsupported_image_operation', 400) client, deployment = resolve_image_generation_client(settings) - if route != IMAGE_API_ROUTE_RESPONSES: - arguments = {'model': deployment, 'prompt': prompt, 'n': 1} - if size: - arguments['size'] = size - optional = {key: value for key, value in {'quality': quality, 'background': background}.items() if value} - if optional: - arguments['extra_body'] = optional - response = client.images.generate(**arguments) - return extract_generated_image_source(response) - - response = client.responses.create( - model=deployment, - input=prompt, - tools=[build_image_generation_tool(size=size, quality=quality, background=background)], - tool_choice={'type': 'image_generation'}, + if operation == 'edit': + response = edit_image( + client, deployment, capability, prompt, source_image, mask, + size=size, quality=quality, background=background, + ) + else: + response = generate_image( + client, deployment, capability, prompt, size=size, quality=quality, background=background + ) + generated_image_url = ( + extract_responses_image_source(response) if route == IMAGE_API_ROUTE_RESPONSES + else extract_generated_image_source(response) ) - generated_image_url = extract_responses_image_source(response) if not generated_image_url: raise ImageGenerationError( 'The selected model returned no image. Ask an administrator to check image service availability for this connection.', 'image_output_missing', ) + if generated_image_url.startswith('data:image/') or capability.get('connection_provider') == 'custom': + mime_type, image_bytes = resolve_generated_image_bytes(generated_image_url, settings=settings) + generated_image_url = f'data:{mime_type};base64,{base64.b64encode(image_bytes).decode("ascii")}' return generated_image_url except Exception as exc: error = normalize_image_generation_error(exc) @@ -484,6 +558,10 @@ def request_generated_image_source(settings, prompt, size='', quality='', backgr def extract_generated_image_source(image_response): """Extract a usable image URL or data URL from an Azure OpenAI image response.""" response_dict = _as_response_dict(image_response) + if response_dict.get('error') or response_dict.get('status') in ( + 'failed', 'incomplete', 'in_progress', 'queued', 'cancelled', + ): + extract_responses_image_source(response_dict) if not isinstance(response_dict.get('data'), list) or not response_dict['data']: raise ImageGenerationError('The image service returned no image data.', 'image_output_missing') @@ -508,25 +586,87 @@ def extract_generated_image_source(image_response): raise ImageGenerationError('The image service returned no usable image.', 'image_output_missing') -def resolve_generated_image_bytes(generated_image_url): +def _validated_image_bytes(image_bytes): + """Validate actual output pixels instead of trusting a provider's MIME label or base64.""" + # The editor's bounded decoder is shared with generation without importing app configuration. + from PIL import Image + from functions_image_edit import MAX_SOURCE_IMAGE_BYTES, ImageEditError, _open_image + + if not image_bytes or len(image_bytes) > MAX_SOURCE_IMAGE_BYTES: + raise ImageGenerationError('The image output is empty or exceeds the application size limit.', 'image_output_invalid') + try: + with _open_image(image_bytes, 'generated image') as image: + mime_type = Image.MIME.get(image.format) + except ImageEditError as exc: + raise ImageGenerationError('The provider returned unreadable image output.', 'image_output_invalid') from exc + if mime_type not in ('image/png', 'image/jpeg', 'image/webp'): + raise ImageGenerationError('The provider returned an unsupported image format.', 'image_output_invalid') + return mime_type, image_bytes + + +def _download_custom_image_bytes(image_url, settings): + """Fetch provider output without forwarding inference credentials or client certificates.""" + # Reuse the Custom egress boundary for derived URLs as well as model requests. + from functions_image_edit import MAX_SOURCE_IMAGE_BYTES + from functions_model_endpoint_validation import ModelEndpointValidationError, custom_endpoint_setting_enabled + from model_endpoint_clients import build_custom_openai_sync_http_client + + try: + with build_custom_openai_sync_http_client( + allow_private=custom_endpoint_setting_enabled(settings, 'allow_private_custom_model_endpoints'), + allow_insecure=custom_endpoint_setting_enabled(settings, 'allow_insecure_custom_model_endpoints'), + ca_bundle_path=str(settings.get('custom_model_endpoint_ca_bundle_path') or '').strip(), + client_cert=None, + ) as client: + with client.stream('GET', image_url, timeout=30) as response: + response.raise_for_status() + image_bytes = bytearray() + for chunk in response.iter_bytes(chunk_size=65536): + image_bytes.extend(chunk) + if len(image_bytes) > MAX_SOURCE_IMAGE_BYTES: + raise ImageGenerationError( + 'The image output exceeds the application size limit.', 'image_output_invalid' + ) + return bytes(image_bytes) + except (httpx.HTTPError, ModelEndpointValidationError) as exc: + raise ImageGenerationError( + 'The generated image could not be downloaded through the configured network policy.', + 'image_download_failed', context=image_generation_error_log_context(exc), + ) from exc + + +def resolve_generated_image_bytes(generated_image_url, settings=None): """Resolve generated image output into bytes and a MIME type for blob storage.""" normalized_image_url = str(generated_image_url or '').strip() if not normalized_image_url: raise ImageGenerationError('The image service returned no usable image.', 'image_output_missing') if normalized_image_url.startswith('data:image/'): + from functions_image_edit import MAX_SOURCE_IMAGE_BYTES + + if len(normalized_image_url) > ((MAX_SOURCE_IMAGE_BYTES + 2) // 3) * 4 + 128: + raise ImageGenerationError('The image output exceeds the application size limit.', 'image_output_invalid') try: - return decode_image_content(normalized_image_url) + _mime_type, image_bytes = decode_image_content(normalized_image_url) except ValueError as exc: raise ImageGenerationError('The image service returned unreadable image data.', 'image_output_missing') from exc + return _validated_image_bytes(image_bytes) try: parsed_url = urlparse(normalized_image_url) - if parsed_url.scheme not in {'http', 'https'} or not parsed_url.hostname: + if ( + parsed_url.scheme not in {'http', 'https'} or not parsed_url.hostname + or parsed_url.username or parsed_url.password + ): raise ValueError('Unsupported image URL') except ValueError as exc: raise ImageGenerationError('The image service returned an unsupported image source.', 'image_output_missing') from exc + if settings is not None and image_settings_use_connections(settings): + binding = resolve_shared_image_binding(settings) + if binding.endpoint.get('provider') == 'custom': + return _validated_image_bytes(_download_custom_image_bytes(normalized_image_url, settings)) + try: response = requests.get(normalized_image_url, timeout=30) response.raise_for_status() @@ -539,11 +679,7 @@ def resolve_generated_image_bytes(generated_image_url): if not image_bytes: raise ImageGenerationError('The generated image download was empty.', 'image_output_missing') - content_type = str(response.headers.get('Content-Type') or '').split(';', 1)[0].strip() - if not content_type or not content_type.startswith('image/'): - content_type = mimetypes.guess_type(parsed_url.path)[0] or 'image/png' - - return content_type, image_bytes + return _validated_image_bytes(image_bytes) def _image_extension_for_mime_type(mime_type): diff --git a/application/single_app/functions_keyvault.py b/application/single_app/functions_keyvault.py index f692ba730..63cfe1e60 100644 --- a/application/single_app/functions_keyvault.py +++ b/application/single_app/functions_keyvault.py @@ -59,8 +59,9 @@ SQL_PLUGIN_SENSITIVE_ADDITIONAL_FIELDS = {"connection_string", "password"} SQL_PLUGIN_SENSITIVE_AUTH_FIELDS = {"client_secret"} MODEL_ENDPOINT_SENSITIVE_AUTH_FIELDS = { - "api_key": {"api_key"}, - "client_secret": {"service_principal"}, + "api_key": {"api_key", "key"}, + "client_secret": {"service_principal", "oauth2_client_credentials"}, + "bearer_token": {"bearer"}, } AGENT_SENSITIVE_SECRET_FIELDS = [ { diff --git a/application/single_app/functions_model_capabilities.py b/application/single_app/functions_model_capabilities.py index 3527ec3f4..d878b5b5f 100644 --- a/application/single_app/functions_model_capabilities.py +++ b/application/single_app/functions_model_capabilities.py @@ -29,6 +29,7 @@ deployment. Reported as inferred rather than known, so a caller can say so. """ +import copy import json import os import re @@ -64,6 +65,7 @@ _CATALOG_LOCK = threading.Lock() _CATALOG_CACHE = None +_IMAGE_OPERATION_PROFILES = {} def _normalize_model_identifier(value): @@ -83,7 +85,7 @@ def load_model_capability_catalog(force_refresh=False): A missing or malformed catalog yields an empty mapping rather than raising. Vision keeps its legacy heuristic; reasoning support remains unknown. """ - global _CATALOG_CACHE + global _CATALOG_CACHE, _IMAGE_OPERATION_PROFILES if _CATALOG_CACHE is not None and not force_refresh: return _CATALOG_CACHE @@ -93,11 +95,15 @@ def load_model_capability_catalog(force_refresh=False): return _CATALOG_CACHE catalog = {} + image_profiles = {} path = os.path.join(os.path.dirname(os.path.abspath(__file__)), CATALOG_FILENAME) try: with open(path, "r", encoding="utf-8") as handle: document = json.load(handle) + image_profiles = document.get("imageOperationProfiles") or {} + if not isinstance(image_profiles, Mapping): + raise ValueError("Image operation profiles must be an object.") for model in document.get("models", []): if not isinstance(model, Mapping): continue @@ -107,6 +113,11 @@ def load_model_capability_catalog(force_refresh=False): capabilities = dict(capabilities) if isinstance(model.get("reasoningPolicy"), Mapping): capabilities["reasoningPolicy"] = model["reasoningPolicy"] + for field_name in ("imageProfiles", "imageLifecycle"): + if isinstance(model.get(field_name), Mapping): + capabilities[field_name] = copy.deepcopy(model[field_name]) + if model.get("provider"): + capabilities["publisher"] = model["provider"] if not capabilities: continue @@ -116,11 +127,20 @@ def load_model_capability_catalog(force_refresh=False): catalog[normalized] = capabilities except (OSError, ValueError, TypeError, AttributeError): catalog = {} + image_profiles = {} + _IMAGE_OPERATION_PROFILES = copy.deepcopy(image_profiles) _CATALOG_CACHE = catalog return _CATALOG_CACHE +def get_image_operation_profile(profile_id): + """Return an isolated provider operation profile from the same cached catalog.""" + load_model_capability_catalog() + profile = _IMAGE_OPERATION_PROFILES.get(profile_id) + return copy.deepcopy(profile) if isinstance(profile, Mapping) else None + + def _catalog_lookup(identifier, *, capability=None, reject_version_suffix=False): """Return catalog capabilities for one identifier, or None. @@ -245,14 +265,14 @@ def get_model_catalog_capabilities(model): for identifier in identifiers: normalized = _normalize_model_identifier(identifier) if normalized in catalog: - return dict(catalog[normalized]) + return copy.deepcopy(catalog[normalized]) snapshot = re.fullmatch(r"(.+)-(\d{4})-(\d{2})-(\d{2})", normalized) if snapshot and snapshot.group(1) in catalog: try: date(*(int(value) for value in snapshot.groups()[1:])) except ValueError: continue - return dict(catalog[snapshot.group(1)]) + return copy.deepcopy(catalog[snapshot.group(1)]) return None diff --git a/application/single_app/functions_model_endpoint_auth.py b/application/single_app/functions_model_endpoint_auth.py new file mode 100644 index 000000000..2a4bf0b20 --- /dev/null +++ b/application/single_app/functions_model_endpoint_auth.py @@ -0,0 +1,188 @@ +# functions_model_endpoint_auth.py +"""Server-side API key, bearer, OAuth2, and certificate helpers for Custom APIs.""" + +import hashlib +import json +import threading +import time +from typing import Any + +import httpx + +from functions_model_endpoint_diagnostics import build_sanitized_model_endpoint_error +from functions_model_endpoint_providers import ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, + DEFAULT_CUSTOM_AUTH_TYPES, + normalize_custom_endpoint_auth_type, +) +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + validate_custom_api_key_header, + validate_custom_model_endpoint_url, +) + + +CUSTOM_ENDPOINT_AUTH_TYPES = DEFAULT_CUSTOM_AUTH_TYPES +OAUTH2_EXPIRY_SKEW_SECONDS = 60 +OAUTH2_DEFAULT_EXPIRY_SECONDS = 3600 +OAUTH2_REQUEST_TIMEOUT_SECONDS = 30 +_TOKEN_CACHE: dict[str, tuple[str, float]] = {} +_TOKEN_CACHE_LOCK = threading.Lock() + + +def resolve_api_key_header(auth: dict, default_header: str = "", default_prefix: str = "") -> tuple[str, str]: + auth = auth or {} + validate_custom_api_key_header(auth) + override = str(auth.get("api_key_header") or "").strip() + if override: + return override, str(auth.get("api_key_prefix") or "").strip() + prefix = auth.get("api_key_prefix") if "api_key_prefix" in auth else default_prefix + return str(default_header or "").strip(), str(prefix or "").strip() + + +def build_api_key_headers(auth: dict, default_header: str = "", default_prefix: str = "") -> dict[str, str]: + api_key = str((auth or {}).get("api_key") or "").strip() + if not api_key: + raise ModelEndpointValidationError("Selected Custom endpoint is missing an API key.") + if any(ord(character) < 32 or ord(character) > 126 for character in api_key): + raise ModelEndpointValidationError("Custom API key contains invalid header characters.") + name, prefix = resolve_api_key_header(auth, default_header, default_prefix) + return {name: f"{prefix} {api_key}" if prefix else api_key} if name else {} + + +def build_bearer_headers(auth: dict) -> dict[str, str]: + token = str((auth or {}).get("bearer_token") or "").strip() + if not token: + raise ModelEndpointValidationError("Selected Custom endpoint is missing a bearer token.") + if any(ord(character) < 32 or ord(character) > 126 for character in token): + raise ModelEndpointValidationError("Custom bearer token contains invalid header characters.") + return {"Authorization": f"Bearer {token}"} + + +def clear_oauth2_token_cache() -> None: + with _TOKEN_CACHE_LOCK: + _TOKEN_CACHE.clear() + + +def fetch_oauth2_client_credentials_token( + auth: dict, + *, + allow_private: bool = False, + allow_insecure: bool = False, + ca_bundle_path: str = "", + client_cert: Any = None, + http_client_factory=None, +) -> str: + """Fetch through the pinned transport; cache by credentials AND network policy.""" + auth = auth or {} + if not all(str(auth.get(field) or "").strip() for field in ("token_url", "client_id", "client_secret")): + raise ModelEndpointValidationError("Custom OAuth2 requires a token URL, client ID, and client secret.") + token_url = validate_custom_model_endpoint_url( + auth["token_url"], allow_private=allow_private, allow_insecure=allow_insecure, + ) + cache_material = ( + token_url, str(auth["client_id"]), str(auth["client_secret"]), str(auth.get("scope") or ""), + allow_private, allow_insecure, ca_bundle_path, client_cert, + ) + cache_key = hashlib.sha256(json.dumps(cache_material).encode("utf-8")).hexdigest() + with _TOKEN_CACHE_LOCK: + cached = _TOKEN_CACHE.get(cache_key) + if cached and cached[1] > time.monotonic(): + return cached[0] + # Deferred to avoid the auth -> transport -> auth import cycle. + from model_endpoint_clients import build_custom_openai_sync_http_client + + factory = http_client_factory or build_custom_openai_sync_http_client + client = factory( + allow_private=allow_private, allow_insecure=allow_insecure, + ca_bundle_path=ca_bundle_path, client_cert=client_cert, + ) + payload = { + "grant_type": "client_credentials", + "client_id": str(auth["client_id"]).strip(), + "client_secret": str(auth["client_secret"]).strip(), + } + if auth.get("scope"): + payload["scope"] = str(auth["scope"]).strip() + try: + with client: + response = client.post( + token_url, data=payload, timeout=OAUTH2_REQUEST_TIMEOUT_SECONDS, follow_redirects=False, + ) + response.raise_for_status() + token_payload = response.json() + except (httpx.HTTPError, ValueError) as exc: + raise build_sanitized_model_endpoint_error( + "Custom endpoint token request failed.", exc, request_url=token_url, + status_code=getattr(getattr(exc, "response", None), "status_code", None), + ) from exc + token = token_payload.get("access_token") if isinstance(token_payload, dict) else None + if not isinstance(token, str) or not token.strip() or any(ord(character) < 32 or ord(character) > 126 for character in token): + raise build_sanitized_model_endpoint_error( + "Custom endpoint token response did not contain an access token.", request_url=token_url, + ) + token_type = str(token_payload.get("token_type") or "Bearer").lower() + if token_type != "bearer": + raise build_sanitized_model_endpoint_error("Custom endpoint token response uses an unsupported token type.") + try: + expires_in = int(token_payload.get("expires_in", OAUTH2_DEFAULT_EXPIRY_SECONDS)) + except (TypeError, ValueError): + expires_in = OAUTH2_DEFAULT_EXPIRY_SECONDS + lifetime = max(0, expires_in - min(OAUTH2_EXPIRY_SKEW_SECONDS, max(0, expires_in // 10))) + with _TOKEN_CACHE_LOCK: + now = time.monotonic() + expired = [key for key, value in _TOKEN_CACHE.items() if value[1] <= now] + for key in expired: + _TOKEN_CACHE.pop(key, None) + if len(_TOKEN_CACHE) >= 256: + _TOKEN_CACHE.pop(next(iter(_TOKEN_CACHE))) + _TOKEN_CACHE[cache_key] = (token.strip(), now + lifetime) + return token.strip() + + +def resolve_custom_endpoint_credentials( + auth: dict, + *, + default_api_key_header: str = "Authorization", + default_api_key_prefix: str = "Bearer", + allow_private: bool = False, + allow_insecure: bool = False, + ca_bundle_path: str = "", + client_cert: Any = None, +) -> tuple[str, dict[str, str]]: + """Return an OpenAI SDK credential plus exact authentication header overrides. + + Non-Authorization key schemes use a placeholder SDK key and explicitly empty + Authorization, so the real key is sent only in the configured header. Callers + must pass both return values, not discard the headers. + """ + auth = auth or {} + auth_type = normalize_custom_endpoint_auth_type(auth.get("type") or AUTH_TYPE_API_KEY) + if auth_type == AUTH_TYPE_API_KEY: + headers = build_api_key_headers(auth, default_api_key_header, default_api_key_prefix) + authorization = next((value for name, value in headers.items() if name.lower() == "authorization"), None) + if authorization is not None: + return str(auth["api_key"]).strip(), {"Authorization": authorization} + return "custom-header-auth", {"Authorization": "", **headers} + if auth_type == AUTH_TYPE_BEARER: + headers = build_bearer_headers(auth) + return str(auth["bearer_token"]).strip(), headers + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS: + token = fetch_oauth2_client_credentials_token( + auth, allow_private=allow_private, allow_insecure=allow_insecure, + ca_bundle_path=ca_bundle_path, client_cert=client_cert, + ) + return token, {"Authorization": f"Bearer {token}"} + raise ModelEndpointValidationError("Custom endpoint authentication type is not supported.") + + +def resolve_client_certificate(connection: dict): + """Resolve deployment-mounted PEM paths; private key material never enters settings.""" + connection = connection or {} + cert = str(connection.get("client_cert_path") or "").strip() + key = str(connection.get("client_key_path") or "").strip() + if key and not cert: + raise ModelEndpointValidationError("A Custom client key requires a client certificate.") + return (cert, key) if cert and key else cert or None diff --git a/application/single_app/functions_model_endpoint_diagnostics.py b/application/single_app/functions_model_endpoint_diagnostics.py new file mode 100644 index 000000000..2921f0b59 --- /dev/null +++ b/application/single_app/functions_model_endpoint_diagnostics.py @@ -0,0 +1,104 @@ +# functions_model_endpoint_diagnostics.py +"""Correlated, user-safe Custom endpoint failures with server-side stack context.""" + +import logging +import re +import traceback +import uuid +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +from functions_appinsights import log_event + + +CORRELATION_ID_LENGTH = 8 +MAX_LOGGED_DETAIL_LENGTH = 2000 +_REDACTION_PATTERNS = ( + re.compile(r'(?i)((?:api[-_]?key|x-goog-api-key|client[-_]?secret|access[-_]?token|refresh[-_]?token|bearer[-_]?token|password)["\']?\s*[:=]\s*["\']?)([^"\'\s,&]+)'), + re.compile(r'(?i)(bearer\s+)([A-Za-z0-9\-._~+/]+=*)'), + re.compile(r'(sk-[A-Za-z0-9_-]{8,})'), +) + + +class SanitizedModelEndpointError(RuntimeError): + """Only stable messages and a correlation reference may cross an API boundary.""" + + def __init__(self, public_message): + super().__init__(public_message) + self.public_message = public_message + + +def redact_model_endpoint_secrets(value: Any) -> str: + text = str(value or "") + for pattern in _REDACTION_PATTERNS: + text = pattern.sub( + (lambda match: f"{match.group(1)}[REDACTED]") if pattern.groups >= 2 else "[REDACTED]", + text, + ) + return text[:MAX_LOGGED_DETAIL_LENGTH] + + +def new_model_endpoint_correlation_id() -> str: + return uuid.uuid4().hex[:CORRELATION_ID_LENGTH] + + +def log_custom_model_endpoint_failure( + summary: str, + exception: BaseException | None = None, + *, + api_type: Any = "", + protocol: Any = "", + request_url: Any = "", + status_code: Any = None, + detail: Any = "", +) -> str: + """Log the stack and transport context, never an arbitrary provider body. + + Provider bodies and exception strings may echo credentials without labels. + A frame-only traceback keeps the failing code path without dumping those + strings or local variables. ``detail`` is accepted for historical callers but + intentionally not recorded. + """ + correlation_id = new_model_endpoint_correlation_id() + context = {"correlation_id": correlation_id, "api_type": str(api_type), "protocol": str(protocol)} + if request_url: + try: + parsed = urlsplit(str(request_url)) + host = parsed.hostname or "" + context["request_url"] = redact_model_endpoint_secrets( + urlunsplit((parsed.scheme, host, parsed.path, "", "")) + ) + except ValueError: + context["request_url"] = "[invalid URL]" + if status_code is not None: + context["status_code"] = status_code + if exception is not None: + context["error_type"] = type(exception).__name__ + context["traceback"] = "\n".join( + f"{frame.filename}:{frame.lineno} in {frame.name}" + for frame in traceback.extract_tb(exception.__traceback__) + ) + if exception.__cause__ is not None: + context["cause_type"] = type(exception.__cause__).__name__ + log_event( + f"[MODEL_ENDPOINT] {summary} (reference {correlation_id})", + extra=context, level=logging.ERROR, + ) + return correlation_id + + +def build_sanitized_model_endpoint_error( + message: str, + exception: BaseException | None = None, + *, + api_type: Any = "", + protocol: Any = "", + request_url: Any = "", + status_code: Any = None, + detail: Any = "", +) -> SanitizedModelEndpointError: + reference = log_custom_model_endpoint_failure( + message, exception, api_type=api_type, protocol=protocol, + request_url=request_url, status_code=status_code, detail=detail, + ) + return SanitizedModelEndpointError(f"{message} (reference {reference})") diff --git a/application/single_app/functions_model_endpoint_providers.py b/application/single_app/functions_model_endpoint_providers.py new file mode 100644 index 000000000..3ea08b5a2 --- /dev/null +++ b/application/single_app/functions_model_endpoint_providers.py @@ -0,0 +1,161 @@ +# functions_model_endpoint_providers.py +"""Dependency-light registry of the protocols supported by Custom connections. + +These descriptors describe a transport contract, not a model's capabilities. +In particular, an OpenAI-compatible endpoint does not imply image support. +""" + +from dataclasses import dataclass +from typing import Any + + +MODEL_ENDPOINT_PROVIDER_CUSTOM = "custom" +MODEL_ENDPOINT_API_TYPE_OPENAI = "openai" +MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI = "azure_openai" +MODEL_ENDPOINT_API_TYPE_ANTHROPIC = "anthropic" +MODEL_ENDPOINT_API_TYPE_GEMINI = "gemini" +MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI = "azure_openai" +MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE = "openai_style" +MODEL_ENDPOINT_PROTOCOL_ANTHROPIC = "anthropic" + +URL_POLICY_APPEND_V1_IF_MISSING = "append_v1_if_missing" +URL_POLICY_AS_GIVEN = "as_given" +URL_POLICY_AZURE_DEPLOYMENT = "azure_deployment" +URL_POLICY_ANTHROPIC_MESSAGES = "anthropic_messages" +CUSTOM_ENDPOINT_URL_MODE_AUTO = "auto" +CUSTOM_ENDPOINT_URL_MODE_EXACT = "exact" +CUSTOM_ENDPOINT_URL_MODES = (CUSTOM_ENDPOINT_URL_MODE_AUTO, CUSTOM_ENDPOINT_URL_MODE_EXACT) +MODEL_IDENTIFIER_MODEL_NAME = "model_name" +MODEL_IDENTIFIER_DEPLOYMENT_NAME = "deployment_name" +DEFAULT_ANTHROPIC_VERSION = "2023-06-01" + +AUTH_TYPE_API_KEY = "api_key" +AUTH_TYPE_KEY = "key" +AUTH_TYPE_BEARER = "bearer" +AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS = "oauth2_client_credentials" +DEFAULT_CUSTOM_AUTH_TYPES = ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, +) + + +def normalize_custom_endpoint_url_mode(url_mode: Any) -> str: + """Return a supported URL policy override.""" + normalized = str(url_mode or "").strip().lower() + return normalized if normalized in CUSTOM_ENDPOINT_URL_MODES else CUSTOM_ENDPOINT_URL_MODE_AUTO + + +def normalize_custom_endpoint_auth_type(auth_type: Any) -> str: + """Return the canonical auth type, or an empty string for unsupported types.""" + normalized = str(auth_type or "").strip().lower() + if normalized == AUTH_TYPE_KEY: + return AUTH_TYPE_API_KEY + return normalized if normalized in DEFAULT_CUSTOM_AUTH_TYPES else "" + + +@dataclass(frozen=True) +class ModelEndpointProvider: + """An API type's wire protocol, URL, credential, and model identity contract.""" + + api_type: str + display_name: str + protocol: str + model_identifier: str + url_policy: str + auth_types: tuple[str, ...] = DEFAULT_CUSTOM_AUTH_TYPES + default_api_key_header: str = "Authorization" + default_api_key_prefix: str = "Bearer" + requires_api_version: bool = False + version_field: str = "" + default_version: str = "" + supports_streaming: bool = True + supports_tools: bool = True + supports_stream_options: bool = False + description: str = "" + + @property + def uses_model_name(self) -> bool: + return self.model_identifier == MODEL_IDENTIFIER_MODEL_NAME + + def to_ui_option(self) -> dict: + """Return non-secret configuration metadata for the global editors.""" + return { + "value": self.api_type, + "label": self.display_name, + "protocol": self.protocol, + "urlPolicy": self.url_policy, + "usesModelName": self.uses_model_name, + "requiresApiVersion": self.requires_api_version, + "versionField": self.version_field, + "defaultVersion": self.default_version, + "authTypes": list(self.auth_types), + "defaultApiKeyHeader": self.default_api_key_header, + "defaultApiKeyPrefix": self.default_api_key_prefix, + "description": self.description, + } + + +MODEL_ENDPOINT_PROVIDERS = ( + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_OPENAI, + display_name="OpenAI API", + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + url_policy=URL_POLICY_APPEND_V1_IF_MISSING, + supports_stream_options=True, + description="OpenAI or an OpenAI-compatible gateway. Compatibility alone does not grant image capabilities.", + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + display_name="Azure OpenAI API", + protocol=MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, + model_identifier=MODEL_IDENTIFIER_DEPLOYMENT_NAME, + url_policy=URL_POLICY_AZURE_DEPLOYMENT, + default_api_key_header="api-key", + default_api_key_prefix="", + requires_api_version=True, + version_field="api_version", + description="Azure OpenAI addressed by deployment name and an explicit API version.", + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + display_name="Anthropic", + protocol=MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + url_policy=URL_POLICY_ANTHROPIC_MESSAGES, + default_api_key_header="x-api-key", + default_api_key_prefix="", + version_field="anthropic_version", + default_version=DEFAULT_ANTHROPIC_VERSION, + description="Anthropic's messages API, direct or through a compatible gateway.", + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_GEMINI, + display_name="Google Gemini (OpenAI-compatible)", + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + url_policy=URL_POLICY_AS_GIVEN, + description="Gemini's OpenAI-compatible base, normally https://generativelanguage.googleapis.com/v1beta/openai/.", + ), +) +MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE = { + provider.api_type: provider for provider in MODEL_ENDPOINT_PROVIDERS +} +MODEL_ENDPOINT_CUSTOM_API_TYPES = frozenset(MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE) + + +def normalize_api_type_value(api_type: Any) -> str: + return str(api_type or "").strip().lower().replace("-", "_") + + +def get_model_endpoint_provider(api_type: Any) -> ModelEndpointProvider | None: + return MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE.get(normalize_api_type_value(api_type)) + + +def get_model_endpoint_provider_ui_options() -> list[dict]: + return [provider.to_ui_option() for provider in MODEL_ENDPOINT_PROVIDERS] + + +def is_supported_custom_api_type(api_type: Any) -> bool: + return normalize_api_type_value(api_type) in MODEL_ENDPOINT_CUSTOM_API_TYPES diff --git a/application/single_app/functions_model_endpoint_runtime.py b/application/single_app/functions_model_endpoint_runtime.py index 922e5c6fd..9921728a1 100644 --- a/application/single_app/functions_model_endpoint_runtime.py +++ b/application/single_app/functions_model_endpoint_runtime.py @@ -1,7 +1,7 @@ # functions_model_endpoint_runtime.py """Runtime helpers for configured model endpoint clients and Semantic Kernel services.""" -from openai import AsyncOpenAI, AzureOpenAI +from openai import AsyncOpenAI, AzureOpenAI, OpenAI from azure.identity import ClientSecretCredential, DefaultAzureCredential, get_bearer_token_provider from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion @@ -13,21 +13,40 @@ require_model_capability, ) from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_auth import resolve_client_certificate, resolve_custom_endpoint_credentials +from functions_model_endpoint_providers import get_model_endpoint_provider +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) +from functions_model_endpoint_validation import ( + CUSTOM_ENDPOINT_VERSION_PATTERN, + ModelEndpointValidationError, + custom_endpoint_setting_enabled, + validate_custom_model_endpoint_url, +) from functions_settings import resolve_model_endpoint_foundry_scope from model_endpoint_clients import ( MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, AnthropicSemanticKernelChatCompletion, + CustomAnthropicChatCompletionClient, + OpenAIStyleChatCompletionClient, build_anthropic_chat_client, build_openai_style_chat_client, + build_custom_openai_async_http_client, + build_custom_openai_sync_http_client, infer_model_endpoint_protocol, normalize_openai_style_base_url, resolve_openai_style_request_api_version, + resolve_custom_azure_openai_base_url, + resolve_custom_openai_base_url, + sanitize_custom_async_openai_client, ) -MODEL_ENDPOINT_PROVIDER_ALLOWLIST = {'aoai', 'aifoundry', 'new_foundry', 'anthropic', 'claude'} +MODEL_ENDPOINT_PROVIDER_ALLOWLIST = {'aoai', 'aifoundry', 'new_foundry', 'anthropic', 'claude', 'custom'} MODEL_CONTEXT_AUTH_FIELDS = ( 'type', 'tenant_id', @@ -67,9 +86,11 @@ def build_model_endpoint_context( endpoint=None, auth=None, api_version=None, + api_type=None, endpoint_id=None, model_id=None, model_deployment=None, + request_model=None, user_id=None, active_group_ids=None, ): @@ -78,9 +99,11 @@ def build_model_endpoint_context( 'provider': str(provider or '').strip().lower(), 'endpoint': str(endpoint or '').strip(), 'api_version': str(api_version or '').strip(), + 'api_type': str(api_type or '').strip().lower(), 'endpoint_id': str(endpoint_id or '').strip(), 'model_id': str(model_id or '').strip(), 'model_deployment': str(model_deployment or '').strip(), + 'request_model': str(request_model or model_deployment or '').strip(), } normalized_user_id = str(user_id or '').strip() @@ -133,7 +156,8 @@ def _require_chat_model_for_endpoint(endpoint_config, deployment_name='', model_ if not model_id and deployment_name: deployment_matches = [ model for model in models - if str(deployment_name).strip() in { + if str(deployment_name).strip() == resolve_model_endpoint_request_model(endpoint_config, model) + or str(deployment_name).strip() in { str(model.get(field) or '').strip() for field in ('deploymentName', 'deployment') } ] @@ -141,6 +165,8 @@ def _require_chat_model_for_endpoint(endpoint_config, deployment_name='', model_ for model in models: if model_id: matches = str(model.get('id') or '').strip() == str(model_id).strip() + elif provider == 'custom': + matches = bool(deployment_name) and str(deployment_name).strip() == resolve_model_endpoint_request_model(endpoint_config, model) else: matches = bool(deployment_name) and str(deployment_name).strip() in { str(model.get(field) or '').strip() @@ -155,6 +181,90 @@ def _require_chat_model_for_endpoint(endpoint_config, deployment_name='', model_ return None +def _resolve_custom_endpoint_runtime_options(endpoint_config, settings): + if not isinstance(endpoint_config, dict) or endpoint_config.get('provider') != 'custom': + raise ModelEndpointValidationError('A configured Custom endpoint is required.') + descriptor = get_model_endpoint_provider(get_model_endpoint_api_type(endpoint_config)) + if descriptor is None: + raise ModelEndpointValidationError('Custom endpoint API type is not supported.') + settings = settings or {} + connection = endpoint_config.get('connection') or {} + transport_options = { + 'allow_private': custom_endpoint_setting_enabled(settings, 'allow_private_custom_model_endpoints'), + 'allow_insecure': custom_endpoint_setting_enabled(settings, 'allow_insecure_custom_model_endpoints'), + 'ca_bundle_path': str(settings.get('custom_model_endpoint_ca_bundle_path') or '').strip(), + 'client_cert': resolve_client_certificate(connection), + } + endpoint = validate_custom_model_endpoint_url( + connection.get('endpoint'), + allow_private=transport_options['allow_private'], + allow_insecure=transport_options['allow_insecure'], + ) + if descriptor.version_field: + version = connection.get(descriptor.version_field) or descriptor.default_version + if descriptor.requires_api_version or version: + if not CUSTOM_ENDPOINT_VERSION_PATTERN.fullmatch(str(version or '')): + raise ModelEndpointValidationError('The configured Custom API version is invalid.') + credential, headers = resolve_custom_endpoint_credentials( + endpoint_config.get('auth') or {}, + default_api_key_header=descriptor.default_api_key_header, + default_api_key_prefix=descriptor.default_api_key_prefix, + **transport_options, + ) + return descriptor, endpoint, credential, headers, transport_options + + +def build_custom_openai_client_kwargs( + endpoint_config, settings, *, request_model='', asynchronous=False, default_headers=None, +): + """Build guarded SDK options reusable by chat, Images, and Responses adapters. + + This constructs transport/auth options only; callers must first enforce their + operation's model capability and publication policy. It never grants image + support based on an OpenAI-compatible protocol. + """ + descriptor, endpoint, credential, headers, transport_options = _resolve_custom_endpoint_runtime_options( + endpoint_config, settings, + ) + connection = endpoint_config.get('connection') or {} + if descriptor.protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI: + base_url = resolve_custom_azure_openai_base_url(endpoint, request_model, connection.get('url_mode')) + elif descriptor.protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: + base_url = resolve_custom_openai_base_url(endpoint, descriptor.api_type, connection.get('url_mode')) + else: + raise ModelEndpointValidationError('This Custom API type does not support the OpenAI SDK.') + merged_headers = {name.lower(): value for name, value in (default_headers or {}).items()} + merged_headers.update({name.lower(): value for name, value in headers.items()}) + if 'authorization' in merged_headers: + # The SDK merges its defaults case-sensitively before HTTPX folds names. + # Match the SDK spelling to replace, rather than duplicate, its header. + merged_headers['Authorization'] = merged_headers.pop('authorization') + factory = build_custom_openai_async_http_client if asynchronous else build_custom_openai_sync_http_client + kwargs = { + 'api_key': credential, + 'base_url': base_url, + 'default_headers': merged_headers, + 'http_client': factory(**transport_options), + } + if descriptor.protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI: + kwargs['default_query'] = {'api-version': connection['api_version']} + return kwargs + + +def _build_custom_anthropic_client(endpoint_config, settings, extra_headers): + descriptor, endpoint, _, headers, transport_options = _resolve_custom_endpoint_runtime_options( + endpoint_config, settings, + ) + connection = endpoint_config.get('connection') or {} + merged_headers = {name.lower(): value for name, value in (extra_headers or {}).items()} + merged_headers.update({name.lower(): value for name, value in headers.items()}) + return CustomAnthropicChatCompletionClient( + endpoint=endpoint, auth_headers=merged_headers, + anthropic_version=connection.get('anthropic_version') or descriptor.default_version, + url_mode=connection.get('url_mode') or 'auto', **transport_options, + ) + + def build_model_endpoint_sync_chat_client( auth_settings, provider, @@ -162,6 +272,7 @@ def build_model_endpoint_sync_chat_client( api_version, deployment_name='', *, + api_type='', settings=None, endpoint_config=None, identity_context=None, @@ -175,7 +286,23 @@ def build_model_endpoint_sync_chat_client( identity_context=identity_context, ) normalized_provider = str(provider or 'aoai').strip().lower() - runtime_protocol = infer_model_endpoint_protocol(normalized_provider, endpoint, deployment_name) + api_type = get_model_endpoint_api_type(endpoint_config) or api_type + runtime_protocol = infer_model_endpoint_protocol(normalized_provider, endpoint, deployment_name, api_type) + if normalized_provider == 'custom': + connection = dict((endpoint_config or {}).get('connection') or {}) + connection.update(endpoint=endpoint, api_version=connection.get('api_version') or api_version) + custom_config = { + **(endpoint_config or {}), 'provider': 'custom', 'api_type': api_type, + 'connection': connection, 'auth': auth_settings, + } + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: + return _build_custom_anthropic_client(custom_config, settings, extra_headers), runtime_protocol + kwargs = build_custom_openai_client_kwargs( + custom_config, settings, request_model=deployment_name, default_headers=extra_headers, + ) + return OpenAIStyleChatCompletionClient( + OpenAI(**kwargs), sanitize_errors=True, api_type=api_type, request_url=kwargs['base_url'], + ), runtime_protocol auth_type = str(auth_settings.get('type') or 'managed_identity').strip().lower() if auth_type in ('api_key', 'key'): @@ -243,7 +370,9 @@ def build_chat_connection_client(binding, settings): endpoint_config = binding.endpoint provider = endpoint_config.get('provider') or 'aoai' model = require_model_capability(binding.model, provider=provider) - deployment_name = model.get('deploymentName') or model.get('deployment') or model.get('id') or '' + deployment_name = resolve_model_endpoint_request_model(endpoint_config, model) + if not deployment_name: + raise AIConnectionError('The selected chat model is missing its request identifier.') _require_chat_model_for_endpoint(endpoint_config, deployment_name) endpoint_config = keyvault_model_endpoint_get_helper( endpoint_config, @@ -287,7 +416,7 @@ def resolve_model_endpoint_from_context(settings, model_context, *, authorize=Fa model_context = model_context if isinstance(model_context, dict) else {} requested_endpoint_id = str(model_context.get('endpoint_id') or '').strip() requested_model_id = str(model_context.get('model_id') or '').strip() - requested_deployment = str(model_context.get('model_deployment') or '').strip() + requested_deployment = str(model_context.get('request_model') or model_context.get('model_deployment') or '').strip() requested_provider = str(model_context.get('provider') or '').strip().lower() if not settings.get('enable_multi_model_endpoints', False): return None @@ -347,7 +476,7 @@ def resolve_model_endpoint_from_context(settings, model_context, *, authorize=Fa models = endpoint_cfg.get('models', []) or [] matched_model = None for model_cfg in models: - deployment = str(model_cfg.get('deploymentName') or model_cfg.get('deployment') or '').strip() + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) if requested_model_id: if str(model_cfg.get('id') or '').strip() != requested_model_id: continue @@ -397,7 +526,7 @@ def build_semantic_kernel_chat_service_for_model( endpoint = str(model_context.get('endpoint') or '').strip() api_version = str(model_context.get('api_version') or '').strip() auth_settings = model_context.get('auth') if isinstance(model_context.get('auth'), dict) else {} - deployment_name = str(model_context.get('model_deployment') or gpt_model or '').strip() + deployment_name = str(model_context.get('request_model') or model_context.get('model_deployment') or gpt_model or '').strip() if resolved_model_endpoint: provider = str(resolved_model_endpoint.get('provider') or provider or 'aoai').strip().lower() @@ -414,13 +543,36 @@ def build_semantic_kernel_chat_service_for_model( resolved_model_endpoint, deployment_name, requested_model_id, ) if matched_model: - deployment_name = str( - matched_model.get('deploymentName') or matched_model.get('deployment') or deployment_name - ).strip() + deployment_name = resolve_model_endpoint_request_model(resolved_model_endpoint, matched_model) elif deployment_name: require_model_capability(deployment_name, provider=provider or 'aoai') + if provider == 'custom' and (not resolved_model_endpoint or not endpoint or not deployment_name): + raise AIConnectionError('The selected Custom chat connection or request model is unavailable.') if provider and endpoint and deployment_name: + if provider == 'custom': + if resolved_model_endpoint is None: + raise AIConnectionError('The selected Custom connection must be resolved from saved settings.') + api_type = get_model_endpoint_api_type(resolved_model_endpoint) + runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name, api_type) + extra_headers = build_model_endpoint_identity_headers( + settings, endpoint_config=resolved_model_endpoint, identity_context=model_context, + ) + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: + return AnthropicSemanticKernelChatCompletion( + service_id=service_id, deployment_name=deployment_name, endpoint=endpoint, + custom_client=_build_custom_anthropic_client(resolved_model_endpoint, settings, extra_headers), + ), runtime_protocol + kwargs = build_custom_openai_client_kwargs( + resolved_model_endpoint, settings, request_model=deployment_name, + asynchronous=True, default_headers=extra_headers, + ) + async_client = sanitize_custom_async_openai_client( + AsyncOpenAI(**kwargs), api_type=api_type, request_url=kwargs['base_url'], + ) + return OpenAIChatCompletion( + service_id=service_id, ai_model_id=deployment_name, async_client=async_client, + ), runtime_protocol runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) auth_type = str(auth_settings.get('type') or 'managed_identity').lower() extra_headers = build_model_endpoint_identity_headers( diff --git a/application/single_app/functions_model_endpoint_types.py b/application/single_app/functions_model_endpoint_types.py new file mode 100644 index 000000000..ca6a6b5b1 --- /dev/null +++ b/application/single_app/functions_model_endpoint_types.py @@ -0,0 +1,63 @@ +# functions_model_endpoint_types.py +"""Canonical Custom provider, API type, and request model identifiers.""" + +from typing import Any + +from functions_model_endpoint_providers import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_API_TYPE_GEMINI, + MODEL_ENDPOINT_API_TYPE_OPENAI, + MODEL_ENDPOINT_CUSTOM_API_TYPES, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_provider, + normalize_api_type_value, +) + + +__all__ = [ + "DEFAULT_ANTHROPIC_VERSION", + "MODEL_ENDPOINT_API_TYPE_ANTHROPIC", + "MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI", + "MODEL_ENDPOINT_API_TYPE_GEMINI", + "MODEL_ENDPOINT_API_TYPE_OPENAI", + "MODEL_ENDPOINT_CUSTOM_API_TYPES", + "MODEL_ENDPOINT_PROVIDER_CUSTOM", + "get_model_endpoint_api_type", + "normalize_model_endpoint_api_type", + "resolve_model_endpoint_request_model", +] + + +def normalize_model_endpoint_api_type(provider: Any, api_type: Any) -> str: + if str(provider or "").strip().lower() != MODEL_ENDPOINT_PROVIDER_CUSTOM: + return "" + normalized = normalize_api_type_value(api_type) + return normalized if normalized in MODEL_ENDPOINT_CUSTOM_API_TYPES else "" + + +def get_model_endpoint_api_type(endpoint: Any) -> str: + if not isinstance(endpoint, dict): + return "" + return normalize_model_endpoint_api_type(endpoint.get("provider"), endpoint.get("api_type")) + + +def resolve_model_endpoint_request_model(endpoint: Any, model: Any) -> str: + """Resolve a wire identifier, never the registry ID used for saved selections.""" + endpoint_data = endpoint if isinstance(endpoint, dict) else {} + model_data = model if isinstance(model, dict) else {} + if str(endpoint_data.get("provider") or "").strip().lower() == MODEL_ENDPOINT_PROVIDER_CUSTOM: + provider = get_model_endpoint_provider(get_model_endpoint_api_type(endpoint_data)) + if provider is None: + return "" + if provider.uses_model_name: + return str(model_data.get("modelName") or model_data.get("name") or "").strip() + return str(model_data.get("deploymentName") or model_data.get("deployment") or "").strip() + return str( + model_data.get("deploymentName") + or model_data.get("deployment") + or model_data.get("modelName") + or model_data.get("name") + or "" + ).strip() diff --git a/application/single_app/functions_model_endpoint_validation.py b/application/single_app/functions_model_endpoint_validation.py new file mode 100644 index 000000000..cdf80de67 --- /dev/null +++ b/application/single_app/functions_model_endpoint_validation.py @@ -0,0 +1,258 @@ +# functions_model_endpoint_validation.py +"""Configuration and outbound-address validation for Custom model endpoints.""" + +import ipaddress +import re +import socket +from typing import Any +from urllib.parse import urlparse, urlunparse + +from functions_model_endpoint_providers import ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, + CUSTOM_ENDPOINT_URL_MODES, + get_model_endpoint_provider, + normalize_custom_endpoint_auth_type, +) +from functions_model_endpoint_types import ( + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) + + +CUSTOM_ENDPOINT_MAX_URL_LENGTH = 2048 +CUSTOM_ENDPOINT_VERSION_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,64}$") +CUSTOM_ENDPOINT_HEADER_PATTERN = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") +CUSTOM_ENDPOINT_BLOCKED_HEADERS = { + "host", "content-length", "connection", "transfer-encoding", + "cookie", "set-cookie", "upgrade", "trailer", "te", +} +CUSTOM_ENDPOINT_BLOCKED_HOSTNAMES = { + "instance-data.ec2.internal", "localhost", "localhost.localdomain", + "metadata.azure.com", "metadata.google.internal", +} +CUSTOM_ENDPOINT_BLOCKED_IPS = { + ipaddress.ip_address(value) + for value in ("168.63.129.16", "169.254.169.254", "169.254.169.250", "169.254.169.251") +} +CUSTOM_ENDPOINT_PRIVATE_NETWORKS = tuple( + ipaddress.ip_network(value) + for value in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7") +) + + +class ModelEndpointValidationError(ValueError): + """A stable, user-safe configuration or outbound-policy error.""" + + def __init__(self, public_message): + super().__init__(public_message) + self.public_message = public_message + + +class ModelEndpointUnresolvableError(ModelEndpointValidationError): + """DNS is unavailable; saving is allowed, but a request must fail closed.""" + + +def custom_endpoint_setting_enabled(settings: dict | None, key: str) -> bool: + """Do not treat a serialized false value as permission to widen network access.""" + return (settings or {}).get(key) is True + + +def validate_custom_model_endpoint_address(address: str, *, allow_private: bool = False) -> None: + try: + parsed = ipaddress.ip_address(address) + except ValueError as exc: + raise ModelEndpointValidationError("Custom endpoint resolved to an invalid address.") from exc + if isinstance(parsed, ipaddress.IPv6Address) and (parsed.ipv4_mapped or parsed.sixtofour or parsed.teredo): + raise ModelEndpointValidationError("Custom endpoint must not use an IPv4 transition address.") + if parsed in CUSTOM_ENDPOINT_BLOCKED_IPS: + raise ModelEndpointValidationError("Custom endpoint resolves to a blocked platform address.") + if parsed.is_loopback or parsed.is_link_local: + raise ModelEndpointValidationError("Custom endpoint must not use a loopback or link-local address.") + if parsed.is_multicast or parsed.is_reserved or parsed.is_unspecified: + raise ModelEndpointValidationError("Custom endpoint must resolve to a usable network address.") + if any(parsed in network for network in CUSTOM_ENDPOINT_PRIVATE_NETWORKS if network.version == parsed.version): + if not allow_private: + raise ModelEndpointValidationError("Private Custom endpoint hosts are not enabled by the administrator.") + return + if not parsed.is_global: + raise ModelEndpointValidationError("Custom endpoint must resolve to a globally routable address.") + + +def resolve_custom_model_endpoint_addresses( + hostname: str, port: int = 443, *, allow_private: bool = False, +) -> tuple[str, ...]: + """Validate every answer, then return exactly the addresses a transport may use.""" + try: + answers = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + raise ModelEndpointUnresolvableError("Custom endpoint hostname could not be resolved.") from exc + if not answers: + raise ModelEndpointUnresolvableError("Custom endpoint hostname did not resolve to an address.") + addresses = [] + for answer in answers: + address = answer[4][0] + validate_custom_model_endpoint_address(address, allow_private=allow_private) + if address not in addresses: + addresses.append(address) + return tuple(addresses) + + +def validate_custom_model_endpoint_url( + endpoint: Any, + *, + allow_private: bool = False, + allow_insecure: bool = False, + require_resolvable: bool = True, +) -> str: + """Normalize an administrator URL; connect-time DNS pinning remains mandatory.""" + text = str(endpoint or "").strip() + if not text or len(text) > CUSTOM_ENDPOINT_MAX_URL_LENGTH: + raise ModelEndpointValidationError("Custom endpoint URL is required and must not exceed 2048 characters.") + if any(ord(character) < 33 for character in text) or "\\" in text: + raise ModelEndpointValidationError("Custom endpoint URL contains invalid characters.") + try: + parsed = urlparse(text) + port = parsed.port + hostname = str(parsed.hostname or "").lower().rstrip(".") + except ValueError as exc: + raise ModelEndpointValidationError("Custom endpoint URL is invalid.") from exc + if parsed.scheme != "https": + if parsed.scheme != "http" or not (allow_private and allow_insecure): + raise ModelEndpointValidationError( + "Custom endpoint URL must use HTTPS. HTTP requires both private-host and insecure-endpoint permission." + ) + if not parsed.netloc or not hostname or port == 0: + raise ModelEndpointValidationError("Custom endpoint URL must include a valid hostname and port.") + if parsed.username is not None or parsed.password is not None: + raise ModelEndpointValidationError("Custom endpoint URL must not contain credentials.") + if parsed.query or parsed.fragment or parsed.params: + raise ModelEndpointValidationError("Custom endpoint URL must not contain query parameters or a fragment.") + if "%" in hostname: + raise ModelEndpointValidationError("Custom endpoint hostname is invalid.") + try: + hostname = hostname.encode("idna").decode("ascii") + except UnicodeError as exc: + raise ModelEndpointValidationError("Custom endpoint hostname is invalid.") from exc + if hostname in CUSTOM_ENDPOINT_BLOCKED_HOSTNAMES or hostname.endswith(".localhost"): + raise ModelEndpointValidationError("Custom endpoint hostname is blocked.") + try: + literal = ipaddress.ip_address(hostname) + except ValueError: + literal = None + if literal is not None: + if not allow_private: + raise ModelEndpointValidationError("Custom endpoint IP addresses require private-host permission.") + validate_custom_model_endpoint_address(hostname, allow_private=allow_private) + else: + if not re.fullmatch(r"[a-z0-9.-]+", hostname) or ".." in hostname: + raise ModelEndpointValidationError("Custom endpoint hostname is invalid.") + if not allow_private and ("." not in hostname or hostname.endswith((".internal", ".local"))): + raise ModelEndpointValidationError("Custom endpoint must use a public fully qualified hostname.") + try: + resolve_custom_model_endpoint_addresses( + hostname, port or (443 if parsed.scheme == "https" else 80), allow_private=allow_private, + ) + except ModelEndpointUnresolvableError: + if require_resolvable: + raise + netloc = f"[{hostname}]" if literal is not None and literal.version == 6 else hostname + if port and port != (443 if parsed.scheme == "https" else 80): + netloc = f"{netloc}:{port}" + return urlunparse((parsed.scheme, netloc, parsed.path, "", "", "")).rstrip("/") + + +def validate_custom_api_key_header(auth: dict) -> None: + """Prevent auth overrides from altering routing or HTTP message framing.""" + name = str(auth.get("api_key_header") or "").strip() + prefix = str(auth.get("api_key_prefix") or "") + if name and ( + not CUSTOM_ENDPOINT_HEADER_PATTERN.fullmatch(name) + or name.lower() in CUSTOM_ENDPOINT_BLOCKED_HEADERS + or name.lower().startswith("proxy-") + ): + raise ModelEndpointValidationError("Custom API key header is not an allowed authentication header.") + if any(ord(character) < 32 or ord(character) > 126 for character in prefix): + raise ModelEndpointValidationError("Custom API key prefix contains invalid characters.") + + +def validate_custom_model_endpoint( + endpoint: Any, settings: dict | None = None, *, require_api_key: bool = True, +) -> None: + """Validate a normalized record without requiring the deployment to be reachable.""" + if not isinstance(endpoint, dict): + raise ModelEndpointValidationError("Custom endpoint configuration must be an object.") + if str(endpoint.get("provider") or "").strip().lower() != MODEL_ENDPOINT_PROVIDER_CUSTOM: + return + if not str(endpoint.get("name") or "").strip(): + raise ModelEndpointValidationError("Custom endpoint name is required.") + provider = get_model_endpoint_provider(get_model_endpoint_api_type(endpoint)) + if provider is None: + raise ModelEndpointValidationError("Custom endpoint API type is not supported.") + auth = endpoint.get("auth") or {} + connection = endpoint.get("connection") or {} + if not isinstance(auth, dict) or not isinstance(connection, dict): + raise ModelEndpointValidationError("Custom endpoint authentication and connection must be objects.") + auth_type = normalize_custom_endpoint_auth_type(auth.get("type")) + if auth_type not in provider.auth_types: + raise ModelEndpointValidationError("Custom endpoints require API key, bearer token, or OAuth2 authentication.") + validate_custom_api_key_header(auth) + if require_api_key: + required_fields = { + AUTH_TYPE_API_KEY: ("api_key",), + AUTH_TYPE_BEARER: ("bearer_token",), + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS: ("token_url", "client_id", "client_secret"), + }[auth_type] + if not all(str(auth.get(field) or "").strip() for field in required_fields): + raise ModelEndpointValidationError("The selected Custom authentication scheme requires its credentials.") + allow_private = custom_endpoint_setting_enabled(settings, "allow_private_custom_model_endpoints") + allow_insecure = custom_endpoint_setting_enabled(settings, "allow_insecure_custom_model_endpoints") + connection["endpoint"] = validate_custom_model_endpoint_url( + connection.get("endpoint"), allow_private=allow_private, + allow_insecure=allow_insecure, require_resolvable=False, + ) + endpoint["connection"] = connection + if connection.get("url_mode", "auto") not in CUSTOM_ENDPOINT_URL_MODES: + raise ModelEndpointValidationError("Custom endpoint URL mode must be auto or exact.") + if connection.get("client_key_path") and not connection.get("client_cert_path"): + raise ModelEndpointValidationError("A Custom client key requires a client certificate.") + for field in ("client_cert_path", "client_key_path"): + if field in connection and not isinstance(connection[field], str): + raise ModelEndpointValidationError("Custom certificate paths must be text.") + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS: + if not auth.get("token_url") or not auth.get("client_id"): + raise ModelEndpointValidationError("Custom OAuth2 requires a token URL and client ID.") + validate_custom_model_endpoint_url( + auth["token_url"], allow_private=allow_private, + allow_insecure=allow_insecure, require_resolvable=False, + ) + if provider.version_field: + version = connection.get(provider.version_field) or provider.default_version + if provider.requires_api_version or version: + if not CUSTOM_ENDPOINT_VERSION_PATTERN.fullmatch(str(version or "")): + raise ModelEndpointValidationError("Custom API version must contain letters, numbers, dots, underscores, or hyphens.") + models = endpoint.get("models") + if not isinstance(models, list) or not models: + raise ModelEndpointValidationError("Custom endpoints require at least one manually configured model.") + seen = set() + for model in models: + request_model = resolve_model_endpoint_request_model(endpoint, model) + if not isinstance(model, dict) or not request_model: + field = "Model Name" if provider.uses_model_name else "Deployment Name" + raise ModelEndpointValidationError(f"Custom endpoint models require {field}.") + if len(request_model) > 512 or any(ord(character) < 32 for character in request_model): + raise ModelEndpointValidationError("Custom request model identifier is invalid.") + if request_model.casefold() in seen: + raise ModelEndpointValidationError("Custom request model identifiers must be unique.") + seen.add(request_model.casefold()) + + +def validate_custom_model_endpoints( + endpoints: Any, settings: dict | None = None, *, require_api_key: bool = True, +) -> None: + if not isinstance(endpoints, list): + raise ModelEndpointValidationError("Model endpoints must be a list.") + for endpoint in endpoints: + validate_custom_model_endpoint(endpoint, settings, require_api_key=require_api_key) diff --git a/application/single_app/functions_orchestration_models.py b/application/single_app/functions_orchestration_models.py index df629b8b6..d5c6ae5ac 100644 --- a/application/single_app/functions_orchestration_models.py +++ b/application/single_app/functions_orchestration_models.py @@ -9,6 +9,7 @@ from typing import Any from functions_model_capabilities import resolve_model_reasoning_effort +from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model from model_endpoint_clients import ( MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, ModelEndpointBehavior, @@ -247,12 +248,12 @@ def resolve_orchestration_model(settings, *, user_id, seeds=None, planner=False, model = next(( item for item in models if isinstance(item, dict) and ( _text(item.get('id')) == selection['model_id'] if selection['model_id'] - else _text(item.get('deploymentName') or item.get('deployment')) == selection['model_deployment'] + else resolve_model_endpoint_request_model(endpoint, item) == selection['model_deployment'] ) ), None) if not model or not model.get('enabled', True): raise OrchestrationModelError() - deployment = _text(model.get('deploymentName') or model.get('deployment')) + deployment = resolve_model_endpoint_request_model(endpoint, model) provider = _text(endpoint.get('provider')).lower() if ( not deployment or provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST @@ -264,7 +265,7 @@ def resolve_orchestration_model(settings, *, user_id, seeds=None, planner=False, connection = endpoint.get('connection') or {} address = _text(connection.get('endpoint')) api_version = _text(connection.get('openai_api_version') or connection.get('api_version')) - protocol = infer_model_endpoint_protocol(provider, address, deployment) + protocol = infer_model_endpoint_protocol(provider, address, deployment, get_model_endpoint_api_type(endpoint)) if not address or (protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version): raise OrchestrationModelError() answer_selection = { diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 916a7bce4..a69bdae96 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -36,6 +36,12 @@ normalize_model_endpoint_identity_header_override, normalize_model_endpoint_identity_header_value_type, ) +from functions_model_endpoint_providers import ( + get_model_endpoint_provider, + normalize_api_type_value, + normalize_custom_endpoint_auth_type, + normalize_custom_endpoint_url_mode, +) from functions_mcp_server_config import INBOUND_MCP_SETTINGS_DEFAULTS, normalize_inbound_mcp_settings from functions_rate_limit import ( RATE_LIMIT_MESSAGE_DEFAULT, @@ -1428,6 +1434,9 @@ def get_settings(use_cosmos=False, include_source=False): }, 'enable_multi_model_endpoints': False, 'model_endpoints': [], + 'allow_private_custom_model_endpoints': False, + 'allow_insecure_custom_model_endpoints': False, + 'custom_model_endpoint_ca_bundle_path': '', 'model_endpoint_identity_header_enabled': False, 'model_endpoint_identity_header_name': DEFAULT_MODEL_ENDPOINT_IDENTITY_HEADER_NAME, 'model_endpoint_identity_header_value_type': DEFAULT_MODEL_ENDPOINT_IDENTITY_HEADER_VALUE_TYPE, @@ -2536,6 +2545,8 @@ def normalize_model_endpoint_auth_for_environment(endpoint_copy): """Normalize endpoint auth cloud fields that are owned by app environment.""" if not isinstance(endpoint_copy, dict): return False + if str(endpoint_copy.get("provider") or "").strip().lower() == "custom": + return False changed = False auth = endpoint_copy.get("auth") @@ -2624,9 +2635,36 @@ def normalize_model_endpoints(endpoints): endpoint_copy = json.loads(json.dumps(endpoint)) endpoint_copy.pop("has_api_key", None) endpoint_copy.pop("has_client_secret", None) + endpoint_copy.pop("has_bearer_token", None) + provider = str(endpoint_copy.get("provider") or "aoai").strip().lower() + if endpoint_copy.get("provider") != provider: + endpoint_copy["provider"] = provider + changed = True connection = endpoint_copy.get("connection") or {} if not isinstance(connection, dict): raise AIConnectionError("Connection configuration must be an object.") + if provider == "custom": + api_type = normalize_api_type_value(endpoint_copy.get("api_type")) + if endpoint_copy.get("api_type") != api_type: + endpoint_copy["api_type"] = api_type + changed = True + url_mode = normalize_custom_endpoint_url_mode(connection.get("url_mode")) + if connection.get("url_mode") != url_mode: + connection["url_mode"] = url_mode + changed = True + descriptor = get_model_endpoint_provider(api_type) + if descriptor and descriptor.default_version and not connection.get(descriptor.version_field): + connection[descriptor.version_field] = descriptor.default_version + changed = True + auth = endpoint_copy.get("auth") or {} + if not isinstance(auth, dict): + raise AIConnectionError("Connection authentication must be an object.") + auth_type = normalize_custom_endpoint_auth_type(auth.get("type") or "api_key") + if auth_type and auth.get("type") != auth_type: + auth["type"] = auth_type + changed = True + endpoint_copy["auth"] = auth + endpoint_copy["connection"] = connection operation_settings = connection.get("operation_settings") if operation_settings is not None: if not isinstance(operation_settings, dict): @@ -2715,7 +2753,7 @@ def normalize_model_endpoints(endpoints): def is_frontend_visible_model_endpoint_provider(provider): """Return whether the provider should be exposed in user-facing endpoint UIs.""" normalized_provider = (provider or "aoai").lower() - return normalized_provider in {"aoai", "aifoundry", "new_foundry"} + return normalized_provider in {"aoai", "aifoundry", "new_foundry", "custom"} def merge_model_endpoint_auth(existing_auth, incoming_auth): @@ -2727,6 +2765,14 @@ def merge_model_endpoint_auth(existing_auth, incoming_auth): merged = dict(existing_auth) for key, value in incoming_auth.items(): + if key in ("api_key", "client_secret", "bearer_token") and ( + value in (None, "") or is_admin_settings_redacted_secret(value) + or (value == "Stored_In_KeyVault" and existing_auth.get(key)) + ): + continue + if key in ("api_key_header", "api_key_prefix", "scope") and value == "": + merged[key] = value + continue if value in (None, ""): continue merged[key] = value @@ -2789,7 +2835,7 @@ def merge_model_endpoints_with_existing(incoming_endpoints, existing_endpoints): return merged -def sanitize_model_endpoints_for_frontend(endpoints): +def sanitize_model_endpoints_for_frontend(endpoints, *, include_connection_details=True): """Return model endpoint configs with secrets stripped for frontend use.""" normalized, _ = normalize_model_endpoints(endpoints) if not isinstance(normalized, list): @@ -2805,16 +2851,27 @@ def sanitize_model_endpoints_for_frontend(endpoints): auth = endpoint_copy.get("auth") or {} has_api_key = bool(auth.get("api_key")) has_client_secret = bool(auth.get("client_secret")) + has_bearer_token = bool(auth.get("bearer_token")) auth.pop("api_key", None) auth.pop("client_secret", None) + auth.pop("bearer_token", None) + if endpoint_copy.get("provider") == "custom": + public_auth_fields = { + "type", "client_id", "token_url", "scope", "api_key_header", "api_key_prefix", + } + auth = {key: value for key, value in auth.items() if key in public_auth_fields} endpoint_copy["auth"] = auth endpoint_copy["has_api_key"] = has_api_key endpoint_copy["has_client_secret"] = has_client_secret + endpoint_copy["has_bearer_token"] = has_bearer_token for model in endpoint_copy.get("models") or []: if isinstance(model, dict): model["capability_status"] = describe_model_capabilities( - model, endpoint_copy.get("provider") + model, endpoint_copy.get("provider"), endpoint=endpoint, ) + if not include_connection_details: + for field in ("auth", "connection", "management", "identity_header"): + endpoint_copy.pop(field, None) sanitized.append(endpoint_copy) return sanitized @@ -3390,6 +3447,8 @@ def sanitize_settings_for_user(full_settings: dict) -> dict: sanitized = {} for k, v in full_settings.items(): + if k in ('custom_model_endpoint_ca_bundle_path', 'client_cert_path', 'client_key_path', 'bearer_token', 'token_url'): + continue if k == 'support_feedback_recipient_email': continue if k == 'agents_page_promoted_popular_agents': @@ -3399,7 +3458,7 @@ def sanitize_settings_for_user(full_settings: dict) -> dict: if any(term in k.lower() for term in sensitive_terms): continue if k in ('model_endpoints', 'personal_model_endpoints') and isinstance(v, list): - sanitized[k] = sanitize_model_endpoints_for_frontend(v) + sanitized[k] = sanitize_model_endpoints_for_frontend(v, include_connection_details=False) continue if isinstance(v, dict): sanitized[k] = sanitize_settings_for_user(v) @@ -3449,7 +3508,7 @@ def sanitize_settings_for_logging(full_settings: dict) -> dict: return full_settings sanitized = {} - sensitive_key_terms = ["key", "base64", "image", "storage_account_url", "_secret"] + sensitive_key_terms = ["key", "base64", "image", "storage_account_url", "_secret", "bearer_token", "access_token"] for k, v in full_settings.items(): # Skip keys with sensitive terms diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index b2a6ac4d3..1883f16e0 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -162,6 +162,7 @@ MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, ) from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_types import resolve_model_endpoint_request_model from functions_model_endpoint_runtime import ( build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, @@ -6027,6 +6028,7 @@ def _build_multi_endpoint_client(user_id, endpoint_id, model_id, settings, group candidates = [] group_id = str(group_id or '').strip() if group_id and settings.get('allow_group_custom_endpoints', False): + assert_group_role(user_id, group_id, allowed_roles=('Owner', 'Admin', 'DocumentManager', 'User')) group_endpoints, _ = normalize_model_endpoints(get_group_model_endpoints(group_id) or []) for endpoint in group_endpoints: item = dict(endpoint) @@ -6069,13 +6071,12 @@ def _build_multi_endpoint_client(user_id, endpoint_id, model_id, settings, group connection = resolved_endpoint.get('connection', {}) if isinstance(resolved_endpoint, dict) else {} auth = resolved_endpoint.get('auth', {}) if isinstance(resolved_endpoint, dict) else {} provider = str(resolved_endpoint.get('provider') or endpoint_cfg.get('provider') or 'aoai').strip().lower() - deployment_name = ( - model_cfg.get('deploymentName') - or model_cfg.get('deployment') - or model_cfg.get('displayName') - or model_id - ) - api_version = connection.get('api_version') or connection.get('openai_api_version') or settings.get('azure_openai_gpt_api_version') + deployment_name = resolve_model_endpoint_request_model(resolved_endpoint, model_cfg) + if not deployment_name: + raise ValueError('The selected workflow model is missing its request identifier.') + api_version = connection.get('api_version') or connection.get('openai_api_version') + if provider != 'custom': + api_version = api_version or settings.get('azure_openai_gpt_api_version') endpoint = connection.get('endpoint') auth_type = str(auth.get('type') or 'api_key').strip().lower() auth_settings = { diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index 77ce3c6b2..25132bf50 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -3,13 +3,21 @@ import json import asyncio +import re +import ssl from collections.abc import Mapping from types import SimpleNamespace from typing import Any, Dict, Iterable, Iterator, List -from urllib.parse import urlparse +from urllib.parse import quote, urlparse, urlunparse +import anyio +import httpcore +import httpx import requests -from openai import BadRequestError, OpenAI +from openai import ( + APIError, BadRequestError, DEFAULT_CONNECTION_LIMITS, + DefaultAsyncHttpxClient, DefaultHttpxClient, OpenAI, +) from pydantic import Field from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase from semantic_kernel.connectors.ai.function_calling_utils import update_settings_from_function_call_configuration @@ -17,6 +25,7 @@ from semantic_kernel.contents.chat_message_content import ChatMessageContent from semantic_kernel.contents.function_call_content import FunctionCallContent from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.image_content import ImageContent from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent from semantic_kernel.contents.streaming_text_content import StreamingTextContent from semantic_kernel.contents.text_content import TextContent @@ -27,12 +36,27 @@ from functions_debug import debug_print from functions_appinsights import log_event from functions_model_capabilities import resolve_model_reasoning_effort +from functions_model_endpoint_diagnostics import ( + SanitizedModelEndpointError, + build_sanitized_model_endpoint_error, +) +from functions_model_endpoint_providers import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, + MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + URL_POLICY_AS_GIVEN, + get_model_endpoint_provider, + normalize_custom_endpoint_url_mode, +) +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + resolve_custom_model_endpoint_addresses, + validate_custom_model_endpoint_url, +) -MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI = "azure_openai" -MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE = "openai_style" -MODEL_ENDPOINT_PROTOCOL_ANTHROPIC = "anthropic" - ANTHROPIC_MODEL_MARKERS = ("claude",) OPENAI_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") MODEL_CONTEXT_MODE_SYSTEM = "system" @@ -106,6 +130,8 @@ def response_length_parameter(self) -> str: def is_reasoning_parameter_rejection(error: Exception) -> bool: """Recognize only an SDK HTTP 400 rejecting this specific parameter or value.""" + if isinstance(error, SanitizedModelEndpointError): + return getattr(error, "reasoning_parameter_rejected", False) is True if not isinstance(error, BadRequestError) or error.status_code != 400: return False body = error.body @@ -136,7 +162,7 @@ def create_completion_with_reasoning(create_callable, params, model_name, *, on_ on_resolution(dict(resolution)) try: return create_callable(**parameters), resolution - except BadRequestError as error: + except (BadRequestError, SanitizedModelEndpointError) as error: if "reasoning_effort" not in parameters or not is_reasoning_parameter_rejection(error): raise parameters.pop("reasoning_effort") @@ -195,9 +221,14 @@ def endpoint_uses_openai_style_protocol(endpoint: Any) -> bool: ) -def infer_model_endpoint_protocol(provider: Any, endpoint: Any, deployment_name: Any = "") -> str: +def infer_model_endpoint_protocol(provider: Any, endpoint: Any, deployment_name: Any = "", api_type: Any = "") -> str: """Infer the runtime protocol from provider, endpoint path, and deployment name.""" normalized_provider = str(provider or "aoai").strip().lower() + if normalized_provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + descriptor = get_model_endpoint_provider(api_type) + if descriptor is None: + raise ModelEndpointValidationError("Custom endpoints require a supported API type.") + return descriptor.protocol endpoint_path = get_endpoint_path(endpoint) if normalized_provider in ("anthropic", "claude"): @@ -237,13 +268,72 @@ def normalize_openai_style_base_url(raw_endpoint: Any) -> str: return endpoint.rstrip("/") + "/openai/v1/" -def normalize_anthropic_messages_url(raw_endpoint: Any) -> str: +CUSTOM_OPENAI_OPERATION_SUFFIXES = ( + "/chat/completions", "/responses", "/models", "/images/generations", "/images/edits", +) +CUSTOM_OPENAI_VERSION_SEGMENT_PATTERN = re.compile(r"^v\d+(?:[a-z][a-z0-9]*)?$", re.IGNORECASE) + + +def normalize_custom_openai_base_url(raw_endpoint: Any) -> str: + """Append v1 only when neither a version nor a full operation defines the base.""" + endpoint = normalize_endpoint_text(raw_endpoint) + if not endpoint: + raise ModelEndpointValidationError("A Custom endpoint URL is required.") + for suffix in CUSTOM_OPENAI_OPERATION_SUFFIXES: + if endpoint.lower().endswith(suffix): + return endpoint[:-len(suffix)].rstrip("/") + "/" + last_segment = urlparse(endpoint).path.rstrip("/").rsplit("/", 1)[-1] + if CUSTOM_OPENAI_VERSION_SEGMENT_PATTERN.fullmatch(last_segment): + return endpoint + "/" + return endpoint + "/v1/" + + +def resolve_custom_openai_base_url(raw_endpoint: Any, api_type: Any = "", url_mode: Any = "") -> str: + """Resolve an OpenAI-compatible base without adding Azure deployment semantics.""" + descriptor = get_model_endpoint_provider(api_type or "openai") + if descriptor is None or descriptor.protocol != MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: + raise ModelEndpointValidationError("This Custom API type does not use an OpenAI-compatible base URL.") + if normalize_custom_endpoint_url_mode(url_mode) == "exact" or descriptor.url_policy == URL_POLICY_AS_GIVEN: + endpoint = normalize_endpoint_text(raw_endpoint) + if not endpoint: + raise ModelEndpointValidationError("A Custom endpoint URL is required.") + return endpoint + "/" + return normalize_custom_openai_base_url(raw_endpoint) + + +def resolve_custom_azure_openai_base_url(raw_endpoint: Any, deployment_name: Any, url_mode: Any = "") -> str: + """Build the dated deployment API base, retaining gateway prefixes.""" + endpoint = normalize_endpoint_text(raw_endpoint) + deployment = str(deployment_name or "").strip() + if not endpoint or not deployment: + raise ModelEndpointValidationError("Custom Azure OpenAI requires an endpoint and deployment name.") + if normalize_custom_endpoint_url_mode(url_mode) == "exact": + return endpoint + "/" + for suffix in CUSTOM_OPENAI_OPERATION_SUFFIXES: + if endpoint.lower().endswith(suffix): + endpoint = endpoint[:-len(suffix)].rstrip("/") + break + deployment_index = endpoint.lower().find("/openai/deployments/") + if deployment_index >= 0: + endpoint = endpoint[:deployment_index] + elif endpoint.lower().endswith("/openai"): + endpoint = endpoint[:-len("/openai")] + return f"{endpoint}/openai/deployments/{quote(deployment, safe='')}/" + + +def normalize_anthropic_messages_url(raw_endpoint: Any, *, direct_custom: bool = False, url_mode: Any = "") -> str: """Normalize a Foundry endpoint to the Anthropic messages URL.""" endpoint = normalize_endpoint_text(raw_endpoint) if not endpoint: raise ValueError("A Foundry endpoint is required for Anthropic inference.") lowered_endpoint = endpoint.lower() + if direct_custom: + if normalize_custom_endpoint_url_mode(url_mode) == "exact" or lowered_endpoint.endswith("/messages"): + return endpoint + if lowered_endpoint.endswith("/v1"): + return endpoint + "/messages" + return endpoint + "/v1/messages" messages_index = lowered_endpoint.find("/anthropic/v1/messages") if messages_index >= 0: return endpoint[: messages_index + len("/anthropic/v1/messages")] @@ -295,36 +385,338 @@ def extract_chat_completion_response_text(response: Any) -> str: return normalize_chat_completion_text(getattr(message, "content", None)) +def _resolve_custom_connection_addresses(host, port, allow_private): + hostname = host.decode("ascii") if isinstance(host, bytes) else str(host) + try: + return resolve_custom_model_endpoint_addresses(hostname, port, allow_private=allow_private) + except ModelEndpointValidationError as exc: + raise httpcore.ConnectError("Custom endpoint connection blocked.") from exc + + +class _PinnedCustomEndpointSyncBackend(httpcore.NetworkBackend): + """Keep DNS validation and TCP dialing bound to the same address set.""" + + def __init__(self, *, allow_private=False): + self._allow_private = allow_private + self._backend = httpcore.SyncBackend() + + def connect_tcp(self, host, port, timeout=None, local_address=None, socket_options=None): + addresses = _resolve_custom_connection_addresses(host, port, self._allow_private) + last_error = None + for address in addresses: + try: + return self._backend.connect_tcp( + address, port, timeout=timeout, local_address=local_address, socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_error = exc + raise last_error or httpcore.ConnectError("Custom endpoint connection failed.") + + def connect_unix_socket(self, path, timeout=None, socket_options=None): + raise httpcore.ConnectError("Custom endpoint UNIX sockets are not supported.") + + def sleep(self, seconds): + self._backend.sleep(seconds) + + +class _PinnedCustomEndpointAsyncBackend(httpcore.AsyncNetworkBackend): + def __init__(self, *, allow_private=False): + self._allow_private = allow_private + self._backend = httpcore.AnyIOBackend() + + async def connect_tcp(self, host, port, timeout=None, local_address=None, socket_options=None): + addresses = await anyio.to_thread.run_sync( + _resolve_custom_connection_addresses, host, port, self._allow_private, + ) + last_error = None + for address in addresses: + try: + return await self._backend.connect_tcp( + address, port, timeout=timeout, local_address=local_address, socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_error = exc + raise last_error or httpcore.ConnectError("Custom endpoint connection failed.") + + async def connect_unix_socket(self, path, timeout=None, socket_options=None): + raise httpcore.ConnectError("Custom endpoint UNIX sockets are not supported.") + + async def sleep(self, seconds): + await self._backend.sleep(seconds) + + +def build_custom_endpoint_ssl_context(ca_bundle_path: Any = "", client_cert: Any = None): + """Trust explicit CA roots, not ambient proxy/SSL environment configuration.""" + try: + context = ( + ssl.create_default_context(cafile=str(ca_bundle_path)) + if ca_bundle_path else httpx.create_ssl_context(verify=True, trust_env=False) + ) + if client_cert: + if isinstance(client_cert, (tuple, list)): + context.load_cert_chain(*client_cert) + else: + context.load_cert_chain(client_cert) + return context + except (OSError, ssl.SSLError, ValueError) as exc: + raise ModelEndpointValidationError("The Custom endpoint CA bundle or client certificate could not be loaded.") from exc + + +def _validate_custom_transport_request(request, *, allow_private, allow_insecure): + # SDK operation queries are legitimate; credentials in configured URLs are not. + parsed = urlparse(str(request.url)) + address = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", "")) + validate_custom_model_endpoint_url( + address, allow_private=allow_private, allow_insecure=allow_insecure, + ) + expected_host = request.url.netloc.decode("ascii").lower() + if request.headers.get("host", expected_host).lower() != expected_host: + raise ModelEndpointValidationError("Custom endpoint Host overrides are not supported.") + + +class _PinnedCustomEndpointHTTPTransport(httpx.HTTPTransport): + def __init__(self, *, allow_private=False, allow_insecure=False, ca_bundle_path="", client_cert=None): + self._allow_private = allow_private + self._allow_insecure = allow_insecure + self._pool = httpcore.ConnectionPool( + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path, client_cert), + max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, + max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, + keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, + network_backend=_PinnedCustomEndpointSyncBackend(allow_private=allow_private), + ) + + def handle_request(self, request): + _validate_custom_transport_request( + request, allow_private=self._allow_private, allow_insecure=self._allow_insecure, + ) + response = super().handle_request(request) + if 300 <= response.status_code < 400: + response.close() + raise ModelEndpointValidationError("Custom endpoint redirects are not allowed.") + return response + + +class _PinnedCustomEndpointAsyncHTTPTransport(httpx.AsyncHTTPTransport): + def __init__(self, *, allow_private=False, allow_insecure=False, ca_bundle_path="", client_cert=None): + self._allow_private = allow_private + self._allow_insecure = allow_insecure + self._pool = httpcore.AsyncConnectionPool( + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path, client_cert), + max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, + max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, + keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, + network_backend=_PinnedCustomEndpointAsyncBackend(allow_private=allow_private), + ) + + async def handle_async_request(self, request): + await anyio.to_thread.run_sync( + lambda: _validate_custom_transport_request( + request, allow_private=self._allow_private, allow_insecure=self._allow_insecure, + ) + ) + response = await super().handle_async_request(request) + if 300 <= response.status_code < 400: + await response.aclose() + raise ModelEndpointValidationError("Custom endpoint redirects are not allowed.") + return response + + +def build_custom_openai_sync_http_client( + *, allow_private=False, allow_insecure=False, ca_bundle_path="", client_cert=None, +): + """Return the mandatory DNS-pinned HTTP client for ALL Custom SDK operations.""" + return DefaultHttpxClient( + transport=_PinnedCustomEndpointHTTPTransport( + allow_private=allow_private, allow_insecure=allow_insecure, + ca_bundle_path=ca_bundle_path, client_cert=client_cert, + ), + follow_redirects=False, trust_env=False, + ) + + +def build_custom_openai_async_http_client( + *, allow_private=False, allow_insecure=False, ca_bundle_path="", client_cert=None, +): + return DefaultAsyncHttpxClient( + transport=_PinnedCustomEndpointAsyncHTTPTransport( + allow_private=allow_private, allow_insecure=allow_insecure, + ca_bundle_path=ca_bundle_path, client_cert=client_cert, + ), + follow_redirects=False, trust_env=False, + ) + + +CUSTOM_ENDPOINT_REQUEST_ERRORS = (APIError, httpx.HTTPError, ModelEndpointValidationError) + + +def _sanitized_custom_request_error(exc, *, api_type="", request_url="", streaming=False): + error = build_sanitized_model_endpoint_error( + "Custom model stream failed." if streaming else "Custom model request failed.", + exc, api_type=api_type, request_url=request_url, + status_code=getattr(exc, "status_code", None), + ) + error.reasoning_parameter_rejected = is_reasoning_parameter_rejection(exc) + return error + + +class _SanitizedSyncIterator: + def __init__(self, source, *, api_type="", request_url=""): + self._source = source + self._iterator = iter(source) + self._api_type = api_type + self._request_url = request_url + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self._iterator) + except (*CUSTOM_ENDPOINT_REQUEST_ERRORS, ValueError) as exc: + self.close() + raise _sanitized_custom_request_error( + exc, api_type=self._api_type, request_url=self._request_url, streaming=True, + ) from exc + + def close(self): + close = getattr(self._source, "close", None) + if close: + close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def __getattr__(self, name): + return getattr(self._source, name) + + +class _SanitizedAsyncIterator: + def __init__(self, source, *, api_type="", request_url=""): + self._source = source + self._iterator = source.__aiter__() + self._api_type = api_type + self._request_url = request_url + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return await self._iterator.__anext__() + except (*CUSTOM_ENDPOINT_REQUEST_ERRORS, ValueError) as exc: + await self.close() + raise _sanitized_custom_request_error( + exc, api_type=self._api_type, request_url=self._request_url, streaming=True, + ) from exc + + async def close(self): + await self._source.close() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.close() + + def __getattr__(self, name): + return getattr(self._source, name) + + +def sanitize_custom_async_openai_client(client, *, api_type="", request_url=""): + """Keep SDK/Semantic Kernel errors safe, including failures after streaming begins.""" + original_create = client.chat.completions.create + descriptor = get_model_endpoint_provider(api_type) + + async def create(**kwargs): + if not descriptor or not descriptor.supports_stream_options: + kwargs.pop("stream_options", None) + try: + response = await original_create(**kwargs) + except CUSTOM_ENDPOINT_REQUEST_ERRORS as exc: + raise _sanitized_custom_request_error(exc, api_type=api_type, request_url=request_url) from exc + if kwargs.get("stream"): + return _SanitizedAsyncIterator(response, api_type=api_type, request_url=request_url) + return response + + client.chat.completions.create = create + return client + + def build_openai_style_chat_client( token_or_key: str, base_url: str, api_version: Any = "", default_headers: Dict[str, str] | None = None, + *, + direct_custom: bool = False, + api_type: Any = "", + url_mode: Any = "", + allow_private_custom_endpoints: bool = False, + allow_insecure_custom_endpoints: bool = False, + ca_bundle_path: Any = "", + client_cert: Any = None, ): """Build an OpenAI-compatible chat client for Foundry data-plane endpoints.""" request_api_version = resolve_openai_style_request_api_version(api_version) client_kwargs: Dict[str, Any] = { "api_key": token_or_key, - "base_url": normalize_openai_style_base_url(base_url), + "base_url": ( + resolve_custom_openai_base_url(base_url, api_type, url_mode) + if direct_custom else normalize_openai_style_base_url(base_url) + ), } + if direct_custom: + client_kwargs["http_client"] = build_custom_openai_sync_http_client( + allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, + ca_bundle_path=ca_bundle_path, client_cert=client_cert, + ) if default_headers: client_kwargs["default_headers"] = default_headers if request_api_version: client_kwargs["default_query"] = {"api-version": request_api_version} - return OpenAIStyleChatCompletionClient(OpenAI(**client_kwargs)) + return OpenAIStyleChatCompletionClient( + OpenAI(**client_kwargs), sanitize_errors=direct_custom, + api_type=api_type, request_url=client_kwargs["base_url"], + ) class OpenAIStyleChatCompletionClient: """Small wrapper that makes OpenAI-compatible Foundry calls tolerant of Azure-only options.""" - def __init__(self, client: OpenAI): + def __init__(self, client: OpenAI, *, sanitize_errors=False, api_type="", request_url=""): self._client = client + self._sanitize_errors = sanitize_errors + self._api_type = api_type + self._request_url = request_url self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) def create(self, **kwargs: Any): request_kwargs = dict(kwargs) - request_kwargs.pop("stream_options", None) - return self._client.chat.completions.create(**request_kwargs) + descriptor = get_model_endpoint_provider(self._api_type) if self._sanitize_errors else None + if not descriptor or not descriptor.supports_stream_options: + request_kwargs.pop("stream_options", None) + try: + response = self._client.chat.completions.create(**request_kwargs) + except CUSTOM_ENDPOINT_REQUEST_ERRORS as exc: + if not self._sanitize_errors: + raise + raise _sanitized_custom_request_error( + exc, api_type=self._api_type, request_url=self._request_url, + ) from exc + if self._sanitize_errors and request_kwargs.get("stream"): + return _SanitizedSyncIterator(response, api_type=self._api_type, request_url=self._request_url) + return response + + def close(self): + self._client.close() + + def __getattr__(self, name): + return getattr(self._client, name) def build_anthropic_chat_client( @@ -604,9 +996,13 @@ def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: try: event_payload = json.loads(event_data) except json.JSONDecodeError: + if getattr(self, "direct_custom", False): + raise debug_print(f"[MODEL_ENDPOINT] Ignoring invalid Anthropic stream payload: {event_data[:200]}") continue + if getattr(self, "direct_custom", False) and not isinstance(event_payload, dict): + raise ValueError("Invalid Custom Anthropic stream event.") event_type = event_payload.get("type") if event_type == "error": error_payload = event_payload.get("error") @@ -663,6 +1059,109 @@ def _raise_response_error(self, response: requests.Response) -> None: ) +class CustomAnthropicChatCompletionClient(AnthropicChatCompletionClient): + """Anthropic's native HTTP surface using the same pinned transport as Custom OpenAI.""" + + def __init__( + self, *, endpoint, auth_headers, anthropic_version=DEFAULT_ANTHROPIC_VERSION, + url_mode="auto", allow_private=False, allow_insecure=False, ca_bundle_path="", + client_cert=None, timeout=90, + ): + self.endpoint = normalize_anthropic_messages_url(endpoint, direct_custom=True, url_mode=url_mode) + self.direct_custom = True + self.extra_headers = dict(auth_headers) + self.anthropic_version = anthropic_version + self.timeout = timeout + self._transport_options = { + "allow_private": allow_private, "allow_insecure": allow_insecure, + "ca_bundle_path": ca_bundle_path, "client_cert": client_cert, + } + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) + + def _build_headers(self, *, stream=False): + return { + "Content-Type": "application/json", + "Accept": "text/event-stream" if stream else "application/json", + "anthropic-version": self.anthropic_version, + **self.extra_headers, + } + + def create(self, **kwargs): + payload = self._build_payload(kwargs) + stream = bool(kwargs.get("stream")) + client = build_custom_openai_sync_http_client(**self._transport_options) + response = None + try: + request = client.build_request( + "POST", self.endpoint, headers=self._build_headers(stream=stream), + json=payload, timeout=httpx.Timeout(self.timeout, connect=30), + ) + response = client.send(request, stream=stream, follow_redirects=False) + response.raise_for_status() + if stream: + return self._iter_custom_response(response, client) + result = response.json() + if not isinstance(result, dict) or "error" in result or not isinstance(result.get("content"), list): + raise ValueError("Invalid Anthropic response shape.") + return self._build_completion_response(result) + except (httpx.HTTPError, ValueError) as exc: + if response is not None: + response.close() + client.close() + raise _sanitized_custom_request_error( + exc, api_type="anthropic", request_url=self.endpoint, + ) from exc + finally: + if not stream: + if response is not None: + response.close() + client.close() + + def _iter_custom_response(self, response, client): + adapter = SimpleNamespace( + iter_lines=lambda decode_unicode=True: response.iter_lines(), + close=response.close, + ) + try: + yield from self._iter_stream_chunks(adapter) + except (httpx.HTTPError, ValueError, RuntimeError) as exc: + raise _sanitized_custom_request_error( + exc, api_type="anthropic", request_url=self.endpoint, streaming=True, + ) from exc + finally: + response.close() + client.close() + + def _normalize_content(self, content): + if not isinstance(content, list): + return super()._normalize_content(content) + converted = [] + for item in content: + if not isinstance(item, dict) or item.get("type") != "image_url": + converted.append(item) + continue + image = item.get("image_url") + url = image.get("url") if isinstance(image, dict) else image + url = str(url or "") + match = re.fullmatch(r"data:(image/(?:png|jpeg|gif|webp));base64,([A-Za-z0-9+/=\s]+)", url) + if match: + converted.append({ + "type": "image", + "source": {"type": "base64", "media_type": match[1], "data": match[2]}, + }) + elif url.startswith("https://"): + converted.append({"type": "image", "source": {"type": "url", "url": url}}) + else: + raise ModelEndpointValidationError("Custom Anthropic image inputs require supported image data or an HTTPS URL.") + blocks = [] + for item in converted: + if isinstance(item, str): + blocks.append({"type": "text", "text": item}) + elif isinstance(item, dict) and item.get("type") in {"text", "image", "tool_use", "tool_result"}: + blocks.append(item) + return blocks + + class AnthropicSemanticKernelChatCompletion(ChatCompletionClientBase): """Semantic Kernel chat service for Anthropic-compatible model endpoints.""" @@ -673,6 +1172,7 @@ class AnthropicSemanticKernelChatCompletion(ChatCompletionClientBase): bearer_token: str = "" extra_headers: Dict[str, str] = Field(default_factory=dict) timeout: int = 90 + custom_client: Any = Field(default=None, exclude=True) prompt_execution_settings: OpenAIChatPromptExecutionSettings | None = Field(default=None) def __init__( @@ -685,6 +1185,7 @@ def __init__( bearer_token: str = "", extra_headers: Dict[str, str] | None = None, timeout: int = 90, + custom_client: Any = None, ): super().__init__( ai_model_id=deployment_name, @@ -694,6 +1195,7 @@ def __init__( bearer_token=bearer_token, extra_headers=extra_headers or {}, timeout=timeout, + custom_client=custom_client, ) def get_prompt_execution_settings_class(self): @@ -855,6 +1357,8 @@ def _to_streaming_chat_message_content( ) def _build_client(self): + if self.custom_client is not None: + return self.custom_client return build_anthropic_chat_client( endpoint=self.endpoint, api_key=self.api_key, @@ -941,6 +1445,10 @@ def _convert_message_content(self, message: ChatMessageContent) -> str | List[Di }) elif isinstance(item, TextContent): text_parts.append(item.text) + elif isinstance(item, ImageContent) and self.custom_client is not None: + image_url = getattr(item, "data_uri", None) or getattr(item, "uri", None) + if image_url: + blocks.append({"type": "image_url", "image_url": {"url": str(image_url)}}) if blocks: if text_parts: diff --git a/application/single_app/route_backend_agents.py b/application/single_app/route_backend_agents.py index b2dff6745..019344c2c 100644 --- a/application/single_app/route_backend_agents.py +++ b/application/single_app/route_backend_agents.py @@ -602,6 +602,8 @@ def _format_model_provider_label(provider): return 'Foundry (classic)' if normalized_provider == 'new_foundry': return 'New Foundry' + if normalized_provider == 'custom': + return 'Custom' return 'Azure OpenAI' @@ -2530,6 +2532,7 @@ def build_combined_model_endpoints(settings, user_id=None, group_id=None): return sanitize_model_endpoints_for_frontend( filter_model_endpoints_by_capability(endpoints, preserve_empty=True), + include_connection_details=False, ) diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index be7ed76b0..feaa8d543 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -64,6 +64,7 @@ build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, ) +from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model from functions_mixed_source_orchestration import ( MixedSourceCancellationError, MixedSourceFinalizationError, @@ -13778,7 +13779,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ endpoint_cfg = next((endpoint for endpoint in endpoint_candidates if endpoint.get('id') == requested_endpoint_id), None) if not endpoint_cfg: - if selection_source == 'request': + if selection_source == 'request' or requested_provider == 'custom': raise LookupError('Selected model endpoint could not be found.') debug_print( f"[STREAMING][Model Resolution] Default model endpoint_id={requested_endpoint_id} was not found. Falling back to legacy streaming config." @@ -13786,7 +13787,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ return None if not endpoint_cfg.get('enabled', True): - if selection_source == 'request': + if selection_source == 'request' or endpoint_cfg.get('provider') == 'custom': raise ValueError('Selected model endpoint is disabled.') debug_print( f"[STREAMING][Model Resolution] Default model endpoint_id={requested_endpoint_id} is disabled. Falling back to legacy streaming config." @@ -13811,13 +13812,13 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ model_cfg = next( ( model for model in models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == requested_deployment + if resolve_model_endpoint_request_model(resolved_endpoint_cfg, model) == requested_deployment ), None, ) if not model_cfg: - if selection_source == 'request': + if selection_source == 'request' or resolved_endpoint_cfg.get('provider') == 'custom': raise LookupError('Selected model could not be found on the configured endpoint.') debug_print( f"[STREAMING][Model Resolution] Default model_id={requested_model_id} was not found on endpoint_id={requested_endpoint_id}. Falling back to legacy streaming config." @@ -13825,7 +13826,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ return None if not model_cfg.get('enabled', True): - if selection_source == 'request': + if selection_source == 'request' or resolved_endpoint_cfg.get('provider') == 'custom': raise ValueError('Selected model is disabled.') debug_print( f"[STREAMING][Model Resolution] Default model_id={requested_model_id} is disabled. Falling back to legacy streaming config." @@ -13843,10 +13844,12 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ connection = resolved_endpoint_cfg.get('connection', {}) or {} auth_settings = resolved_endpoint_cfg.get('auth', {}) or {} - deployment = str(model_cfg.get('deploymentName') or model_cfg.get('deployment') or '').strip() + deployment = resolve_model_endpoint_request_model(resolved_endpoint_cfg, model_cfg) endpoint = str(connection.get('endpoint') or '').strip() api_version = str(connection.get('openai_api_version') or connection.get('api_version') or '').strip() - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment) + runtime_protocol = infer_model_endpoint_protocol( + provider, endpoint, deployment, get_model_endpoint_api_type(resolved_endpoint_cfg), + ) model_icon = _normalize_model_icon_payload(model_cfg.get('icon')) model_response_length = normalize_model_response_length_from_model(model_cfg) model_behavior_name = _build_model_endpoint_behavior_name(model_cfg, deployment) @@ -13865,7 +13868,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version ) if missing_required_config: - if selection_source == 'request': + if selection_source == 'request' or provider == 'custom': if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI: raise ValueError('Selected model endpoint is missing endpoint, API version, or deployment configuration.') raise ValueError('Selected model endpoint is missing endpoint or deployment configuration.') @@ -25224,9 +25227,10 @@ def add_message_image_revision_api(message_id): conversation_id=conversation_id, complete_content=loaded['content'], origin=data.get('origin') or IMAGE_ORIGIN_AI, + operation=data.get('operation', ''), instruction=data.get('instruction') or '', prompt=data.get('prompt') or '', - mask_data_url=data.get('mask') or '', + mask_data_url=data.get('mask', ''), mask_regions=data.get('mask_regions') or 0, size=data.get('size') or '', quality=data.get('quality') or '', diff --git a/application/single_app/route_backend_collaboration.py b/application/single_app/route_backend_collaboration.py index 2e33e52e1..b16e17544 100644 --- a/application/single_app/route_backend_collaboration.py +++ b/application/single_app/route_backend_collaboration.py @@ -1963,9 +1963,10 @@ def add_collaboration_image_revision_api(conversation_id, message_id): conversation_id=loaded['source_conversation_id'], complete_content=loaded['content'], origin=data.get('origin') or IMAGE_ORIGIN_AI, + operation=data.get('operation', ''), instruction=data.get('instruction') or '', prompt=data.get('prompt') or '', - mask_data_url=data.get('mask') or '', + mask_data_url=data.get('mask', ''), mask_regions=data.get('mask_regions') or 0, size=data.get('size') or '', quality=data.get('quality') or '', diff --git a/application/single_app/route_backend_conversation_export.py b/application/single_app/route_backend_conversation_export.py index 65e9ba49b..79af2cab5 100644 --- a/application/single_app/route_backend_conversation_export.py +++ b/application/single_app/route_backend_conversation_export.py @@ -63,6 +63,8 @@ is_external_image_url, ) from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_runtime import build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model from functions_message_artifacts import ( build_message_artifact_payload_map, hydrate_agent_citations_from_artifacts, @@ -1455,6 +1457,12 @@ def _build_summary_model_endpoint_client( ) auth_type = _normalize_summary_model_value(auth_settings.get('type') or 'managed_identity').lower() normalized_provider = _normalize_summary_model_value(provider or 'aoai').lower() + if normalized_provider == 'custom': + client, _ = build_model_endpoint_sync_chat_client( + auth_settings, normalized_provider, endpoint, api_version, deployment_name, + settings=settings, endpoint_config=endpoint_config, identity_context=identity_context, + ) + return client runtime_protocol = infer_model_endpoint_protocol(normalized_provider, endpoint, deployment_name) if auth_type in ('api_key', 'key'): @@ -1568,12 +1576,12 @@ def _resolve_summary_multi_endpoint_client( require_model_capability(model_cfg, provider=provider) connection = resolved_endpoint_cfg.get('connection', {}) or {} auth_settings = resolved_endpoint_cfg.get('auth', {}) or {} - deployment = _normalize_summary_model_value( - model_cfg.get('deploymentName') or model_cfg.get('deployment') or model_cfg.get('id') - ) + deployment = resolve_model_endpoint_request_model(resolved_endpoint_cfg, model_cfg) endpoint = _normalize_summary_model_value(connection.get('endpoint')) api_version = _normalize_summary_model_value(connection.get('openai_api_version') or connection.get('api_version')) - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment) + runtime_protocol = infer_model_endpoint_protocol( + provider, endpoint, deployment, get_model_endpoint_api_type(resolved_endpoint_cfg), + ) missing_required_config = not endpoint or not deployment or ( runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version diff --git a/application/single_app/route_backend_models.py b/application/single_app/route_backend_models.py index 23510d00b..5d2fa99c2 100644 --- a/application/single_app/route_backend_models.py +++ b/application/single_app/route_backend_models.py @@ -13,6 +13,10 @@ from functions_image_api_route import is_image_capable_model_name from functions_ai_connections import describe_model_capabilities, supports_model_capability from functions_model_capabilities import resolve_model_vision_support +from functions_model_endpoint_diagnostics import SanitizedModelEndpointError +from functions_model_endpoint_runtime import build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model +from functions_model_endpoint_validation import ModelEndpointValidationError, validate_custom_model_endpoint from model_endpoint_clients import ( MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, @@ -272,7 +276,13 @@ def build_legacy_aoai_discovery_auth_settings(): "client_secret": MICROSOFT_PROVIDER_AUTHENTICATION_SECRET, } - def build_inference_client(endpoint, api_version, auth_settings, provider="aoai", deployment_name=""): + def build_inference_client(endpoint, api_version, auth_settings, provider="aoai", deployment_name="", endpoint_config=None): + if provider == "custom": + client, _ = build_model_endpoint_sync_chat_client( + auth_settings, provider, endpoint, api_version, deployment_name, + settings=get_settings(), endpoint_config=endpoint_config, + ) + return client auth_type = (auth_settings.get("type") or "managed_identity").lower() runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) if auth_type == "api_key": @@ -369,6 +379,14 @@ def handle_fetch_model_list(scope="global"): auth_settings = data.get("auth") or {} management = data.get("management") or {} auth_type = (auth_settings.get("type") or "managed_identity").lower() + if provider == "custom": + if not get_model_endpoint_api_type(data): + return jsonify({"error": "Custom endpoint API type is not supported."}), 400 + return jsonify({ + "models": [], + "manual_models_required": True, + "message": "Custom connections use manually configured model names or deployments; management discovery is not available.", + }) log_models_debug( "Fetch model list request" f" provider={provider} auth_type={auth_type}" @@ -396,7 +414,7 @@ def handle_fetch_model_list(scope="global"): "modelName": model_name or "" }) for model in mapped: - model["capability_status"] = describe_model_capabilities(model, provider) + model["capability_status"] = describe_model_capabilities(model, provider, endpoint=data) return jsonify({"models": mapped}) if provider == "aoai": @@ -433,7 +451,7 @@ def handle_fetch_model_list(scope="global"): "modelName": model_name }) for model in mapped: - model["capability_status"] = describe_model_capabilities(model, provider) + model["capability_status"] = describe_model_capabilities(model, provider, endpoint=data) return jsonify({"models": mapped}) return jsonify({"error": "Model provider not found."}), 400 @@ -473,8 +491,12 @@ def handle_test_model_connection(scope="global"): endpoint = connection.get("endpoint") or "" api_version = connection.get("openai_api_version") or connection.get("api_version") or "" - deployment_name = model.get("deploymentName") or "" - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) + deployment_name = resolve_model_endpoint_request_model(data, model) + runtime_protocol = infer_model_endpoint_protocol( + provider, endpoint, deployment_name, get_model_endpoint_api_type(data), + ) + if provider == "custom": + validate_custom_model_endpoint({**data, "models": [model]}, get_settings()) auth_type = (auth_settings.get("type") or "managed_identity").lower() log_models_debug( @@ -484,12 +506,12 @@ def handle_test_model_connection(scope="global"): ) if not endpoint or not deployment_name: - return jsonify({"error": "Endpoint and deployment name are required."}), 400 + return jsonify({"error": "Endpoint and request model name or deployment are required."}), 400 if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version: return jsonify({"error": "Endpoint, API version, and deployment name are required."}), 400 - if provider not in ("aoai", "aifoundry", "new_foundry", "anthropic", "claude"): + if provider not in ("aoai", "aifoundry", "new_foundry", "anthropic", "claude", "custom"): return jsonify({"error": "Model provider not found."}), 400 gpt_client = build_inference_client( @@ -498,17 +520,27 @@ def handle_test_model_connection(scope="global"): auth_settings, provider=provider, deployment_name=deployment_name, + endpoint_config={**data, "models": [model]}, ) - response = gpt_client.chat.completions.create( - model=deployment_name, - messages=[{"role": "user", "content": "Testing access."}] - ) + try: + response = gpt_client.chat.completions.create( + model=deployment_name, + messages=[{"role": "user", "content": "Testing access."}] + ) + finally: + close = getattr(gpt_client, "close", None) + if callable(close): + close() - if response: + if getattr(response, "choices", None): return jsonify({"success": True}), 200 return jsonify({"error": "No response returned from model."}), 400 + except ModelEndpointValidationError as exc: + return jsonify({"error": exc.public_message}), 400 + except SanitizedModelEndpointError as exc: + return jsonify({"error": exc.public_message}), 400 except LookupError as exc: log_event( "[MODELS] Test model request blocked because the model endpoint was not found", @@ -692,11 +724,9 @@ def get_image_models(): """ Fetch available image-capable Azure OpenAI deployments using Azure Management API. - Two kinds qualify. A gpt-image or DALL-E deployment serves /images/generations - directly. A chat model serves no image endpoint at all, but can still produce an - image through the Responses API's image_generation tool, and where gpt-image is - unavailable it is the only deployment that can. Both are listed; which route a - selection takes is decided from its model name at call time. + This legacy Azure discovery path offers compatible dedicated image deployments. + GPT chat deployments are excluded by the standalone-only Azure image policy. + Direct OpenAI image tools and other Foundry image APIs use shared connections. """ settings = get_settings() @@ -760,6 +790,13 @@ def test_model_inference_connection(): ) try: + if provider == "custom": + validate_custom_model_endpoint(data, get_settings()) + return jsonify({ + "success": True, + "validation_only": True, + "message": "Custom configuration is valid. Use Test Model to verify authentication and inference.", + }) if provider in ("aifoundry", "new_foundry"): endpoint = connection.get("endpoint") api_version = connection.get("project_api_version") or connection.get("api_version") or "v1" @@ -799,6 +836,8 @@ def test_model_inference_connection(): return jsonify({"success": True, "count": count}) return jsonify({"error": "Model provider not found."}), 400 + except ModelEndpointValidationError as exc: + return jsonify({"error": exc.public_message}), 400 except LookupError as e: log_event( "[MODELS] Test connection blocked because the model endpoint was not found", @@ -807,7 +846,7 @@ def test_model_inference_connection(): return build_safe_error_response("The selected model endpoint could not be found.", 404) except PermissionError as e: log_models_exception("Test connection blocked by governance policy", e, level=logging.WARNING) - return build_safe_error_response(str(e), 403) + return build_safe_error_response("The selected model endpoint is not permitted by governance policy.", 403) except ValueError as e: log_models_exception("Test connection validation failed", e, level=logging.WARNING) return build_safe_error_response( diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index e26e36008..96ad33297 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -17,6 +17,7 @@ resolve_model_endpoint_from_context, ) from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_types import resolve_model_endpoint_request_model from functions_activity_logging import ( log_admin_feedback_email_submission, log_general_admin_action, @@ -1438,18 +1439,14 @@ def _test_multimodal_vision_connection(payload): matched_model = next( ( model for model in resolved_models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == model_context['model_deployment'] + if resolve_model_endpoint_request_model(resolved_endpoint, model) == model_context['model_deployment'] ), None, ) if not matched_model: return jsonify({'error': 'Selected vision model could not be resolved from saved settings'}), 400 - vision_model = str( - matched_model.get('deploymentName') - or matched_model.get('deployment') - or model_context['model_deployment'] - ).strip() + vision_model = resolve_model_endpoint_request_model(resolved_endpoint, matched_model) vision_model_name = str(matched_model.get('modelName') or vision_model).strip() connection = resolved_endpoint.get('connection', {}) or {} gpt_client, _ = build_model_endpoint_sync_chat_client( diff --git a/application/single_app/route_backend_v2.py b/application/single_app/route_backend_v2.py index 7a09427dc..296057ed6 100644 --- a/application/single_app/route_backend_v2.py +++ b/application/single_app/route_backend_v2.py @@ -158,6 +158,8 @@ from functions_agent_catalog import build_accessible_agent_catalog from functions_ai_notice import get_ai_notice_config, is_ai_notice_dismissed from functions_model_capabilities import resolve_model_vision_support +from functions_model_endpoint_providers import get_model_endpoint_provider_ui_options +from functions_model_endpoint_validation import ModelEndpointValidationError, validate_custom_model_endpoints from functions_documents import get_audio_runtime_capabilities from config import VERSION from swagger_wrapper import get_auth_security, swagger_route @@ -417,15 +419,18 @@ def _build_capabilities(settings): like a settings key to everything that reads the application's surface, including the documentation inventory. It is reported separately instead. - Computed from the raw settings and reduced to an enum, so no deployment detail beyond the - model's name reaches the browser. + Computed from raw settings, then reduced to safe operation metadata. Connection URLs, + authentication, provider payloads and internal model-routing paths never reach the browser. """ capability = resolve_image_edit_capability(settings) return { "image_edit": { - "mode": capability["mode"], - "model_name": capability["model_name"], - "reason": capability["reason"], + key: capability[key] + for key in ( + "enabled", "mode", "model_name", "reason", "provider_label", "cloud_label", + "availability", "availability_reason", "editing", "masking", + "sizes", "qualities", "backgrounds", + ) }, } @@ -917,6 +922,8 @@ def _persist_global_model_endpoints(normalized, existing): This mirrors what the classic admin form does on submit, so an endpoint saved from either interface ends up stored identically. """ + settings = get_settings() + validate_custom_model_endpoints(normalized, settings) existing_by_id = { endpoint.get("id"): endpoint for endpoint in existing @@ -933,7 +940,6 @@ def _persist_global_model_endpoints(normalized, existing): for endpoint in normalized ] - settings = get_settings() updates = {"model_endpoints": saved_endpoints} multi_endpoint_enabled = bool(settings.get("enable_multi_model_endpoints", False)) @@ -1284,6 +1290,15 @@ def v2_admin_patch_settings(): normalized, errors, warnings = normalize_admin_settings_updates( updates, current_settings ) + for key in ('allow_private_custom_model_endpoints', 'allow_insecure_custom_model_endpoints'): + if key in updates and not isinstance(updates[key], bool): + errors[key] = "Custom network permissions must be true or false." + ca_path_key = 'custom_model_endpoint_ca_bundle_path' + if ca_path_key in updates: + if not isinstance(updates[ca_path_key], str): + errors[ca_path_key] = "The CA bundle must be a deployment-mounted file path." + else: + normalized[ca_path_key] = updates[ca_path_key].strip() if errors: log_event( @@ -1575,6 +1590,12 @@ def v2_admin_list_model_endpoints(): ), "migration": get_settings().get(MIGRATION_NOTICE_KEY), "default_notices": get_settings().get("ai_connection_default_notices", {}), + "custom_api_types": get_model_endpoint_provider_ui_options(), + "custom_network_policy": { + "allow_private_custom_model_endpoints": get_settings().get("allow_private_custom_model_endpoints") is True, + "allow_insecure_custom_model_endpoints": get_settings().get("allow_insecure_custom_model_endpoints") is True, + "custom_model_endpoint_ca_bundle_path": get_settings().get("custom_model_endpoint_ca_bundle_path") or "", + }, } ), 200, @@ -1620,6 +1641,8 @@ def v2_admin_create_model_endpoint(): return _model_endpoint_response(saved, endpoint_id, 201) except AIConnectionError as exc: return jsonify({"error": exc.public_message, "code": exc.code}), 400 + except ModelEndpointValidationError as exc: + return jsonify({"error": exc.public_message, "code": "invalid_custom_endpoint"}), 400 except Exception as exc: log_event( f"[V2_ADMIN_ENDPOINTS] Failed to create model endpoint: {exc}", @@ -1690,6 +1713,8 @@ def v2_admin_update_model_endpoint(endpoint_id): return _model_endpoint_response(saved, current.get("id"), 200) except AIConnectionError as exc: return jsonify({"error": exc.public_message, "code": exc.code}), 400 + except ModelEndpointValidationError as exc: + return jsonify({"error": exc.public_message, "code": "invalid_custom_endpoint"}), 400 except Exception as exc: log_event( f"[V2_ADMIN_ENDPOINTS] Failed to update model endpoint: {exc}", diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 693b19735..c3d060471 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -10,6 +10,8 @@ from functions_keyvault import keyvault_model_endpoint_cleanup_helper, keyvault_model_endpoint_delete_helper, keyvault_model_endpoint_save_helper, redact_model_endpoint_secret_values from functions_settings import * +from functions_model_endpoint_providers import get_model_endpoint_provider_ui_options +from functions_model_endpoint_validation import ModelEndpointValidationError, validate_custom_model_endpoints from functions_ai_connections import ( IMAGE_SELECTION_KEY, filter_model_endpoints_by_capability, @@ -1069,6 +1071,7 @@ def admin_settings(): 'admin_settings.html', app_settings=settings_for_template, settings=settings_for_template, + custom_model_endpoint_api_types=get_model_endpoint_provider_ui_options(), azure_environment=AZURE_ENVIRONMENT, content_understanding_supported=is_content_understanding_supported_environment(), content_understanding_api_version_default=CONTENT_UNDERSTANDING_API_VERSION_DEFAULT, @@ -1702,9 +1705,9 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul else: raise ValueError("Invalid format: model_endpoints must be a list.") except (json.JSONDecodeError, ValueError) as e: - print(f"Error processing model_endpoints_json: {e}") - flash(f"Error processing model endpoints: {e}. Changes for endpoints not saved.", 'danger') - parsed_model_endpoints = settings.get('model_endpoints', []) + log_event("[MODEL_ENDPOINT] Invalid model endpoint settings payload.", level=logging.WARNING) + flash('Model endpoint data is invalid. No settings were saved.', 'danger') + return redirect(url_for('frontend_admin_settings.admin_settings')) existing_multi_endpoints_enabled = settings.get('enable_multi_model_endpoints', False) enable_multi_model_endpoints = coerce_multi_model_endpoint_enablement( @@ -1757,6 +1760,16 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul parsed_model_endpoints = merge_model_endpoints_with_existing(parsed_model_endpoints, existing_model_endpoints) parsed_model_endpoints, _ = normalize_model_endpoints(parsed_model_endpoints) + custom_network_policy = { + 'allow_private_custom_model_endpoints': form_data.get('allow_private_custom_model_endpoints') == 'on', + 'allow_insecure_custom_model_endpoints': form_data.get('allow_insecure_custom_model_endpoints') == 'on', + 'custom_model_endpoint_ca_bundle_path': str(form_data.get('custom_model_endpoint_ca_bundle_path') or '').strip(), + } + try: + validate_custom_model_endpoints(parsed_model_endpoints, {**settings, **custom_network_policy}) + except ModelEndpointValidationError as exc: + flash(exc.public_message, 'danger') + return redirect(url_for('frontend_admin_settings.admin_settings')) existing_endpoints_by_id = { endpoint.get('id'): endpoint @@ -2432,6 +2445,7 @@ def is_valid_url(url): 'gpt_model': gpt_model_obj, 'enable_multi_model_endpoints': enable_multi_model_endpoints, 'model_endpoints': parsed_model_endpoints, + **custom_network_policy, 'model_endpoint_identity_header_enabled': model_endpoint_identity_header_enabled, 'model_endpoint_identity_header_name': model_endpoint_identity_header_name, 'model_endpoint_identity_header_value_type': model_endpoint_identity_header_value_type, diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index b5f6f08a6..a87fae145 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -7,6 +7,7 @@ from functions_settings import * from functions_agent_catalog import build_accessible_agent_catalog from functions_ai_connections import filter_model_endpoints_by_capability +from functions_model_endpoint_types import resolve_model_endpoint_request_model from functions_model_capabilities import REASONING_IDENTIFIER_FIELDS, resolve_model_reasoning_policy from functions_ai_notice import get_ai_notice_config, is_ai_notice_dismissed from functions_collaboration import ( @@ -599,7 +600,7 @@ def append_models(endpoints, scope_type, scope_id=None, scope_name=None): continue model_id = model.get('id') or model.get('deploymentName') or model.get('deployment') or model.get('modelName') or model.get('name') or '' - deployment_name = model.get('deploymentName') or model.get('deployment') or '' + deployment_name = resolve_model_endpoint_request_model(endpoint, model) display_name = model.get('displayName') or model.get('modelName') or deployment_name or model.get('name') or model_id selection_key = f"{scope_type}:{scope_id or ''}:{endpoint_id}:{model_id or deployment_name}" @@ -825,7 +826,7 @@ def chats(): multi_endpoint_models.append({ "id": model.get("id"), "display_name": model.get("displayName") or model.get("deploymentName") or model.get("modelName") or "", - "deployment_name": model.get("deploymentName") or "", + "deployment_name": resolve_model_endpoint_request_model(endpoint, model), "endpoint_id": endpoint.get("id"), "provider": endpoint.get("provider"), "icon": model.get("icon") if isinstance(model.get("icon"), dict) else {} diff --git a/application/single_app/semantic_kernel_loader.py b/application/single_app/semantic_kernel_loader.py index f32a83ef2..9f361e2d2 100644 --- a/application/single_app/semantic_kernel_loader.py +++ b/application/single_app/semantic_kernel_loader.py @@ -44,6 +44,8 @@ ) from functions_appinsights import log_event, get_appinsights_logger from functions_ai_connections import require_model_capability +from functions_model_endpoint_runtime import build_semantic_kernel_chat_service_for_model +from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model from functions_authentication import get_current_user_id_or_none from semantic_kernel_plugins.plugin_health_checker import PluginHealthChecker, PluginErrorRecovery from semantic_kernel_plugins.logged_plugin_loader import create_logged_plugin_loader @@ -168,6 +170,7 @@ def resolve_agent_endpoint_protocol(agent_config): agent_config.get("model_provider") or agent_config.get("provider") or "aoai", agent_config.get("endpoint"), agent_config.get("deployment"), + agent_config.get("api_type"), ) @@ -185,8 +188,25 @@ def resolve_agent_endpoint_token(agent_config): def create_model_endpoint_chat_completion_service(agent_config, service_id): """Create the correct Semantic Kernel chat service for an endpoint-bound agent.""" if not agent_config.get("endpoint") or not agent_config.get("deployment"): + if agent_config.get("model_provider") == "custom": + raise ValueError("The selected Custom agent connection or request model is unavailable.") return None + if agent_config.get("model_provider") == "custom": + endpoint_config = agent_config.get("model_endpoint_config") + if not isinstance(endpoint_config, dict): + raise ValueError("The selected Custom agent connection could not be resolved.") + service, _ = build_semantic_kernel_chat_service_for_model( + agent_config["deployment"], get_settings(), service_id=service_id, + model_context={ + "provider": "custom", "api_type": agent_config.get("api_type"), + "model_id": agent_config.get("model_id"), + "model_deployment": agent_config["deployment"], + }, + resolved_model_endpoint=endpoint_config, + ) + return service + runtime_protocol = resolve_agent_endpoint_protocol(agent_config) token_or_key = resolve_agent_endpoint_token(agent_config) if not token_or_key: @@ -580,7 +600,7 @@ def resolve_multi_endpoint_agent_binding(endpoint_candidates, endpoint_id, model connection = endpoint_cfg.get("connection", {}) or {} auth = endpoint_cfg.get("auth", {}) or {} - deployment = model_cfg.get("deploymentName") or model_cfg.get("deployment") or "" + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) api_version = connection.get("openai_api_version") or connection.get("api_version") endpoint = connection.get("endpoint") return { @@ -590,6 +610,8 @@ def resolve_multi_endpoint_agent_binding(endpoint_candidates, endpoint_id, model "deployment": deployment, "auth": auth, "model": model_cfg, + "api_type": get_model_endpoint_api_type(endpoint_cfg), + "model_endpoint_config": endpoint_cfg, } def resolve_multi_endpoint_agent_config(): @@ -812,7 +834,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): if not per_user_enabled: try: token_provider = None - if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow"): + if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow", "custom"): auth = multi_endpoint_config.get("auth", {}) or {} auth_type = (auth.get("type") or "managed_identity").lower() provider = multi_endpoint_config.get("provider") @@ -820,7 +842,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): deployment = multi_endpoint_config.get("deployment") api_version = multi_endpoint_config.get("api_version") key = auth.get("api_key") or "" - if auth_type != "api_key": + if auth_type != "api_key" and provider != "custom": token_provider = build_token_provider(auth, provider=provider, endpoint=endpoint) return { "endpoint": endpoint, @@ -847,6 +869,8 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): "model_endpoint_id": agent.get("model_endpoint_id", ""), "model_id": agent.get("model_id", ""), "model_provider": provider, + "api_type": multi_endpoint_config.get("api_type"), + "model_endpoint_config": multi_endpoint_config.get("model_endpoint_config"), } if global_apim_enabled: g_apim = get_global_apim() @@ -889,7 +913,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): can_use_agent_endpoints = allow_custom_agent_endpoints user_apim_allowed = user_apim_enabled and can_use_agent_endpoints - if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow"): + if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow", "custom"): auth = multi_endpoint_config.get("auth", {}) or {} auth_type = (auth.get("type") or "managed_identity").lower() provider = multi_endpoint_config.get("provider") @@ -898,7 +922,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): api_version = multi_endpoint_config.get("api_version") key = auth.get("api_key") or "" token_provider = None - if auth_type != "api_key": + if auth_type != "api_key" and provider != "custom": token_provider = build_token_provider(auth, provider=provider, endpoint=endpoint) result = { "endpoint": endpoint, @@ -925,6 +949,8 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): "model_endpoint_id": agent.get("model_endpoint_id", ""), "model_id": agent.get("model_id", ""), "model_provider": provider, + "api_type": multi_endpoint_config.get("api_type"), + "model_endpoint_config": multi_endpoint_config.get("model_endpoint_config"), } return result diff --git a/application/single_app/static/js/admin/admin_model_endpoints.js b/application/single_app/static/js/admin/admin_model_endpoints.js index 30024d542..30d6595e0 100644 --- a/application/single_app/static/js/admin/admin_model_endpoints.js +++ b/application/single_app/static/js/admin/admin_model_endpoints.js @@ -89,6 +89,28 @@ const saveBtn = document.getElementById("model-endpoint-save-btn"); const modelsListEl = document.getElementById("model-endpoint-models-list"); const addModelBtn = document.getElementById("model-endpoint-add-model-btn"); +const customEndpointTemplate = document.getElementById("custom-model-endpoint-fields-template"); +if (customEndpointTemplate?.content && endpointModalEl) { + endpointUrlInput?.closest(".row")?.after(customEndpointTemplate.content.cloneNode(true)); +} +const customApiTypes = JSON.parse(document.getElementById("model-endpoints")?.dataset?.customApiTypes || "[]"); +const customEndpointFields = document.getElementById("custom-model-endpoint-fields"); +const endpointApiTypeSelect = document.getElementById("model-endpoint-api-type"); +const endpointUrlModeSelect = document.getElementById("model-endpoint-url-mode"); +const endpointAnthropicVersionInput = document.getElementById("model-endpoint-anthropic-version"); +const endpointBearerInput = document.getElementById("model-endpoint-bearer-token"); +const endpointTokenUrlInput = document.getElementById("model-endpoint-token-url"); +const endpointOAuthScopeInput = document.getElementById("model-endpoint-oauth-scope"); +const endpointApiKeyHeaderInput = document.getElementById("model-endpoint-api-key-header"); +const endpointApiKeyPrefixInput = document.getElementById("model-endpoint-api-key-prefix"); +const endpointClientCertInput = document.getElementById("model-endpoint-client-cert-path"); +const endpointClientKeyInput = document.getElementById("model-endpoint-client-key-path"); +if (endpointApiTypeSelect && endpointProviderSelect && endpointAuthTypeSelect) { + endpointProviderSelect.add(new Option("Custom", "custom")); + endpointAuthTypeSelect.add(new Option("Bearer Token", "bearer")); + endpointAuthTypeSelect.add(new Option("OAuth2 Client Credentials", "oauth2_client_credentials")); +} + let modelEndpoints = Array.isArray(window.modelEndpoints) ? [...window.modelEndpoints] : []; const savedModelEndpoints = JSON.parse(JSON.stringify(modelEndpoints)); let connectionDraftChanged = false; @@ -153,6 +175,19 @@ function isFoundryProvider(provider) { return provider === "aifoundry" || provider === "new_foundry"; } +function customApiTypeDescriptor(apiType = endpointApiTypeSelect?.value) { + return customApiTypes.find(option => option.value === apiType); +} + +function requestModelName(endpoint, model) { + const usesName = endpoint.provider === "custom" && customApiTypeDescriptor(endpoint.api_type)?.usesModelName; + return String(usesName ? model.modelName || model.name || "" : model.deploymentName || model.deployment || "").trim(); +} + +function modalUsesModelName() { + return endpointProviderSelect?.value === "custom" && customApiTypeDescriptor()?.usesModelName; +} + function endpointIncludesProject(endpoint) { return String(endpoint || "").toLowerCase().includes("/api/projects/"); } @@ -202,7 +237,9 @@ function syncEndpointCopyForProvider() { : "Endpoint Fully Qualified Domain Name (FQDN)"; } if (endpointUrlHelp) { - endpointUrlHelp.textContent = isFoundryProvider(provider) + endpointUrlHelp.textContent = provider === "custom" + ? "Enter the Custom API URL. Private hosts and plaintext HTTP require explicit administrator permission." + : isFoundryProvider(provider) ? "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name." : "For Azure OpenAI, paste the resource endpoint."; } @@ -390,6 +427,7 @@ function markModified() { } function formatProviderLabel(provider) { + if (provider === "custom") return "Custom"; if (provider === "aifoundry") { return "Foundry (classic)"; } @@ -601,7 +639,7 @@ function buildEndpointModelOption(endpoint, model) { option.dataset.endpointId = endpoint.id || ""; option.dataset.modelId = modelId || ""; option.dataset.provider = provider; - option.dataset.deploymentName = model.deploymentName || model.deployment || ""; + option.dataset.deploymentName = requestModelName(endpoint, model); option.disabled = !(endpointEnabled && modelEnabled); option.textContent = `${endpointLabel} — ${modelLabel} (${providerLabel})`; if (option.disabled) { @@ -833,8 +871,21 @@ function handleMetadataExtractionModelChange() { } function updateAuthVisibility() { - const authType = endpointAuthTypeSelect?.value || "managed_identity"; const provider = endpointProviderSelect?.value || "aoai"; + const isCustom = provider === "custom"; + const descriptor = customApiTypeDescriptor(); + const customAuthTypes = descriptor?.authTypes || ["api_key", "bearer", "oauth2_client_credentials"]; + if (endpointAuthTypeSelect) { + Array.from(endpointAuthTypeSelect.options).forEach(option => { + const allowed = isCustom ? customAuthTypes.includes(option.value) : ["managed_identity", "service_principal", "api_key"].includes(option.value); + option.disabled = !allowed; + option.hidden = !allowed; + }); + if (endpointAuthTypeSelect.selectedOptions[0]?.disabled) { + endpointAuthTypeSelect.value = isCustom ? "api_key" : "managed_identity"; + } + } + const authType = endpointAuthTypeSelect?.value || "managed_identity"; const isApiKey = authType === "api_key"; const isFoundry = isFoundryProvider(provider); const projectNameFromEndpoint = syncProjectNameFromEndpoint(); @@ -842,21 +893,29 @@ function updateAuthVisibility() { syncVersionCustomVisibility(); setElementVisibility(endpointProjectGroup, isFoundry && !projectNameFromEndpoint); setElementVisibility(endpointProjectApiVersionGroup, isFoundry); - setElementVisibility(endpointOpenAiApiVersionGroup, true); + setElementVisibility(endpointOpenAiApiVersionGroup, !isCustom || descriptor?.requiresApiVersion); setElementVisibility(endpointSubscriptionGroup, provider === "aoai" && !isApiKey); setElementVisibility(endpointResourceGroup, provider === "aoai" && !isApiKey); setElementVisibility(miTypeGroup, authType === "managed_identity"); setElementVisibility(miClientGroup, authType === "managed_identity" && (miTypeSelect?.value === "user_assigned")); setElementVisibility(tenantGroup, authType === "service_principal"); - setElementVisibility(clientGroup, authType === "service_principal"); - setElementVisibility(secretGroup, authType === "service_principal"); + setElementVisibility(clientGroup, authType === "service_principal" || (isCustom && authType === "oauth2_client_credentials")); + setElementVisibility(secretGroup, authType === "service_principal" || (isCustom && authType === "oauth2_client_credentials")); setElementVisibility(apiKeyGroup, authType === "api_key"); setElementVisibility(endpointManagementCloudGroup, authType === "service_principal" && isFoundry); setElementVisibility(endpointCustomAuthorityGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); setElementVisibility(endpointFoundryScopeGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); - setElementVisibility(apiKeyNote, authType === "api_key"); - setElementVisibility(addModelBtn, authType === "api_key"); - setElementVisibility(fetchBtn, authType !== "api_key"); + setElementVisibility(apiKeyNote, authType === "api_key" && !isCustom); + setElementVisibility(addModelBtn, authType === "api_key" || isCustom); + setElementVisibility(fetchBtn, authType !== "api_key" && !isCustom); + setElementVisibility(customEndpointFields, isCustom); + setElementVisibility(document.getElementById("model-endpoint-anthropic-version-group"), isCustom && descriptor?.versionField === "anthropic_version"); + setElementVisibility(document.getElementById("model-endpoint-bearer-group"), isCustom && authType === "bearer"); + setElementVisibility(document.getElementById("model-endpoint-token-url-group"), isCustom && authType === "oauth2_client_credentials"); + setElementVisibility(document.getElementById("model-endpoint-oauth-scope-group"), isCustom && authType === "oauth2_client_credentials"); + setElementVisibility(document.getElementById("model-endpoint-api-key-override-group"), isCustom && authType === "api_key"); + const customHelp = document.getElementById("model-endpoint-api-type-help"); + if (customHelp) customHelp.textContent = descriptor?.description || ""; } function resetModal() { @@ -868,6 +927,14 @@ function resetModal() { if (endpointIdInput) endpointIdInput.value = ""; if (endpointNameInput) endpointNameInput.value = ""; if (endpointProviderSelect) endpointProviderSelect.value = "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = "openai"; + if (endpointUrlModeSelect) endpointUrlModeSelect.value = "auto"; + [endpointAnthropicVersionInput, endpointBearerInput, endpointTokenUrlInput, endpointOAuthScopeInput, + endpointApiKeyHeaderInput, endpointClientCertInput, endpointClientKeyInput].forEach(input => { + if (input) input.value = ""; + }); + if (endpointApiKeyPrefixInput) endpointApiKeyPrefixInput.value = customApiTypeDescriptor()?.defaultApiKeyPrefix || ""; + if (endpointBearerInput) endpointBearerInput.placeholder = ""; if (endpointUrlInput) endpointUrlInput.value = ""; if (endpointProjectInput) endpointProjectInput.value = ""; setSelectedVersionValue( @@ -918,6 +985,22 @@ function openModalForEndpoint(endpoint) { if (endpointIdInput) endpointIdInput.value = endpoint.id || ""; if (endpointNameInput) endpointNameInput.value = endpoint.name || ""; if (endpointProviderSelect) endpointProviderSelect.value = endpoint.provider || "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = endpoint.api_type || "openai"; + if (endpointUrlModeSelect) endpointUrlModeSelect.value = endpoint.connection?.url_mode || "auto"; + if (endpointAnthropicVersionInput) endpointAnthropicVersionInput.value = endpoint.connection?.anthropic_version || customApiTypeDescriptor()?.defaultVersion || ""; + if (endpointBearerInput) { + endpointBearerInput.value = endpoint.auth?.bearer_token || ""; + endpointBearerInput.placeholder = endpoint.has_bearer_token ? "Stored" : ""; + } + if (endpointTokenUrlInput) endpointTokenUrlInput.value = endpoint.auth?.token_url || ""; + if (endpointOAuthScopeInput) endpointOAuthScopeInput.value = endpoint.auth?.scope || ""; + if (endpointApiKeyHeaderInput) endpointApiKeyHeaderInput.value = endpoint.auth?.api_key_header || ""; + if (endpointApiKeyPrefixInput) { + endpointApiKeyPrefixInput.value = endpoint.auth?.api_key_prefix ?? + (endpoint.auth?.api_key_header ? "" : customApiTypeDescriptor()?.defaultApiKeyPrefix ?? ""); + } + if (endpointClientCertInput) endpointClientCertInput.value = endpoint.connection?.client_cert_path || ""; + if (endpointClientKeyInput) endpointClientKeyInput.value = endpoint.connection?.client_key_path || ""; if (endpointUrlInput) endpointUrlInput.value = endpoint.connection?.endpoint || ""; if (endpointProjectInput) endpointProjectInput.value = endpoint.connection?.project_name || ""; setSelectedVersionValue( @@ -992,6 +1075,14 @@ function cloneEndpointForDuplicate(endpoint) { duplicate.auth.api_key = ''; duplicate.has_api_key = false; } + if (duplicate.provider === "custom") { + for (const field of ["api_key", "bearer_token", "client_secret"]) { + if (duplicate.auth) delete duplicate.auth[field]; + } + duplicate.has_api_key = false; + duplicate.has_bearer_token = false; + duplicate.has_client_secret = false; + } return duplicate; } @@ -1281,10 +1372,11 @@ let modelVisionRequest = 0; * @returns {Promise} */ async function loadModelVisionCapability(models) { + const endpoint = { provider: endpointProviderSelect?.value, api_type: endpointApiTypeSelect?.value }; const wanted = (models || []) - .filter((model) => model && model.deploymentName) + .filter((model) => model && requestModelName(endpoint, model)) .map((model) => ({ - deploymentName: model.deploymentName, + deploymentName: requestModelName(endpoint, model), modelName: model.modelName || "", displayName: model.displayName || "", supportsVision: model.supportsVision, @@ -1327,7 +1419,7 @@ function createModelVisionControl(model, modelId) { const vision = model.capability_status?.vision; const resolved = vision ? { supports_vision: vision.supported, source: vision.source } - : modelVisionCapability[model.deploymentName || ""] || {}; + : modelVisionCapability[requestModelName({ provider: endpointProviderSelect?.value, api_type: endpointApiTypeSelect?.value }, model)] || {}; const source = typeof model.supportsVision === "boolean" ? "declared" : resolved.source; const checked = typeof model.supportsVision === "boolean" ? model.supportsVision @@ -1391,19 +1483,46 @@ function createModelCapabilityControls(model, modelId) { ? "Save the connection to resolve capabilities." : !status.supported ? status.reason || "Not supported." - : `${status.api === "images" ? "Direct image output" : status.api === "responses" ? "Image output through the Responses image tool" : "Text output"} · ${status.source || "unknown source"}`; + : `${status.api === "images" ? "Direct image output" : status.api === "responses" ? "Image output through the Responses image tool" : status.api === "mai" ? "MAI image output" : status.api === "flux" ? "FLUX image output" : "Text output"} · ${status.source || "unknown source"}`; container.append(wrapper, help); }); + const imageStatus = model.capability_status?.image_generation; + if (imageStatus?.supported) { + const imageInfo = createElement("p", "form-text"); + const operation = imageStatus.masking + ? "Masked and whole-image edits" + : imageStatus.editing ? "Reference-image edits; no uploaded masks" : "Generation only"; + imageInfo.textContent = [imageStatus.provider_label, imageStatus.cloud_label, imageStatus.lifecycle, operation].filter(Boolean).join(" · "); + container.appendChild(imageInfo); + } + if (imageStatus?.availability_reason) { + const notice = createElement("p", "form-text text-warning"); + notice.textContent = imageStatus.availability_reason; + container.appendChild(notice); + } const details = document.createElement("details"); details.className = "mt-2"; + details.dataset.modelCapabilityDetailsFor = String(modelId); const summary = document.createElement("summary"); summary.textContent = "Capability metadata"; const explanation = createElement("p", "form-text"); explanation.textContent = "Keep automatic metadata unless you have verified the deployment's capabilities. Publication choices do not change technical support. Image input is separate from image output."; details.append(summary, explanation); - capabilities.forEach(([, , metadataKey]) => { + const applyMetadataChange = () => { + modalModels = collectModalModels(); + renderModalModels(modalModels); + const disclosure = Array.from(modelsListEl.querySelectorAll("details")) + .find(item => item.dataset.modelCapabilityDetailsFor === String(modelId)); + if (disclosure) disclosure.open = true; + }; + [ + ["supportsChat", "Text output support"], + ["supportsImageGeneration", "Image generation support"], + ["supportsImageEditing", "Source-image editing support"], + ["supportsImageMasking", "Uploaded-mask support"], + ].forEach(([metadataKey, labelText]) => { const label = createElement("label", "form-label d-block mt-2"); - label.textContent = metadataKey === "supportsChat" ? "Text output support" : "Image generation support"; + label.textContent = labelText; const select = document.createElement("select"); select.id = getModelIconDomId(modelId, metadataKey); label.htmlFor = select.id; @@ -1412,12 +1531,26 @@ function createModelCapabilityControls(model, modelId) { select.dataset.metadataKey = metadataKey; select.append(new Option("Automatic", ""), new Option("Supported (verified by administrator)", "true"), new Option("Not supported", "false")); select.value = typeof model[metadataKey] === "boolean" ? String(model[metadataKey]) : ""; - select.addEventListener("change", () => { - modalModels = collectModalModels(); - renderModalModels(modalModels); - }); + select.addEventListener("change", applyMetadataChange); details.append(label, select); }); + const apiLabel = createElement("label", "form-label d-block mt-2"); + apiLabel.textContent = "Image API for explicit metadata"; + const apiSelect = document.createElement("select"); + apiSelect.id = getModelIconDomId(modelId, "image-api"); + apiLabel.htmlFor = apiSelect.id; + apiSelect.className = "form-select form-select-sm"; + apiSelect.dataset.imageApiFor = String(modelId); + apiSelect.append( + new Option("Automatic (catalog)", ""), new Option("Images API", "images"), + new Option("OpenAI Responses image tool", "responses"), new Option("Foundry MAI Image", "mai"), + new Option("Foundry FLUX", "flux"), + ); + apiSelect.value = model.image_generation_api || ""; + apiSelect.addEventListener("change", applyMetadataChange); + const apiHelp = createElement("p", "form-text"); + apiHelp.textContent = "Unknown Custom image models need an explicit compatible API. Editing and masks are separate capabilities; provider restrictions still apply."; + details.append(apiLabel, apiSelect, apiHelp); container.appendChild(details); return container; } @@ -1455,7 +1588,7 @@ function renderModalModels(models) { models.forEach((model) => { const wrapper = document.createElement("div"); wrapper.className = "border rounded p-2 mb-2"; - const deploymentName = model.deploymentName || ""; + const deploymentName = modalUsesModelName() ? model.modelName || model.name || "" : model.deploymentName || ""; const modelName = model.modelName || ""; const displayName = model.displayName || deploymentName; const description = model.description || ""; @@ -1486,8 +1619,11 @@ function renderModalModels(models) { const fieldsRow = createElement("div", "row g-2"); const deploymentCol = createElement("div", "col-md-4"); - deploymentCol.appendChild(createSmallLabel("Deployment Name")); - deploymentCol.appendChild(createModelTextInput(modelId, "deploymentNameFor", deploymentName, Boolean(deploymentReadonly))); + const requestNameLabel = createSmallLabel(modalUsesModelName() ? "Model Name" : "Deployment Name"); + const requestNameInput = createModelTextInput(modelId, modalUsesModelName() ? "modelNameFor" : "deploymentNameFor", deploymentName, Boolean(deploymentReadonly)); + requestNameInput.id = getModelIconDomId(modelId, "request-name"); + requestNameLabel.htmlFor = requestNameInput.id; + deploymentCol.append(requestNameLabel, requestNameInput); const displayCol = createElement("div", "col-md-4"); displayCol.appendChild(createSmallLabel("Display Name")); displayCol.appendChild(createModelTextInput(modelId, "displayNameFor", displayName)); @@ -1514,7 +1650,7 @@ function renderModalModels(models) { descriptionCol.appendChild(createModelTextInput(modelId, "descriptionFor", description)); fieldsRow.appendChild(deploymentCol); fieldsRow.appendChild(displayCol); - fieldsRow.appendChild(modelNameCol); + if (!modalUsesModelName()) fieldsRow.appendChild(modelNameCol); fieldsRow.appendChild(iconCol); fieldsRow.appendChild(responseLengthCol); fieldsRow.appendChild(descriptionCol); @@ -1596,6 +1732,14 @@ function collectModalModels() { if (next === undefined) delete model[key]; else model[key] = next; }); + const imageApiInput = Array.from(modelsListEl.querySelectorAll("[data-image-api-for]")) + .find(input => input.dataset.imageApiFor === String(model.id)); + if (imageApiInput) { + const nextApi = imageApiInput.value; + if ((model.image_generation_api || "") !== nextApi) delete model.capability_status; + if (nextApi) model.image_generation_api = nextApi; + else delete model.image_generation_api; + } const availabilityInputs = Array.from(modelsListEl.querySelectorAll("[data-capability-for]")) .filter(input => input.dataset.capabilityFor === model.id); if (availabilityInputs.some(input => String(input.checked) !== input.dataset.initialChecked)) { @@ -1625,16 +1769,14 @@ function collectModalModels() { async function testModelConnection(model) { const payload = buildEndpointPayload(); - if (!payload || !model?.deploymentName) { - showToast("Model deployment name is required for testing.", "warning"); + if (!payload || !model || !requestModelName(payload, model)) { + showToast("A request model name or deployment is required for testing.", "warning"); return; } const requestBody = { ...payload, - model: { - deploymentName: model.deploymentName - } + model: { ...model } }; try { @@ -1741,6 +1883,68 @@ async function fetchModels() { } } +function buildCustomEndpointPayload(source, existing, identityHeader) { + const descriptor = customApiTypeDescriptor(); + const name = endpointNameInput.value.trim(); + const endpoint = endpointUrlInput.value.trim(); + const authType = endpointAuthTypeSelect.value; + if (!descriptor || !name || !endpoint) { + showToast("Custom endpoint name, URL, and API type are required.", "warning"); + return null; + } + const connection = { + ...(source?.connection || {}), endpoint, url_mode: endpointUrlModeSelect?.value || "auto", + client_cert_path: endpointClientCertInput?.value.trim() || "", + client_key_path: endpointClientKeyInput?.value.trim() || "" + }; + delete connection.openai_api_version; + delete connection.project_api_version; + delete connection.project_name; + delete connection.api_version; + delete connection.anthropic_version; + if (descriptor.versionField) { + connection[descriptor.versionField] = descriptor.versionField === "anthropic_version" + ? endpointAnthropicVersionInput?.value.trim() || descriptor.defaultVersion + : getSelectedVersionValue(endpointOpenAiApiVersionInput, endpointOpenAiApiVersionCustomInput); + if (descriptor.requiresApiVersion && !connection[descriptor.versionField]) { + showToast("The selected Custom API type requires an API version.", "warning"); + return null; + } + } + const auth = { type: authType }; + if (authType === "api_key") { + auth.api_key = apiKeyInput?.value.trim() || ""; + auth.api_key_header = endpointApiKeyHeaderInput?.value.trim() || ""; + auth.api_key_prefix = endpointApiKeyPrefixInput?.value.trim() || ""; + if (!auth.api_key && !existing?.has_api_key) { + showToast("A Custom API key is required.", "warning"); + return null; + } + } else if (authType === "bearer") { + auth.bearer_token = endpointBearerInput?.value.trim() || ""; + if (!auth.bearer_token && !existing?.has_bearer_token) { + showToast("A Custom bearer token is required.", "warning"); + return null; + } + } else if (authType === "oauth2_client_credentials") { + auth.token_url = endpointTokenUrlInput?.value.trim() || ""; + auth.scope = endpointOAuthScopeInput?.value.trim() || ""; + auth.client_id = clientIdInput?.value.trim() || ""; + auth.client_secret = clientSecretInput?.value.trim() || ""; + if (!auth.token_url || !auth.client_id || (!auth.client_secret && !existing?.has_client_secret)) { + showToast("Custom OAuth2 requires a token URL, client ID, and client secret.", "warning"); + return null; + } + } else { + showToast("The Custom authentication method is not supported.", "warning"); + return null; + } + return { + id: endpointIdInput?.value.trim() || "", name, provider: "custom", api_type: descriptor.value, + connection, auth, management: {}, identity_header: identityHeader + }; +} + function buildEndpointPayload(requireDiscovery = false) { if (!endpointNameInput || !endpointUrlInput || !endpointOpenAiApiVersionInput) { return null; @@ -1771,6 +1975,9 @@ function buildEndpointPayload(requireDiscovery = false) { header_name: endpointIdentityHeaderNameInput?.value || "", value_type: endpointIdentityValueTypeSelect?.value || "" }); + if (provider === "custom") { + return buildCustomEndpointPayload(sourceEndpoint, existingEndpoint, identityHeader); + } if (!name || !endpoint || !openAiApiVersion) { showToast("Endpoint name, URL, and OpenAI API version are required.", "warning"); @@ -1867,18 +2074,27 @@ function saveEndpoint() { } const models = collectModalModels(); + if (payload.provider === "custom") { + const names = models.map(model => requestModelName(payload, model).toLowerCase()); + if (!names.length || names.some(name => !name) || new Set(names).size !== names.length) { + showToast("Custom connections require unique, non-empty request model identifiers.", "warning"); + return; + } + } const endpointId = endpointIdInput?.value || generateId(); const existingEndpoint = modelEndpoints.find((endpoint) => endpoint.id === endpointId); const sourceEndpoint = modalEndpointSource?.id === endpointId ? modalEndpointSource : existingEndpoint; const authType = payload.auth?.type || "managed_identity"; const hasApiKey = authType === "api_key" && (Boolean(payload.auth?.api_key) || Boolean(existingEndpoint?.has_api_key)); - const hasClientSecret = authType === "service_principal" && (Boolean(payload.auth?.client_secret) || Boolean(existingEndpoint?.has_client_secret)); + const hasClientSecret = ["service_principal", "oauth2_client_credentials"].includes(authType) && (Boolean(payload.auth?.client_secret) || Boolean(existingEndpoint?.has_client_secret)); + const hasBearerToken = authType === "bearer" && (Boolean(payload.auth?.bearer_token) || Boolean(existingEndpoint?.has_bearer_token)); const endpointData = { ...sourceEndpoint, id: endpointId, name: payload.name, provider: payload.provider, + api_type: payload.api_type, enabled: endpointModalEl?.dataset.duplicateDisabledDefault === 'true' ? false : (existingEndpoint ? existingEndpoint.enabled !== false : true), @@ -1888,7 +2104,8 @@ function saveEndpoint() { identity_header: payload.identity_header, models, has_api_key: hasApiKey, - has_client_secret: hasClientSecret + has_client_secret: hasClientSecret, + has_bearer_token: hasBearerToken }; const existingIndex = modelEndpoints.findIndex((endpoint) => endpoint.id === endpointId); @@ -2424,8 +2641,22 @@ function init() { } if (endpointProviderSelect) { endpointProviderSelect.addEventListener("change", () => { + modalModels = collectModalModels(); updateAuthVisibility(); syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); + }); + } + if (endpointApiTypeSelect) { + endpointApiTypeSelect.addEventListener("change", () => { + modalModels = collectModalModels(); + const descriptor = customApiTypeDescriptor(); + if (endpointAnthropicVersionInput) endpointAnthropicVersionInput.value = descriptor?.defaultVersion || ""; + if (endpointApiKeyHeaderInput) endpointApiKeyHeaderInput.value = ""; + if (endpointApiKeyPrefixInput) endpointApiKeyPrefixInput.value = descriptor?.defaultApiKeyPrefix || ""; + modalModels = modalModels.map(({ capability_status: status, ...model }) => model); + updateAuthVisibility(); + renderModalModels(modalModels); }); } if (endpointUrlInput) { diff --git a/application/single_app/static/json/model_capabilities.json b/application/single_app/static/json/model_capabilities.json index 771dc58a1..c900fd889 100644 --- a/application/single_app/static/json/model_capabilities.json +++ b/application/single_app/static/json/model_capabilities.json @@ -1,14 +1,14 @@ { "$schema": "https://simplechat.local/schemas/model-capabilities.schema.json", - "schemaVersion": 1, - "lastUpdated": "2026-09-08", + "schemaVersion": 2, + "lastUpdated": "2026-09-16", "description": "SimpleChat model capability catalog. Capability flags inform task-specific eligibility alongside provider adapters and publication policy. Optional reasoningPolicy data defines per-model effort choices and the application fallback for invalid selections, not the provider default; an absent reasoningPolicy is unknown. Reasoning-only entries do not establish vision or image generation support. Optional model token-limit fields are consumed by durable tabular batch planning when present.", "capabilityFields": { "processesText": "Accepts text input.", "generatesText": "Produces text output.", "processesImages": "Accepts image input for vision or image understanding.", "generatesImages": "Produces native/direct image output through an image model/API; distinct from orchestrating an image generation tool.", - "imageGenerationTool": "Verified support for orchestrating the hosted image_generation tool in the Responses API, not native image output or vision. Deployment, provider, region, API version, and image-backend prerequisites still apply.", + "imageGenerationTool": "Publisher documentation for the hosted image_generation tool, not native image output or vision. Runtime eligibility requires a matching provider-qualified imageProfiles entry or a compatible explicit Custom declaration.", "processesAudio": "Accepts audio input or supports speech/audio understanding.", "generatesAudio": "Produces audio or speech output.", "processesVideo": "Accepts video input or documented video-clip summarization input.", @@ -18,11 +18,131 @@ "toolCalling": "Supports function/tool calling or provider-equivalent tool use.", "structuredOutput": "Supports structured outputs, JSON-schema outputs, or provider-equivalent constrained structured responses." }, + "imageOperationProfiles": { + "openai-responses": { + "api": "responses", + "editing": true, + "masking": true, + "sizes": ["1024x1024", "1536x1024", "1024x1536"], + "qualities": ["low", "medium", "high"], + "backgrounds": ["opaque", "transparent"], + "inputFormats": ["image/png", "image/jpeg", "image/webp"], + "outputFormats": ["image/png", "image/jpeg", "image/webp"], + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["image-profiles-openai-tools", "image-profiles-openai-images"] + }, + "openai-images": { + "api": "images", + "editing": true, + "masking": true, + "sizes": ["1024x1024", "1536x1024", "1024x1536"], + "qualities": ["low", "medium", "high"], + "backgrounds": ["opaque", "transparent"], + "inputFormats": ["image/png", "image/jpeg", "image/webp"], + "outputFormats": ["image/png", "image/jpeg", "image/webp"], + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["image-profiles-openai-images"] + }, + "azure-images": { + "api": "images", + "editing": true, + "masking": true, + "sizes": ["1024x1024", "1536x1024", "1024x1536"], + "qualities": ["low", "medium", "high"], + "backgrounds": ["opaque", "transparent"], + "inputFormats": ["image/png", "image/jpeg"], + "outputFormats": ["image/png", "image/jpeg"], + "maxMaskBytes": 4194304, + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["azure-openai-images", "image-profiles-government"] + }, + "mai-images": { + "api": "mai", + "editing": true, + "masking": false, + "maskEvidence": "unknown", + "sizes": ["1024x1024", "1024x768", "768x1024"], + "qualities": [], + "backgrounds": [], + "inputFormats": ["image/png", "image/jpeg"], + "outputFormats": ["image/png"], + "minDimension": 768, + "maxPixels": 1048576, + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["image-profiles-mai", "image-profiles-government"] + }, + "flux-2-pro": { + "api": "flux", + "modelPath": "flux-2-pro", + "editing": true, + "masking": false, + "maskEvidence": "unknown", + "sizes": ["1024x1024", "1536x1024", "1024x1536"], + "qualities": [], + "backgrounds": [], + "inputFormats": ["image/png", "image/jpeg"], + "outputFormats": ["image/png", "image/jpeg"], + "maxPixels": 4000000, + "maxReferenceImages": 8, + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["image-profiles-flux", "image-profiles-government"] + }, + "flux-2-flex": { + "api": "flux", + "modelPath": "flux-2-flex", + "editing": true, + "masking": false, + "maskEvidence": "unknown", + "sizes": ["1024x1024", "1536x1024", "1024x1536"], + "qualities": [], + "backgrounds": [], + "inputFormats": ["image/png", "image/jpeg"], + "outputFormats": ["image/png", "image/jpeg"], + "maxPixels": 4000000, + "maxReferenceImages": 10, + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["image-profiles-flux", "image-profiles-government"] + }, + "flux-kontext": { + "api": "flux", + "modelPath": "flux-kontext-pro", + "transport": "openai_images", + "editing": true, + "masking": false, + "maskEvidence": "unknown", + "sizes": ["1024x1024"], + "qualities": [], + "backgrounds": [], + "inputFormats": ["image/png", "image/jpeg"], + "outputFormats": ["image/png", "image/jpeg"], + "maxPixels": 1048576, + "maxReferenceImages": 1, + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["image-profiles-flux", "image-profiles-government"] + }, + "flux-1.1": { + "api": "flux", + "modelPath": "flux-pro-1.1", + "editing": false, + "masking": false, + "maskEvidence": "unknown", + "sizes": ["1024x1024", "1536x1024", "1024x1536"], + "qualities": [], + "backgrounds": [], + "inputFormats": [], + "outputFormats": ["image/png", "image/jpeg"], + "maxPixels": 1600000, + "availability": {"commercial": "documented", "government": "unknown", "unknown": "unknown"}, + "sourceIds": ["image-profiles-flux", "image-profiles-government"] + } + }, "coverageNotes": [ "OpenAI coverage includes GPT-5.x model IDs seen through Foundry, older Responses orchestration models, and direct GPT Image models.", "Image-generation tool metadata was checked against official model-specific Supported tools documentation on 2026-09-08. Generic function calling, image input, and inclusion in a Responses model list alone do not establish hosted image-generation support; a false or omitted flag means it is not verified for this integration.", - "OpenAI model pages describe api.openai.com, not Azure deployment readiness. Azure requires its v1 Responses surface for the hosted image tool; the dated 2025-04-01-preview Responses contract is not equivalent to v1. Direct Images API versions are a separate contract.", - "Azure lists GPT-5.6 Sol, Terra, and Luna version 2026-07-09 for Responses. Its image-tool examples may require a deployed GPT Image backend and the x-ms-oai-image-generation-deployment header. Catalog support does not promise deployment-free routing, entitlement, region availability, or a successful live request.", + "OpenAI model pages describe api.openai.com, not Azure deployment readiness. imageProfiles separates direct OpenAI, Azure OpenAI, and Foundry image operations. The application hosting cloud never establishes endpoint capabilities or availability.", + "Azure documents Responses image generation with a separate GPT Image deployment. SimpleChat deliberately offers only dedicated image models on Azure/Foundry; the conditional Azure Responses integration is not enabled by publisher imageGenerationTool flags or administrator overrides.", + "Government image-model availability is not established by the reviewed Government model table. Unknown is not unavailable, and a Government-hosted app may use an explicitly configured commercial endpoint without relabeling that endpoint as Government.", + "Image operation profiles describe the implemented single-image workflow. MAI and FLUX support source-image edits where stated, but their reviewed APIs do not document uploaded masks. Masking is not inferred from vision, tool calling, or inpainting prose.", "Azure -chat entries are not automatically equated with OpenAI -chat-latest model IDs. OpenAI's documented tools for a similarly named chat snapshot do not establish the Azure entry's image tool support.", "Capabilities classify model behavior, not live availability. New legacy/deprecated/retired entries describe the cited API lifecycle; OpenAI retirement dates do not automatically apply to Azure. Retained DALL-E compatibility must not be interpreted as an operational service.", "Reasoning policies describe effort support, not endpoint/API availability. No Responses API migration is implied. Responses-only max effort is excluded from this Chat Completions policy. Models without verified level lists remain unknown.", @@ -32,6 +152,42 @@ "Microsoft coverage includes public Phi and MAI model cards with clear capability statements." ], "sources": [ + { + "id": "image-profiles-openai-tools", + "provider": "openai", + "title": "Direct OpenAI image-generation tools, model eligibility and source/mask editing (reviewed 2026-09-16)", + "url": "https://developers.openai.com/api/docs/guides/tools-image-generation" + }, + { + "id": "image-profiles-openai-images", + "provider": "openai", + "title": "Direct OpenAI generation, image edits and mask requirements", + "url": "https://developers.openai.com/api/docs/guides/image-generation" + }, + { + "id": "image-profiles-mai", + "provider": "microsoft", + "title": "Foundry MAI Image 2.5/2.6 APIs and constraints (reviewed 2026-09-16)", + "url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/how-to/use-foundry-models-mai-image" + }, + { + "id": "image-profiles-flux", + "provider": "microsoft", + "title": "Foundry FLUX generation and reference-image editing APIs", + "url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/how-to/use-foundry-models-flux" + }, + { + "id": "image-profiles-government", + "provider": "microsoft", + "title": "Azure Government model availability; image-model availability not established (reviewed 2026-09-16)", + "url": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure-gov" + }, + { + "id": "openai-gpt6-astra", + "provider": "openai", + "title": "GPT-6 Astra model and supported Responses tools", + "url": "https://developers.openai.com/api/docs/models/gpt-6-astra" + }, { "id": "azure-openai-reasoning", "provider": "microsoft", @@ -427,6 +583,7 @@ }, { "id": "gpt-5.6-sol", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -457,6 +614,7 @@ }, { "id": "gpt-5.6-terra", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -487,6 +645,7 @@ }, { "id": "gpt-5.6-luna", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning", "luna-deployed-contract"] @@ -517,6 +676,7 @@ }, { "id": "gpt-5.5", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -573,6 +733,7 @@ }, { "id": "gpt-5.4", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -603,6 +764,7 @@ }, { "id": "gpt-5.4-pro", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["medium", "high", "xhigh"], "default_effort": "medium", "sourceIds": ["openai-gpt5-4-pro"] @@ -633,6 +795,7 @@ }, { "id": "gpt-5.4-mini", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -663,6 +826,7 @@ }, { "id": "gpt-5.4-nano", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -779,6 +943,7 @@ }, { "id": "gpt-5.2", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high", "xhigh"], "default_effort": "low", "sourceIds": ["openai-gpt5-2"] @@ -835,6 +1000,7 @@ }, { "id": "gpt-5.1", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["none", "low", "medium", "high"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -981,6 +1147,7 @@ }, { "id": "gpt-5", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["minimal", "low", "medium", "high"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning", "openai-gpt5"] @@ -1101,6 +1268,7 @@ }, { "id": "gpt-5-nano", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["minimal", "low", "medium", "high"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning", "openai-gpt5"] @@ -1157,6 +1325,7 @@ }, { "id": "gpt-4o", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "unsupported", "efforts": [], "default_effort": null, "sourceIds": ["openai-chat-completions"] @@ -1187,6 +1356,7 @@ }, { "id": "gpt-4o-mini", + "imageProfiles": {"openai": "openai-responses"}, "provider": "openai", "displayName": "GPT-4o Mini", "aliases": [], @@ -1213,6 +1383,7 @@ }, { "id": "gpt-4.1", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "unsupported", "efforts": [], "default_effort": null, "sourceIds": ["openai-chat-completions"] @@ -1269,6 +1440,7 @@ }, { "id": "gpt-4.1-nano", + "imageProfiles": {"openai": "openai-responses"}, "provider": "openai", "displayName": "GPT-4.1 Nano", "aliases": [], @@ -1325,6 +1497,7 @@ }, { "id": "o3", + "imageProfiles": {"openai": "openai-responses"}, "reasoningPolicy": { "status": "supported", "efforts": ["low", "medium", "high"], "default_effort": "low", "sourceIds": ["azure-openai-reasoning"] @@ -1415,6 +1588,8 @@ }, { "id": "gpt-image-2", + "imageProfiles": {"openai": "openai-images", "azure_openai": "azure-images"}, + "imageLifecycle": {"openai": "current", "azure_openai": "current"}, "provider": "openai", "displayName": "GPT Image 2", "aliases": [], @@ -1441,6 +1616,8 @@ }, { "id": "gpt-image-1.5", + "imageProfiles": {"openai": "openai-images", "azure_openai": "azure-images"}, + "imageLifecycle": {"openai": "deprecated", "azure_openai": "limited_access_preview"}, "provider": "openai", "displayName": "GPT Image 1.5", "aliases": [], @@ -1467,6 +1644,8 @@ }, { "id": "gpt-image-1", + "imageProfiles": {"openai": "openai-images", "azure_openai": "azure-images"}, + "imageLifecycle": {"openai": "deprecated", "azure_openai": "limited_access_preview"}, "provider": "openai", "displayName": "GPT Image 1", "aliases": [], @@ -1493,6 +1672,8 @@ }, { "id": "gpt-image-1-mini", + "imageProfiles": {"openai": "openai-images", "azure_openai": "azure-images"}, + "imageLifecycle": {"openai": "deprecated", "azure_openai": "limited_access_preview"}, "provider": "openai", "displayName": "GPT Image 1 Mini", "aliases": [], @@ -1519,6 +1700,7 @@ }, { "id": "dall-e-3", + "imageLifecycle": {"openai": "retired", "azure_openai": "retired"}, "provider": "openai", "displayName": "DALL-E 3 (retired)", "aliases": ["dalle-3"], @@ -2541,6 +2723,192 @@ }, "notes": ["Model card describes broad text generation, reasoning, problem solving, code generation, and code comprehension."], "sourceIds": ["microsoft-mai-ds-r1"] + }, + { + "id": "gpt-6-astra", + "provider": "openai", + "displayName": "GPT-6 Astra", + "aliases": [], + "family": "gpt-6", + "lifecycle": "current", + "capabilities": { + "processesText": true, "generatesText": true, "processesImages": true, + "generatesImages": false, "imageGenerationTool": true, "toolCalling": true + }, + "imageProfiles": {"openai": "openai-responses"}, + "notes": ["Image generation is a direct OpenAI Responses tool, not native image output or an Azure image deployment."], + "sourceIds": ["openai-gpt6-astra"] + }, + { + "id": "gpt-image-2.5-flare", + "provider": "openai", + "displayName": "GPT Image 2.5 Flare", + "aliases": [], + "family": "gpt-image", + "lifecycle": "current", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"openai": "openai-images", "azure_openai": "azure-images"}, + "imageLifecycle": {"openai": "current", "azure_openai": "current"}, + "notes": ["Dedicated image model. Azure model version 2026-09-09 is separate from the direct OpenAI 2026-09-08 snapshot. Consult current regional availability before deployment."], + "sourceIds": ["image-profiles-openai-tools", "azure-openai-images"] + }, + { + "id": "gpt-image-2.5-sunburst", + "provider": "openai", + "displayName": "GPT Image 2.5 Sunburst", + "aliases": [], + "family": "gpt-image", + "lifecycle": "current", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"openai": "openai-images", "azure_openai": "azure-images"}, + "imageLifecycle": {"openai": "current", "azure_openai": "current"}, + "notes": ["Dedicated image model. Azure model version 2026-09-09 is separate from the direct OpenAI 2026-09-08 snapshot. Consult current regional availability before deployment."], + "sourceIds": ["image-profiles-openai-tools", "azure-openai-images"] + }, + { + "id": "MAI-Image-2.5", + "provider": "microsoft", + "displayName": "MAI Image 2.5", + "aliases": [], + "family": "mai-image", + "lifecycle": "preview", + "releaseDate": "2026-06-02", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "mai-images"}, + "imageLifecycle": {"foundry": "preview"}, + "notes": ["Foundry generation and reference-image editing. The reviewed API does not document mask uploads; inpainting prose is not a mask contract."], + "sourceIds": ["image-profiles-mai"] + }, + { + "id": "MAI-Image-2.5-Flash", + "provider": "microsoft", + "displayName": "MAI Image 2.5 Flash", + "aliases": [], + "family": "mai-image", + "lifecycle": "preview", + "releaseDate": "2026-06-02", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "mai-images"}, + "imageLifecycle": {"foundry": "preview"}, + "sourceIds": ["image-profiles-mai"] + }, + { + "id": "MAI-Image-2.5-Pro", + "provider": "microsoft", + "displayName": "MAI Image 2.5 Pro", + "aliases": [], + "family": "mai-image", + "lifecycle": "preview", + "releaseDate": "2026-06-19", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "mai-images"}, + "imageLifecycle": {"foundry": "preview"}, + "sourceIds": ["image-profiles-mai"] + }, + { + "id": "MAI-Image-2.6", + "provider": "microsoft", + "displayName": "MAI Image 2.6", + "aliases": [], + "family": "mai-image", + "lifecycle": "preview", + "releaseDate": "2026-07-31", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "mai-images"}, + "imageLifecycle": {"foundry": "preview"}, + "notes": ["Optional web grounding is not enabled by the SimpleChat image adapter."], + "sourceIds": ["image-profiles-mai"] + }, + { + "id": "MAI-Image-2.6-Flash", + "provider": "microsoft", + "displayName": "MAI Image 2.6 Flash", + "aliases": [], + "family": "mai-image", + "lifecycle": "preview", + "releaseDate": "2026-07-31", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "mai-images"}, + "imageLifecycle": {"foundry": "preview"}, + "notes": ["Optional web grounding is not enabled by the SimpleChat image adapter."], + "sourceIds": ["image-profiles-mai"] + }, + { + "id": "FLUX.2-pro", + "provider": "blackforestlabs", + "displayName": "FLUX.2 Pro", + "aliases": [], + "family": "flux", + "lifecycle": "current", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "flux-2-pro"}, + "sourceIds": ["image-profiles-flux"] + }, + { + "id": "FLUX.2-flex", + "provider": "blackforestlabs", + "displayName": "FLUX.2 Flex", + "aliases": [], + "family": "flux", + "lifecycle": "current", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "flux-2-flex"}, + "sourceIds": ["image-profiles-flux"] + }, + { + "id": "FLUX.1-Kontext-pro", + "provider": "blackforestlabs", + "displayName": "FLUX.1 Kontext Pro", + "aliases": [], + "family": "flux", + "lifecycle": "current", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": true, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "flux-kontext"}, + "sourceIds": ["image-profiles-flux"] + }, + { + "id": "FLUX-1.1-pro", + "provider": "blackforestlabs", + "displayName": "FLUX 1.1 Pro", + "aliases": [], + "family": "flux", + "lifecycle": "current", + "capabilities": { + "processesText": true, "generatesText": false, "processesImages": false, + "generatesImages": true, "imageGenerationTool": false, "toolCalling": false + }, + "imageProfiles": {"foundry": "flux-1.1"}, + "sourceIds": ["image-profiles-flux"] } ] } diff --git a/application/single_app/static/json/schemas/model_capabilities.schema.json b/application/single_app/static/json/schemas/model_capabilities.schema.json new file mode 100644 index 000000000..eeb36b957 --- /dev/null +++ b/application/single_app/static/json/schemas/model_capabilities.schema.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://simplechat.local/schemas/model-capabilities.schema.json", + "title": "SimpleChat model capability catalog", + "type": "object", + "required": ["schemaVersion", "models", "imageOperationProfiles", "sources"], + "properties": { + "schemaVersion": {"const": 2}, + "imageOperationProfiles": { + "type": "object", + "additionalProperties": {"$ref": "#/$defs/imageOperation"} + }, + "sources": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "url"], + "properties": {"id": {"type": "string", "minLength": 1}, "url": {"type": "string", "format": "uri"}} + } + }, + "models": { + "type": "array", + "items": { + "type": "object", + "required": ["id"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "capabilities": {"type": "object", "additionalProperties": {"type": "boolean"}}, + "imageProfiles": { + "type": "object", + "propertyNames": {"enum": ["openai", "azure_openai", "foundry"]}, + "additionalProperties": {"type": "string", "minLength": 1} + }, + "imageLifecycle": { + "type": "object", + "additionalProperties": {"enum": ["current", "preview", "limited_access_preview", "deprecated", "retired"]} + } + } + } + } + }, + "$defs": { + "strings": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "imageOperation": { + "type": "object", + "required": ["api", "editing", "masking", "sizes", "qualities", "backgrounds", "inputFormats", "outputFormats", "availability", "sourceIds"], + "additionalProperties": false, + "properties": { + "api": {"enum": ["images", "responses", "mai", "flux"]}, + "modelPath": {"enum": ["flux-2-pro", "flux-2-flex", "flux-kontext-pro", "flux-pro-1.1"]}, + "transport": {"enum": ["native", "openai_images"]}, + "editing": {"type": "boolean"}, + "masking": {"type": "boolean"}, + "maskEvidence": {"enum": ["documented", "unknown", "unsupported"]}, + "sizes": {"$ref": "#/$defs/strings"}, + "qualities": {"$ref": "#/$defs/strings"}, + "backgrounds": {"$ref": "#/$defs/strings"}, + "inputFormats": {"$ref": "#/$defs/strings"}, + "outputFormats": {"$ref": "#/$defs/strings"}, + "sourceIds": {"$ref": "#/$defs/strings"}, + "maxMaskBytes": {"type": "integer", "minimum": 1}, + "maxReferenceImages": {"type": "integer", "minimum": 1}, + "minDimension": {"type": "integer", "minimum": 1}, + "maxPixels": {"type": "integer", "minimum": 1}, + "availability": { + "type": "object", + "required": ["commercial", "government", "unknown"], + "additionalProperties": {"enum": ["documented", "unknown", "unavailable"]} + } + } + } + } +} diff --git a/application/single_app/templates/admin/_panes/model-endpoints.html b/application/single_app/templates/admin/_panes/model-endpoints.html index 6f19fbf33..2a1022d9e 100644 --- a/application/single_app/templates/admin/_panes/model-endpoints.html +++ b/application/single_app/templates/admin/_panes/model-endpoints.html @@ -1,4 +1,4 @@ -
+

Configure shared AI Connections for chat and image generation. Each capability keeps its own default model. @@ -9,7 +9,7 @@

AI Connections

- Manage Azure OpenAI and Azure AI Foundry connections once. Models can be available for chat, image generation, or both. Image connections work independently of the chat mode switch. + Manage Azure OpenAI, Azure AI Foundry, and Custom connections once. Models can be available for chat, image generation, or both when supported. Image connections work independently of the chat mode switch.

{% if not settings.enable_multi_model_endpoints %} @@ -199,6 +199,23 @@
Model Endpoint Identity H +
+ Custom endpoint network policy +

Applies to Custom inference and OAuth2 token requests. Loopback, link-local, platform metadata addresses, redirects, and environment proxies remain blocked.

+
+ + +
+
+ + +
+

HTTP sends prompts and credentials without encryption. Prefer HTTPS with a private CA bundle.

+ + +

Optional deployment-mounted trust bundle. Empty uses public roots. HTTPS certificate verification cannot be disabled.

+
+
@@ -399,5 +416,66 @@
+ {% include '_multiendpoint_modal.html' %}
diff --git a/application/v2_ui/src/components/admin/CapabilityModelPicker.tsx b/application/v2_ui/src/components/admin/CapabilityModelPicker.tsx index 1048f7708..42d33709f 100644 --- a/application/v2_ui/src/components/admin/CapabilityModelPicker.tsx +++ b/application/v2_ui/src/components/admin/CapabilityModelPicker.tsx @@ -4,6 +4,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import { AlertCircle, Loader2, RefreshCw } from 'lucide-react'; import { capabilityDescription, + canGenerateImage, fetchCapabilityModels, saveCapabilityModel, testImageModel, @@ -17,6 +18,7 @@ import { type ImplementedCapability, } from '../../lib/modelConnections'; import { useModelConnectionsStore } from '../../stores/modelConnectionsStore'; +import { useBootstrapStore } from '../../stores/bootstrapStore'; import { toast } from '../../stores/toastStore'; import { GlassButton } from '../ui/primitives'; @@ -51,6 +53,9 @@ export function CapabilityModelPicker({ if (!controller.signal.aborted && request === requestId.current) { setData(response); setError(null); + if (image) { + void useBootstrapStore.getState().refresh(); + } } }).catch((loadError: unknown) => { if (!controller.signal.aborted && request === requestId.current) { @@ -68,6 +73,7 @@ export function CapabilityModelPicker({ const selectedIndex = data ? findChoiceIndex(data.choices, data.selection) : -1; const dangling = Boolean(data && hasDefaultModel(data.selection) && selectedIndex < 0); const selectedChoice = data?.choices[selectedIndex]; + const selectedImageUnavailable = image && Boolean(selectedChoice) && !canGenerateImage(selectedChoice?.capability); const busy = loading || saving || testing; const enabled = data?.enabled ?? featureEnabled; const migrationFailed = image && data?.migration && data.migration.status !== 'complete'; @@ -80,6 +86,9 @@ export function CapabilityModelPicker({ if (raw !== '' && !choice) { return; } + if (image && choice && !canGenerateImage(choice.capability)) { + return; + } const previous = data; const next = choiceToSelection(choice ?? null); const request = ++requestId.current; @@ -91,6 +100,9 @@ export function CapabilityModelPicker({ const response = await saveCapabilityModel(capability, next); if (request === requestId.current) { setData(response); + if (image) { + void useBootstrapStore.getState().refresh(); + } toast.success(choice ? `${label} set to ${choice.modelLabel}.` : `${label} cleared.`); } } catch (saveError) { @@ -104,7 +116,7 @@ export function CapabilityModelPicker({ }; const runImageTest = async () => { - if (!data || !selectedChoice || busy) { + if (!data || !selectedChoice || busy || !enabled || !canGenerateImage(selectedChoice.capability)) { return; } setTesting(true); @@ -139,7 +151,11 @@ export function CapabilityModelPicker({ {groups.map((group) => ( {group.items.map(({ choice, index }) => ( -