Skip to content

Add provider-aware image generation and Custom endpoints #1481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Paul Lizer (paullizer) merged 1 commit into from
Sep 17, 2026
Merged

Add provider-aware image generation and Custom endpoints #1481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion application/single_app/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
EXECUTOR_TYPE = 'thread'
EXECUTOR_MAX_WORKERS = 30
SESSION_TYPE = 'filesystem'
VERSION = "0.261.106"
VERSION = "0.261.107"
IS_DEVELOPMENT = is_development_env_enabled()

SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')
Expand Down
33 changes: 24 additions & 9 deletions application/single_app/functions_ai_connection_migration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,14 @@ def _legacy_image_connection(settings, apim):
if deployment == active_name:
support = resolve_model_capability(model, IMAGE_GENERATION_CAPABILITY, "aoai")
model_name = str(model.get("modelName") or "").lower()
legacy_route = (
"responses"
if not apim and model_name and not any(marker in model_name for marker in ("image", "dall-e", "dalle"))
else "images"
)
model["supportsImageGeneration"] = True
model["image_generation_api"] = support["api"] or legacy_route
if model.get("supportsImageGeneration") is not False:
if support["supported"]:
model["supportsImageGeneration"] = True
model["image_generation_api"] = support["api"]
elif not model_name:
# An unrecorded legacy Images deployment remains recoverable, not a GPT tool guess.
model["supportsImageGeneration"] = True
model["image_generation_api"] = "images"
models.append(model)

auth_type = "api_key" if apim else str(settings.get("azure_openai_image_gen_authentication_type") or "key")
Expand Down Expand Up @@ -177,7 +178,9 @@ def _compatible_connection(existing, imported):
(model for model in existing.get("models") or [] if model.get("deploymentName") == incoming["deploymentName"]),
None,
)
if match and incoming["enabled"] and not supports_model_capability(match, IMAGE_GENERATION_CAPABILITY, existing.get("provider")):
if match and incoming["enabled"] and not supports_model_capability(
match, IMAGE_GENERATION_CAPABILITY, existing.get("provider"), endpoint=existing
):
return False
return bool(existing.get("id"))

Expand All @@ -196,6 +199,7 @@ def build_image_connection_migration(settings, normalize_endpoint=None):
endpoints = copy.deepcopy(current)
selected = dict(EMPTY_MODEL_SELECTION)
imported_count = 0
default_warning = ""
active_apim = bool(settings.get("enable_image_gen_apim"))
for apim in (False, True):
imported, active_name = _legacy_image_connection(settings, apim)
Expand Down Expand Up @@ -228,7 +232,12 @@ def build_image_connection_migration(settings, normalize_endpoint=None):
"model_id": str(model["id"]),
"provider": str(existing["provider"]).lower(),
}
return {
if not supports_model_capability(model, IMAGE_GENERATION_CAPABILITY, existing["provider"], endpoint=existing):
default_warning = (
"The imported image default is not supported on its provider. "
"Select a dedicated image model in AI Connections; the original settings have been retained."
)
patch = {
"model_endpoints": endpoints,
IMAGE_SELECTION_KEY: selected,
IMAGE_MIGRATION_VERSION_KEY: IMAGE_MIGRATION_VERSION,
Expand All @@ -238,6 +247,12 @@ def build_image_connection_migration(settings, normalize_endpoint=None):
"message": "Existing image configuration is now managed through AI Connections.",
},
}
if default_warning:
patch["ai_connection_default_notices"] = {
**(settings.get("ai_connection_default_notices") or {}),
IMAGE_GENERATION_CAPABILITY: default_warning,
}
return patch


def migrate_image_connections(
Expand Down
87 changes: 51 additions & 36 deletions application/single_app/functions_ai_connections.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
get_model_catalog_capabilities,
resolve_model_vision_support,
)
from functions_image_capabilities import (
IMAGE_APIS,
IMAGE_PROVIDERS,
resolve_image_model_capability,
)


CHAT_CAPABILITY = "chat"
Expand All @@ -23,7 +28,6 @@
IMAGE_MIGRATION_VERSION_KEY = "ai_connections_image_migration_version"
IMAGE_MIGRATION_VERSION = 1
EMPTY_MODEL_SELECTION = {"endpoint_id": "", "model_id": "", "provider": ""}
IMAGE_PROVIDERS = ("aoai", "aifoundry", "new_foundry")
_DIRECT_IMAGE_PATTERN = re.compile(r"^(?:gpt-image(?:-|$)|dall-e(?:-|$)|dalle(?:-|$))")
_NON_CHAT_PATTERN = re.compile(
r"embedding|^whisper|(?:^|-)(?:tts|transcribe|realtime)(?:-|$)"
Expand Down Expand Up @@ -55,7 +59,7 @@ class CapabilityDefinition:
IMAGE_SELECTION_KEY,
"generatesImages",
IMAGE_PROVIDERS,
feature_flag="enable_image_generation", api_routes=("images", "responses"),
feature_flag="enable_image_generation", api_routes=IMAGE_APIS,
),
}
_CLIENT_FACTORIES = {}
Expand Down Expand Up @@ -172,7 +176,7 @@ def _support(supported, source, reason="", api=""):
}


def resolve_model_capability(model, capability, provider="aoai"):
def resolve_model_capability(model, capability, provider="aoai", *, endpoint=None):
"""Describe technical support; availability and transient service health are separate."""
definition = get_capability_definition(capability)
provider = str(provider or "aoai").lower()
Expand Down Expand Up @@ -200,7 +204,8 @@ def resolve_model_capability(model, capability, provider="aoai"):
for field_name in ("modelName", "behavior_name")
)
)
if direct_image or (underlying_is_named and _NON_CHAT_PATTERN.search(name)):
image_only = catalog and catalog.get("generatesImages") is True and catalog.get("generatesText") is False
if direct_image or image_only or (underlying_is_named and _NON_CHAT_PATTERN.search(name)):
return _support(False, "model", "This model is not supported by the text-chat adapter.")
declared = _declared_flag(model, "supportsChat", "supports_chat")
if declared is not None:
Expand All @@ -212,23 +217,8 @@ def resolve_model_capability(model, capability, provider="aoai"):
return _support(bool(name), "legacy", "" if name else "A deployment name is required.", "chat")

if capability == IMAGE_GENERATION_CAPABILITY:
declared = _declared_flag(model, "supportsImageGeneration", "supports_image_generation")
route = str(model.get("image_generation_api") or "") if isinstance(model, Mapping) else ""
if declared is False:
return _support(False, "declared", "Image generation is not supported by this model.")
if direct_image:
return _support(True, "model", api="images")
if declared is True:
return _support(True, "declared", api=route if route in ("images", "responses") else "responses")
if catalog and catalog.get("imageGenerationTool") is True:
return _support(True, "catalog", api="responses")
if catalog and catalog.get("generatesImages") is True:
return _support(
False,
"provider",
"This image model requires a provider-specific image adapter or an explicitly declared compatible image API.",
)
return _support(False, "unknown", "Image generation support has not been established for this model.")
support = resolve_image_model_capability(model, endpoint, provider)
return _support(support["supported"], support["source"], support["reason"], support["api"])

supported = bool(catalog and catalog.get(definition.catalog_flag))
return _support(
Expand All @@ -238,20 +228,20 @@ def resolve_model_capability(model, capability, provider="aoai"):
)


def supports_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai"):
def supports_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai", *, endpoint=None):
"""Return whether a model is technically suitable and published for this operation."""
if isinstance(model, Mapping):
if model.get("enabled") is False:
return False
enabled_capabilities = model.get("enabled_capabilities")
if isinstance(enabled_capabilities, list) and capability not in enabled_capabilities:
return False
return resolve_model_capability(model, capability, provider)["supported"]
return resolve_model_capability(model, capability, provider, endpoint=endpoint)["supported"]


def require_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai"):
def require_model_capability(model, capability=CHAT_CAPABILITY, provider="aoai", *, endpoint=None):
"""Reject incompatible saved/free-form bindings at an inference boundary."""
if not supports_model_capability(model, capability, provider):
if not supports_model_capability(model, capability, provider, endpoint=endpoint):
definition = get_capability_definition(capability)
raise AIConnectionError(
f"The selected model is not available for {definition.label.lower()}. Choose a compatible model.",
Expand All @@ -264,7 +254,7 @@ def normalize_model_capability_fields(model):
"""Validate new metadata without discarding unrelated, existing model properties."""
normalized = dict(model)
normalized.pop("capability_status", None)
for field in ("supportsChat", "supportsImageGeneration"):
for field in ("supportsChat", "supportsImageGeneration", "supportsImageEditing", "supportsImageMasking"):
if field in normalized and not isinstance(normalized[field], bool):
raise AIConnectionError(f"{field} must be true or false.")
if "enabled_capabilities" in normalized:
Expand All @@ -274,8 +264,10 @@ def normalize_model_capability_fields(model):
):
raise AIConnectionError("Model availability must name implemented AI capabilities.")
normalized["enabled_capabilities"] = list(dict.fromkeys(values))
if normalized.get("image_generation_api") not in (None, "", "images", "responses"):
raise AIConnectionError("The image-generation API must be Images or Responses.")
if normalized.get("image_generation_api") not in (None, "", *IMAGE_APIS):
raise AIConnectionError("The image-generation API must name an implemented image operation.")
if normalized.get("supportsImageMasking") is True and normalized.get("supportsImageEditing") is False:
raise AIConnectionError("Masked image editing requires image-editing support.")
return normalized


Expand All @@ -291,7 +283,7 @@ def filter_model_endpoints_by_capability(endpoints, capability=CHAT_CAPABILITY,
for model in endpoint.get("models") or []
if isinstance(model, Mapping)
and endpoint.get("enabled") is not False
and supports_model_capability(model, capability, endpoint.get("provider"))
and supports_model_capability(model, capability, endpoint.get("provider"), endpoint=endpoint)
]
if models or preserve_empty:
projected = copy.deepcopy(endpoint)
Expand Down Expand Up @@ -326,6 +318,11 @@ def resolve_capability_model_selection(selection, endpoints, capability=CHAT_CAP
)
if endpoint is None or endpoint.get("enabled") is False:
return dict(EMPTY_MODEL_SELECTION), f"{definition.label} connection is unavailable. Select another connection."
if (
capability == IMAGE_GENERATION_CAPABILITY and normalized["provider"]
and normalized["provider"] != str(endpoint.get("provider") or "aoai").lower()
):
return dict(EMPTY_MODEL_SELECTION), "The selected image model does not belong to that provider."
model = next(
(
item for item in endpoint.get("models") or []
Expand All @@ -334,7 +331,7 @@ def resolve_capability_model_selection(selection, endpoints, capability=CHAT_CAP
),
None,
)
if model is None or not supports_model_capability(model, capability, endpoint.get("provider")):
if model is None or not supports_model_capability(model, capability, endpoint.get("provider"), endpoint=endpoint):
return dict(EMPTY_MODEL_SELECTION), f"{definition.label} model is unavailable or incompatible. Select a compatible model."
normalized["provider"] = str(endpoint.get("provider") or "aoai").lower()
return normalized, None
Expand Down Expand Up @@ -383,29 +380,47 @@ def build_capability_model_catalog(endpoints, capability=CHAT_CAPABILITY):
model_id = str(model.get("id") or model.get("deploymentName") or "")
if not model_id:
continue
request_model = str(model.get("deploymentName") or model.get("deployment") or model_id)
if endpoint.get("provider") == "custom":
# The optional Custom registry owns model-name versus deployment-name semantics.
from functions_model_endpoint_types import resolve_model_endpoint_request_model

request_model = resolve_model_endpoint_request_model(endpoint, model)
support = (
resolve_image_model_capability(model, endpoint, endpoint.get("provider"))
if capability == IMAGE_GENERATION_CAPABILITY
else resolve_model_capability(model, capability, endpoint.get("provider"), endpoint=endpoint)
)
support.pop("connection_provider", None)
choices.append({
"endpoint_id": endpoint_id,
"model_id": model_id,
"provider": str(endpoint.get("provider") or "aoai"),
"connection_name": str(endpoint.get("name") or "Connection"),
"label": str(model.get("displayName") or model.get("modelName") or model.get("deploymentName") or model_id),
"deployment_name": str(model.get("deploymentName") or model.get("deployment") or model_id),
"capability": resolve_model_capability(model, capability, endpoint.get("provider")),
"deployment_name": request_model,
"capability": support,
})
return sorted(choices, key=lambda item: (item["connection_name"].lower(), item["label"].lower(), item["endpoint_id"], item["model_id"]))


def describe_model_capabilities(model, provider="aoai"):
def describe_model_capabilities(model, provider="aoai", *, endpoint=None):
"""Public technical support and publication metadata, never connection secrets."""
result = {}
for key in CAPABILITY_DEFINITIONS:
support = (
resolve_image_model_capability(model, endpoint, provider)
if key == IMAGE_GENERATION_CAPABILITY
else resolve_model_capability(model, key, provider, endpoint=endpoint)
)
support.pop("connection_provider", None)
result[key] = {
**resolve_model_capability(model, key, provider),
"available": supports_model_capability(model, key, provider),
**support,
"available": supports_model_capability(model, key, provider, endpoint=endpoint),
}
supports_vision, source = resolve_model_vision_support(model)
result["vision"] = {
"supported": supports_vision and resolve_model_capability(model, CHAT_CAPABILITY, provider)["supported"],
"supported": supports_vision and resolve_model_capability(model, CHAT_CAPABILITY, provider, endpoint=endpoint)["supported"],
"source": source,
}
return result
7 changes: 5 additions & 2 deletions application/single_app/functions_documents.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
from functions_keyvault import SecretReturnType, keyvault_model_endpoint_get_helper
from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers
from functions_model_endpoint_runtime import MODEL_ENDPOINT_PROVIDER_ALLOWLIST, build_model_endpoint_sync_chat_client
from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model
from model_endpoint_clients import MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, infer_model_endpoint_protocol
import azure.cognitiveservices.speech as speechsdk

_AUDIO_RUNTIME_CAPABILITIES_CACHE = None
Expand Down Expand Up @@ -186,13 +188,14 @@ def _resolve_metadata_extraction_client(settings, identity_context=None):

connection = endpoint_cfg.get("connection", {}) or {}
auth_settings = endpoint_cfg.get("auth", {}) or {}
deployment = str(model_cfg.get("deploymentName") or model_cfg.get("deployment") or "").strip()
deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg)
endpoint = str(connection.get("endpoint") or "").strip()
api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip()

if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST:
raise ValueError(f"Selected metadata extraction provider '{provider}' is not supported.")
if not endpoint or not api_version or not deployment:
protocol = infer_model_endpoint_protocol(provider, endpoint, deployment, get_model_endpoint_api_type(endpoint_cfg))
if not endpoint or not deployment or (protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version):
raise ValueError("Selected metadata extraction endpoint is missing endpoint, API version, or deployment configuration.")

return _build_model_endpoint_client(
Expand Down
Loading