From 2c1173c2af1d2363cbe10aac167ac08c042833be Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 15 Sep 2026 15:55:49 -0230 Subject: [PATCH 01/18] Record loaded model packages for HTTP serving --- inference/core/managers/base.py | 8 +- .../core/models/inference_models_adapters.py | 28 ++-- inference_models/inference_models/entities.py | 9 ++ .../models/auto_loaders/core.py | 21 +++ .../test_resolved_model_metadata.py | 132 ++++++++++++++++++ 5 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py diff --git a/inference/core/managers/base.py b/inference/core/managers/base.py index c51c785c1b..7dc2797664 100644 --- a/inference/core/managers/base.py +++ b/inference/core/managers/base.py @@ -518,12 +518,18 @@ def infer_from_request_sync( async def model_infer(self, model_id: str, request: InferenceRequest, **kwargs): model = self._get_model_reference(model_id=model_id) - return model.infer_from_request(request) + return self._infer_from_model(model, request) def model_infer_sync( self, model_id: str, request: InferenceRequest, **kwargs ) -> Union[List[InferenceResponse], InferenceResponse]: model = self._get_model_reference(model_id=model_id) + return self._infer_from_model(model, request) + + @staticmethod + def _infer_from_model( + model: Model, request: InferenceRequest + ) -> Union[List[InferenceResponse], InferenceResponse]: return model.infer_from_request(request) def run_tensor_native_inference(self, model_id: str, **kwargs) -> Any: diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index dbf0735870..f7ac1acdbc 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -6,7 +6,7 @@ from io import BytesIO from threading import local from time import perf_counter -from typing import Any, Deque, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Deque, List, Optional, Tuple, Union from uuid import uuid4 from weakref import finalize @@ -255,7 +255,19 @@ def _fixed_input_hw_from_backend(backend: Any) -> Optional[Tuple[int, int]]: return height, width -class InferenceModelsObjectDetectionAdapter(Model): +if TYPE_CHECKING: + from inference_models.entities import ResolvedModelMetadata + + +class InferenceModelsAdapter(Model): + _model: Any + + @property + def resolved_model(self) -> Optional["ResolvedModelMetadata"]: + return getattr(self._model, "resolved_model", None) + + +class InferenceModelsObjectDetectionAdapter(InferenceModelsAdapter): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -421,7 +433,7 @@ def draw_predictions( ) -class InferenceModelsInstanceSegmentationAdapter(Model): +class InferenceModelsInstanceSegmentationAdapter(InferenceModelsAdapter): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1134,7 +1146,7 @@ def rle_masks2poly(masks: InstancesRLEMasks) -> List[np.ndarray]: return segments -class InferenceModelsKeyPointsDetectionAdapter(Model): +class InferenceModelsKeyPointsDetectionAdapter(InferenceModelsAdapter): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1355,7 +1367,7 @@ def model_keypoints_to_response( return results -class InferenceModelsClassificationAdapter(Model): +class InferenceModelsClassificationAdapter(InferenceModelsAdapter): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1695,7 +1707,7 @@ def draw_predictions(inference_request, inference_response, class_names: List[st return buffered.getvalue() -class InferenceModelsSemanticSegmentationAdapter(Model): +class InferenceModelsSemanticSegmentationAdapter(InferenceModelsAdapter): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1853,7 +1865,7 @@ def draw_predictions( ) -class InferenceModelsDepthEstimationAdapter(Model): +class InferenceModelsDepthEstimationAdapter(InferenceModelsAdapter): """Serves any `inference_models` DepthEstimationModel (e.g. YOLO26-depth) behind the depth-estimation contract shared with the DepthAnything adapters: per-image min-max-normalized depth plus a viridis @@ -1976,7 +1988,7 @@ def clear_cache(self, delete_from_disk: bool = True) -> None: pass -class InferenceModelsActionRecognitionAdapter(Model): +class InferenceModelsActionRecognitionAdapter(InferenceModelsAdapter): """Serves a clip to an action recognition model, one window at a time. The model declares how a clip is cut and sampled, so a caller never states diff --git a/inference_models/inference_models/entities.py b/inference_models/inference_models/entities.py index dd3278118f..6f2a86ead8 100644 --- a/inference_models/inference_models/entities.py +++ b/inference_models/inference_models/entities.py @@ -1,6 +1,15 @@ from collections import namedtuple +from dataclasses import dataclass from typing import Literal, Union ImageDimensions = namedtuple("ImageDimensions", ["height", "width"]) ColorFormat = Literal["rgb", "bgr"] Confidence = Union[float, Literal["best", "default"]] + + +@dataclass(frozen=True) +class ResolvedModelMetadata: + model_id: str + model_package_id: str + backend: str + quantization: str diff --git a/inference_models/inference_models/models/auto_loaders/core.py b/inference_models/inference_models/models/auto_loaders/core.py index ccc1ac105f..e1632f8fd1 100644 --- a/inference_models/inference_models/models/auto_loaders/core.py +++ b/inference_models/inference_models/models/auto_loaders/core.py @@ -26,6 +26,7 @@ OFFLINE_MODE_WARM_UP, ROBOFLOW_API_KEY, ) +from inference_models.entities import ResolvedModelMetadata from inference_models.errors import ( CorruptedModelPackageError, DirectLocalStorageAccessError, @@ -1671,6 +1672,10 @@ def from_pretrained( - TextImageEmbeddingModel: For vision-language embeddings (CLIP, etc.) - OpenVocabularyObjectDetectionModel: For open-vocabulary detection + Models loaded from registered packages expose `resolved_model` with the + canonical model ID, package ID, backend, and package quantization. + Direct local-path loads do not infer a package ID. + Raises: UnauthorizedModelAccessError: If API key is invalid or model access is denied. ModelPackageNotFoundError: If no compatible model package is found for your @@ -2419,6 +2424,16 @@ def dependency_directory_pointer(model_dir: str) -> None: model_init_kwargs=model_init_kwargs, ), ) + if ( + cache_entry.canonical_model_id is not None + and cache_entry.backend_type is not None + ): + model.resolved_model = ResolvedModelMetadata( + model_id=cache_entry.canonical_model_id, + model_package_id=cache_entry.model_package_id, + backend=cache_entry.backend_type.value, + quantization=package_config.quantization or Quantization.UNKNOWN.value, + ) if point_model_directory: point_model_directory(model_package_cache_dir) verbose_info( @@ -3216,6 +3231,12 @@ def initialize_model( model_init_kwargs=model_init_kwargs, ), ) + model.resolved_model = ResolvedModelMetadata( + model_id=model_id, + model_package_id=model_package.package_id, + backend=model_package.backend.value, + quantization=quantization, + ) if OFFLINE_MODE: # Read-only leg: nothing was downloaded and the manifest is not # rewritten, so there is no mutation to guard. diff --git a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py new file mode 100644 index 0000000000..fd165c3e7f --- /dev/null +++ b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py @@ -0,0 +1,132 @@ +from dataclasses import asdict +from types import SimpleNamespace + +import pytest + +from inference_models.entities import ResolvedModelMetadata +from inference_models.models.auto_loaders import ( + auto_resolution_cache, + core, + model_cache_paths, +) +from inference_models.models.auto_loaders.entities import BackendType +from inference_models.weights_providers.entities import ( + ModelMetadata, + ModelPackageMetadata, + ONNXPackageDetails, + Quantization, +) + + +@pytest.mark.parametrize("cache_has_backend", [True, False]) +def test_auto_model_reports_canonical_package_after_fresh_and_cached_loads( + tmp_path, monkeypatch, cache_has_backend +): + monkeypatch.setattr(model_cache_paths, "INFERENCE_HOME", str(tmp_path)) + monkeypatch.setattr(auto_resolution_cache, "INFERENCE_HOME", str(tmp_path)) + package = ModelPackageMetadata( + package_id="onnxpackage", + backend=BackendType.ONNX, + quantization=Quantization.FP32, + package_artefacts=[], + onnx_package_details=ONNXPackageDetails(opset=17), + trusted_source=True, + ) + metadata = ModelMetadata( + model_id="canonical/1", + model_architecture="yolov8", + task_type="object-detection", + model_packages=[package], + ) + monkeypatch.setattr(core, "get_model_from_provider", lambda **kwargs: metadata) + model_class = SimpleNamespace( + from_pretrained=lambda *args, **kwargs: SimpleNamespace() + ) + monkeypatch.setattr(core, "resolve_model_class", lambda **kwargs: model_class) + + first = core.AutoModel.from_pretrained("alias/1", backend="onnx", device="cpu") + + first_metadata = getattr(first, "resolved_model", None) + assert isinstance(first_metadata, ResolvedModelMetadata) + assert asdict(first_metadata) == { + "model_id": "canonical/1", + "model_package_id": "onnxpackage", + "backend": "onnx", + "quantization": "fp32", + } + + def unavailable_provider(**kwargs): + raise AssertionError("A cached load must not fetch model metadata") + + monkeypatch.setattr(core, "get_model_from_provider", unavailable_provider) + + class LegacyMetadataCache(auto_resolution_cache.BaseAutoLoadMetadataCache): + def retrieve(self, auto_negotiation_hash): + entry = super().retrieve(auto_negotiation_hash) + if entry is not None and not cache_has_backend: + return entry.model_copy(update={"backend_type": None}) + return entry + + second = core.AutoModel.from_pretrained( + "alias/1", + backend="onnx", + device="cpu", + auto_resolution_cache=LegacyMetadataCache(file_lock_acquire_timeout=1), + ) + assert second is not first + second_metadata = getattr(second, "resolved_model", None) + if cache_has_backend: + assert second_metadata == first_metadata + else: + assert second_metadata is None + + +def test_auto_model_reports_the_successful_fallback_package(tmp_path, monkeypatch): + monkeypatch.setattr(model_cache_paths, "INFERENCE_HOME", str(tmp_path)) + monkeypatch.setattr(auto_resolution_cache, "INFERENCE_HOME", str(tmp_path)) + packages = [ + ModelPackageMetadata( + package_id=package_id, + backend=BackendType.ONNX, + quantization=quantization, + package_artefacts=[], + onnx_package_details=ONNXPackageDetails(opset=17), + trusted_source=True, + ) + for package_id, quantization in [ + ("brokenfp16", Quantization.FP16), + ("workingfp32", Quantization.FP32), + ] + ] + metadata = ModelMetadata( + model_id="canonical/1", + model_architecture="yolov8", + task_type="object-detection", + model_packages=packages, + ) + monkeypatch.setattr(core, "get_model_from_provider", lambda **kwargs: metadata) + attempted_packages = [] + + def load_package(package_path, **kwargs): + package_id = package_path.rsplit("/", 1)[-1] + attempted_packages.append(package_id) + if package_id == "brokenfp16": + raise RuntimeError("Unsupported package") + return SimpleNamespace() + + model_class = SimpleNamespace(from_pretrained=load_package) + monkeypatch.setattr(core, "resolve_model_class", lambda **kwargs: model_class) + + model = core.AutoModel.from_pretrained( + "alias/1", backend="onnx", quantization=["fp16", "fp32"], device="cpu" + ) + + assert attempted_packages == ["brokenfp16", "workingfp32"] + resolved_model = getattr(model, "resolved_model", None) + assert isinstance(resolved_model, ResolvedModelMetadata) + assert asdict(resolved_model) == { + "model_id": "canonical/1", + "model_package_id": "workingfp32", + "backend": "onnx", + "quantization": "fp32", + } From a50c13a3829fd7e84962ad82577f1107abdde3dc Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 15 Sep 2026 15:58:45 -0230 Subject: [PATCH 02/18] Return resolved model metadata in HTTP inference responses --- docs/http-resolved-model.md | 47 ++++ .../core/entities/responses/inference.py | 4 + inference/core/managers/base.py | 29 ++- .../common/inference_response_entities.py | 11 + .../http/test_resolved_model_metadata.py | 224 ++++++++++++++++++ 5 files changed, 314 insertions(+), 1 deletion(-) create mode 100644 docs/http-resolved-model.md create mode 100644 tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py diff --git a/docs/http-resolved-model.md b/docs/http-resolved-model.md new file mode 100644 index 0000000000..db8276f384 --- /dev/null +++ b/docs/http-resolved-model.md @@ -0,0 +1,47 @@ +# Resolved model metadata + +When `USE_INFERENCE_MODELS=true`, standard computer vision HTTP responses include +`resolved_model` if the loaded model has package metadata. The metadata identifies +the model instance that produced that result. + +This requires an `inference-models` release or source installation that provides +resolved package metadata. Older library versions still load, but responses omit +the field. Deploy that library version before the updated HTTP server. + +```json +{ + "image": {"width": 640, "height": 480}, + "predictions": [], + "resolved_model": { + "model_id": "project/3", + "model_package_id": "trtpackage", + "backend": "trt", + "quantization": "fp16" + } +} +``` + +| Field | Meaning | +| --- | --- | +| `model_id` | The canonical model ID returned by the weights provider. An input alias can differ. | +| `model_package_id` | The ID of the package that loaded successfully. | +| `backend` | The backend of that package, such as `onnx` or `trt`. | +| `quantization` | The package quantization, such as `fp32`, `fp16`, or `unknown` when unavailable. | + +Quantization describes the package. It does not describe the input tensor dtype +or guarantee that every runtime operation uses that precision. + +The v0 `/{dataset_id}/{version_id}` endpoint and standard v1 `/infer/...` endpoints +return this metadata. Supported tasks are object detection, classification, +instance segmentation, semantic segmentation, and keypoint detection. JSON results +include the metadata. Responses that contain only a rendered image do not. + +Each result in a v1 batch has its own descriptor. The Python HTTP SDK preserves +this field for both API versions, including batches of inference inputs. + +If a preferred package fails to load and another package succeeds, the response +identifies the successful package. A cache reload preserves its package identity. +An unrelated model removal or replacement does not change metadata for an active request. + +When `USE_INFERENCE_MODELS=false`, responses omit `resolved_model`. Models without +package identity, including direct local-path loads, also omit the field. diff --git a/inference/core/entities/responses/inference.py b/inference/core/entities/responses/inference.py index 8b339ffccf..f98e9b0f01 100644 --- a/inference/core/entities/responses/inference.py +++ b/inference/core/entities/responses/inference.py @@ -10,6 +10,7 @@ InferenceResponse, InferenceResponseImage, InstanceSegmentationInferenceResponse, + ResolvedModel, WithVisualizationResponse, ) from inference.core.workflows.core_steps.common.segmentation_entities import ( # noqa: F401 @@ -232,6 +233,7 @@ class InstanceSegmentationInferenceResponseDC: frame_id: object = None time: object = None visualization: object = None + resolved_model: Optional[ResolvedModel] = None # Internal stream-pipeline fast path: lets workflow execution carry a # response future through Model.infer_from_request without blocking the # inference thread. `_is_response_dc_to_dict` intentionally ignores it. @@ -283,6 +285,8 @@ def _is_response_dc_to_dict(r: InstanceSegmentationInferenceResponseDC) -> dict: d["time"] = r.time if r.visualization is not None: d["visualization"] = r.visualization + if r.resolved_model is not None: + d["resolved_model"] = r.resolved_model.model_dump() return d diff --git a/inference/core/managers/base.py b/inference/core/managers/base.py index 7dc2797664..d511b79216 100644 --- a/inference/core/managers/base.py +++ b/inference/core/managers/base.py @@ -530,7 +530,9 @@ def model_infer_sync( def _infer_from_model( model: Model, request: InferenceRequest ) -> Union[List[InferenceResponse], InferenceResponse]: - return model.infer_from_request(request) + response = model.infer_from_request(request) + attach_resolved_model_metadata(model, response) + return response def run_tensor_native_inference(self, model_id: str, **kwargs) -> Any: with start_span( @@ -870,3 +872,28 @@ def try_releasing_cuda_memory() -> None: pass except Exception as error: logger.warning(f"Attempted to purge CUDA memory but failed with error: {error}") + + +def attach_resolved_model_metadata(model: Model, response: Any) -> None: + if not USE_INFERENCE_MODELS: + return + from inference.core.entities.responses.inference import ( + CvInferenceResponse, + InstanceSegmentationInferenceResponseDC, + ResolvedModel, + ) + + metadata = getattr(model, "resolved_model", None) + fields = { + name: getattr(metadata, name, None) + for name in ("model_id", "model_package_id", "backend", "quantization") + } + if not all(isinstance(value, str) for value in fields.values()): + return + resolved_model = ResolvedModel.model_validate(fields) + responses = response if isinstance(response, list) else [response] + for item in responses: + if isinstance( + item, (CvInferenceResponse, InstanceSegmentationInferenceResponseDC) + ): + item.resolved_model = resolved_model diff --git a/inference/core/workflows/core_steps/common/inference_response_entities.py b/inference/core/workflows/core_steps/common/inference_response_entities.py index 90ac7ea3f4..25d7fe23b8 100644 --- a/inference/core/workflows/core_steps/common/inference_response_entities.py +++ b/inference/core/workflows/core_steps/common/inference_response_entities.py @@ -60,6 +60,13 @@ class InferenceResponse(BaseModel): ) +class ResolvedModel(BaseModel): + model_id: str + model_package_id: str + backend: str + quantization: str + + class CvInferenceResponse(InferenceResponse): """Computer Vision inference response. @@ -68,6 +75,10 @@ class CvInferenceResponse(InferenceResponse): """ image: Union[List[InferenceResponseImage], InferenceResponseImage] + resolved_model: Optional[ResolvedModel] = Field( + default=None, + description="The model package that produced this result. Present when USE_INFERENCE_MODELS is enabled and the loaded model has package metadata. Quantization describes the package, not every runtime operation.", + ) class WithVisualizationResponse(BaseModel): diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py new file mode 100644 index 0000000000..3250b2ced7 --- /dev/null +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -0,0 +1,224 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import numpy as np +import pytest +from starlette.testclient import TestClient + +from inference.core.entities.responses.inference import ( + InferenceResponseImage, + ObjectDetectionInferenceResponse, +) +from inference.core.interfaces.http import http_api +from inference.core.managers import base as manager_module +from inference.core.managers.base import ModelManager +from inference.core.models.base import Model +from inference.core.registries.base import ModelRegistry +from inference_models.entities import ResolvedModelMetadata +from inference_models.utils.content_addressed_artifact_cache import ( + NullContentAddressedArtifactCache, +) +from inference_sdk import InferenceHTTPClient + + +class PackageModel(Model): + task_type = "object-detection" + model_id = "test/1" + resolved_model = ResolvedModelMetadata( + model_id="canonical/1", + model_package_id="onnxpackage", + backend="onnx", + quantization="fp32", + ) + + def infer_from_request(self, request): + response = ObjectDetectionInferenceResponse( + image=InferenceResponseImage(width=640, height=480), predictions=[] + ) + images = getattr(request, "image", None) + if isinstance(images, list): + return [response.model_copy() for _ in images] + return response + + +def build_client(monkeypatch, flag=True, model=None, registry=None): + monkeypatch.setattr(manager_module, "USE_INFERENCE_MODELS", flag) + monkeypatch.setattr(manager_module, "MODELS_CACHE_AUTH_ENABLED", False) + monkeypatch.setattr(manager_module, "DISABLE_INFERENCE_CACHE", True) + monkeypatch.setattr(http_api, "USE_INFERENCE_MODELS", flag) + monkeypatch.setattr(http_api, "MAX_INFERENCE_MODELS_CACHE_SIZE_MB", 0) + monkeypatch.setattr(http_api, "GCP_SERVERLESS", False) + monkeypatch.setattr(http_api, "LAMBDA", False) + monkeypatch.setattr( + http_api, + "InferenceInstrumentator", + lambda *args, **kwargs: SimpleNamespace( + set_stream_manager_client=lambda client: None + ), + ) + monkeypatch.setattr( + http_api.usage_collector, "async_push_usage_payloads", AsyncMock() + ) + manager = ModelManager( + model_registry=registry or ModelRegistry({}), + models={"test/1": model or PackageModel()}, + content_addressed_artifact_cache=NullContentAddressedArtifactCache(), + ) + return TestClient(http_api.HttpInterface(model_manager=manager).app), manager + + +def test_http_inference_reports_the_serving_package(monkeypatch): + client, _ = build_client(monkeypatch) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["resolved_model"] == { + "model_id": "canonical/1", + "model_package_id": "onnxpackage", + "backend": "onnx", + "quantization": "fp32", + } + + +def test_legacy_http_inference_reports_the_serving_package(monkeypatch): + client, _ = build_client(monkeypatch) + + response = client.post( + "/test/1", + content="image", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["resolved_model"]["model_package_id"] == "onnxpackage" + + +def test_http_batch_reports_package_for_each_image(monkeypatch): + client, _ = build_client(monkeypatch) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": [ + {"type": "base64", "value": "first"}, + {"type": "base64", "value": "second"}, + ], + }, + ) + + assert response.status_code == 200, response.text + assert [item["resolved_model"]["model_package_id"] for item in response.json()] == [ + "onnxpackage", + "onnxpackage", + ] + + +def test_http_inference_omits_metadata_when_flag_is_disabled(monkeypatch): + client, _ = build_client(monkeypatch, flag=False) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json() == { + "image": {"width": 640, "height": 480}, + "predictions": [], + } + + +@pytest.mark.parametrize("api_version", ["v0", "v1"]) +def test_sdk_preserves_http_package_metadata(monkeypatch, requests_mock, api_version): + server, _ = build_client(monkeypatch) + sdk = InferenceHTTPClient(api_url="http://testserver", api_key="test") + getattr(sdk, f"select_api_{api_version}")() + + def infer(request, context): + response = server.post( + request.path_url, + content=request.body, + headers={"Content-Type": request.headers["Content-Type"]}, + ) + context.status_code = response.status_code + return response.json() + + requests_mock.get( + "http://testserver/model/registry", + json=server.get("/model/registry").json(), + ) + endpoint = "/test/1" if api_version == "v0" else "/infer/object_detection" + requests_mock.post(f"http://testserver{endpoint}", json=infer) + + response = sdk.infer(np.zeros((480, 640, 3), dtype=np.uint8), model_id="test/1") + + assert isinstance(response, dict) + assert response["resolved_model"] == { + "model_id": "canonical/1", + "model_package_id": "onnxpackage", + "backend": "onnx", + "quantization": "fp32", + } + + +def test_http_response_keeps_serving_package_when_model_is_replaced(monkeypatch): + replacement = PackageModel() + replacement.resolved_model = ResolvedModelMetadata( + model_id="canonical/1", + model_package_id="trtpackage", + backend="trt", + quantization="fp16", + ) + registry = SimpleNamespace( + get_model=lambda *args, **kwargs: lambda **kwargs: replacement + ) + + class EvictedModel(PackageModel): + def infer_from_request(self, request): + manager.remove("test/1", delete_from_disk=False) + manager.add_model("test/1", api_key="test") + return super().infer_from_request(request) + + client, manager = build_client(monkeypatch, model=EvictedModel(), registry=registry) + request = { + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + } + + first = client.post("/infer/object_detection", json=request) + second = client.post("/infer/object_detection", json=request) + + assert first.status_code == 200, first.text + assert second.status_code == 200, second.text + assert first.json()["resolved_model"]["model_package_id"] == "onnxpackage" + assert second.json()["resolved_model"]["model_package_id"] == "trtpackage" + + +def test_http_inference_omits_metadata_with_an_older_model_library(monkeypatch): + from inference_models import entities as model_entities + + monkeypatch.delattr(model_entities, "ResolvedModelMetadata") + monkeypatch.setattr(PackageModel, "resolved_model", None) + client, _ = build_client(monkeypatch) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert "resolved_model" not in response.json() From 718ceab997424219b27a201f3b21a3625376eb4f Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 15 Sep 2026 16:42:43 -0230 Subject: [PATCH 03/18] Attach resolved metadata in model adapters --- docs/http-resolved-model.md | 7 +- inference/core/managers/base.py | 37 +---- .../core/models/inference_models_adapters.py | 39 ++++- .../test_resolved_model_metadata.py | 24 +++- .../http/test_resolved_model_metadata.py | 133 ++++++++++++++---- 5 files changed, 167 insertions(+), 73 deletions(-) diff --git a/docs/http-resolved-model.md b/docs/http-resolved-model.md index db8276f384..5b381852ea 100644 --- a/docs/http-resolved-model.md +++ b/docs/http-resolved-model.md @@ -4,9 +4,10 @@ When `USE_INFERENCE_MODELS=true`, standard computer vision HTTP responses includ `resolved_model` if the loaded model has package metadata. The metadata identifies the model instance that produced that result. -This requires an `inference-models` release or source installation that provides -resolved package metadata. Older library versions still load, but responses omit -the field. Deploy that library version before the updated HTTP server. +Install an `inference-models` release that provides resolved package metadata. +Pin the server dependency to that release before you deploy the updated server. +For source-based testing, install this repository's `inference_models/` package +into the server environment. ```json { diff --git a/inference/core/managers/base.py b/inference/core/managers/base.py index d511b79216..c51c785c1b 100644 --- a/inference/core/managers/base.py +++ b/inference/core/managers/base.py @@ -518,21 +518,13 @@ def infer_from_request_sync( async def model_infer(self, model_id: str, request: InferenceRequest, **kwargs): model = self._get_model_reference(model_id=model_id) - return self._infer_from_model(model, request) + return model.infer_from_request(request) def model_infer_sync( self, model_id: str, request: InferenceRequest, **kwargs ) -> Union[List[InferenceResponse], InferenceResponse]: model = self._get_model_reference(model_id=model_id) - return self._infer_from_model(model, request) - - @staticmethod - def _infer_from_model( - model: Model, request: InferenceRequest - ) -> Union[List[InferenceResponse], InferenceResponse]: - response = model.infer_from_request(request) - attach_resolved_model_metadata(model, response) - return response + return model.infer_from_request(request) def run_tensor_native_inference(self, model_id: str, **kwargs) -> Any: with start_span( @@ -872,28 +864,3 @@ def try_releasing_cuda_memory() -> None: pass except Exception as error: logger.warning(f"Attempted to purge CUDA memory but failed with error: {error}") - - -def attach_resolved_model_metadata(model: Model, response: Any) -> None: - if not USE_INFERENCE_MODELS: - return - from inference.core.entities.responses.inference import ( - CvInferenceResponse, - InstanceSegmentationInferenceResponseDC, - ResolvedModel, - ) - - metadata = getattr(model, "resolved_model", None) - fields = { - name: getattr(metadata, name, None) - for name in ("model_id", "model_package_id", "backend", "quantization") - } - if not all(isinstance(value, str) for value in fields.values()): - return - resolved_model = ResolvedModel.model_validate(fields) - responses = response if isinstance(response, list) else [response] - for item in responses: - if isinstance( - item, (CvInferenceResponse, InstanceSegmentationInferenceResponseDC) - ): - item.resolved_model = resolved_model diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index f7ac1acdbc..f16b927826 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -6,7 +6,7 @@ from io import BytesIO from threading import local from time import perf_counter -from typing import TYPE_CHECKING, Any, Deque, List, Optional, Tuple, Union +from typing import Any, Deque, List, Optional, Tuple, Union from uuid import uuid4 from weakref import finalize @@ -28,6 +28,7 @@ ) from inference.core.entities.responses.inference import ( ClassificationInferenceResponse, + CvInferenceResponse, InferenceResponse, InferenceResponseImage, InferenceResponseImageDC, @@ -45,6 +46,7 @@ ObjectDetectionPrediction, Point, PointDC, + ResolvedModel, SemanticSegmentationInferenceResponse, SemanticSegmentationPrediction, ) @@ -56,6 +58,7 @@ GCP_SERVERLESS, MAX_VIDEO_DURATION_SECONDS, RFDETR_ONNX_MAX_RESOLUTION, + USE_INFERENCE_MODELS, VALID_INFERENCE_MODELS_BACKENDS, WORKFLOWS_ASYNC_FUTURE_RESULT_TIMEOUT, ) @@ -102,6 +105,7 @@ MAX_RFDETR_PIPELINE_DEPTH, get_rfdetr_pipeline_depth, ) +from inference_models.entities import ResolvedModelMetadata from inference_models.models.base.action_recognition import ( ActionRecognitionModel, effective_max_frame_side, @@ -255,17 +259,39 @@ def _fixed_input_hw_from_backend(backend: Any) -> Optional[Tuple[int, int]]: return height, width -if TYPE_CHECKING: - from inference_models.entities import ResolvedModelMetadata - - class InferenceModelsAdapter(Model): _model: Any @property - def resolved_model(self) -> Optional["ResolvedModelMetadata"]: + def resolved_model(self) -> Optional[ResolvedModelMetadata]: return getattr(self._model, "resolved_model", None) + def infer_from_request( + self, request: InferenceRequest + ) -> Union[List[InferenceResponse], InferenceResponse]: + responses = super().infer_from_request(request) + self._attach_resolved_model_metadata(responses) + return responses + + def _attach_resolved_model_metadata(self, responses: Any) -> None: + if not USE_INFERENCE_MODELS: + return + metadata = self.resolved_model + if metadata is None: + return + resolved_model = ResolvedModel( + model_id=metadata.model_id, + model_package_id=metadata.model_package_id, + backend=metadata.backend, + quantization=metadata.quantization, + ) + responses = responses if isinstance(responses, list) else [responses] + for response in responses: + if isinstance( + response, (CvInferenceResponse, InstanceSegmentationInferenceResponseDC) + ): + response.resolved_model = resolved_model + class InferenceModelsObjectDetectionAdapter(InferenceModelsAdapter): def __init__(self, model_id: str, api_key: str = None, **kwargs): @@ -1520,6 +1546,7 @@ def infer_from_request( if not isinstance(request.image, list): responses = responses[0] + self._attach_resolved_model_metadata(responses) return responses diff --git a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py index fd165c3e7f..69e82c9776 100644 --- a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py +++ b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py @@ -1,10 +1,12 @@ -from dataclasses import asdict +from dataclasses import asdict, replace from types import SimpleNamespace import pytest +from packaging.version import Version from inference_models.entities import ResolvedModelMetadata from inference_models.models.auto_loaders import ( + auto_negotiation, auto_resolution_cache, core, model_cache_paths, @@ -18,6 +20,22 @@ ) +@pytest.fixture(autouse=True) +def available_onnx_backend(monkeypatch): + runtime = replace( + core.x_ray_runtime_environment(), + onnxruntime_version=Version("1.20.1"), + available_onnx_execution_providers={"CPUExecutionProvider"}, + ) + monkeypatch.setattr(core, "x_ray_runtime_environment", lambda: runtime) + monkeypatch.setattr(auto_negotiation, "x_ray_runtime_environment", lambda: runtime) + monkeypatch.setattr( + auto_negotiation, + "get_selected_onnx_execution_providers", + lambda: ["CPUExecutionProvider"], + ) + + @pytest.mark.parametrize("cache_has_backend", [True, False]) def test_auto_model_reports_canonical_package_after_fresh_and_cached_loads( tmp_path, monkeypatch, cache_has_backend @@ -60,7 +78,7 @@ def unavailable_provider(**kwargs): monkeypatch.setattr(core, "get_model_from_provider", unavailable_provider) - class LegacyMetadataCache(auto_resolution_cache.BaseAutoLoadMetadataCache): + class PartialMetadataCache(auto_resolution_cache.BaseAutoLoadMetadataCache): def retrieve(self, auto_negotiation_hash): entry = super().retrieve(auto_negotiation_hash) if entry is not None and not cache_has_backend: @@ -71,7 +89,7 @@ def retrieve(self, auto_negotiation_hash): "alias/1", backend="onnx", device="cpu", - auto_resolution_cache=LegacyMetadataCache(file_lock_acquire_timeout=1), + auto_resolution_cache=PartialMetadataCache(file_lock_acquire_timeout=1), ) assert second is not first second_metadata = getattr(second, "resolved_model", None) diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py index 3250b2ced7..e16ab50925 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -1,4 +1,5 @@ from types import SimpleNamespace +from typing import Optional from unittest.mock import AsyncMock import numpy as np @@ -12,7 +13,7 @@ from inference.core.interfaces.http import http_api from inference.core.managers import base as manager_module from inference.core.managers.base import ModelManager -from inference.core.models.base import Model +from inference.core.models import inference_models_adapters as adapters from inference.core.registries.base import ModelRegistry from inference_models.entities import ResolvedModelMetadata from inference_models.utils.content_addressed_artifact_cache import ( @@ -20,28 +21,33 @@ ) from inference_sdk import InferenceHTTPClient +RESOLVED_MODEL = ResolvedModelMetadata( + model_id="canonical/1", + model_package_id="onnxpackage", + backend="onnx", + quantization="fp32", +) + -class PackageModel(Model): +class PackageModel(adapters.InferenceModelsAdapter): task_type = "object-detection" model_id = "test/1" - resolved_model = ResolvedModelMetadata( - model_id="canonical/1", - model_package_id="onnxpackage", - backend="onnx", - quantization="fp32", - ) - def infer_from_request(self, request): + def __init__(self, metadata: Optional[ResolvedModelMetadata] = RESOLVED_MODEL): + self._model = SimpleNamespace() + if metadata is not None: + self._model.resolved_model = metadata + + def infer(self, image, **kwargs): response = ObjectDetectionInferenceResponse( image=InferenceResponseImage(width=640, height=480), predictions=[] ) - images = getattr(request, "image", None) - if isinstance(images, list): - return [response.model_copy() for _ in images] - return response + images = image if isinstance(image, list) else [image] + return [response.model_copy() for _ in images] def build_client(monkeypatch, flag=True, model=None, registry=None): + monkeypatch.setattr(adapters, "USE_INFERENCE_MODELS", flag) monkeypatch.setattr(manager_module, "USE_INFERENCE_MODELS", flag) monkeypatch.setattr(manager_module, "MODELS_CACHE_AUTH_ENABLED", False) monkeypatch.setattr(manager_module, "DISABLE_INFERENCE_CACHE", True) @@ -133,7 +139,10 @@ def test_http_inference_omits_metadata_when_flag_is_disabled(monkeypatch): ) assert response.status_code == 200, response.text - assert response.json() == { + payload = response.json() + assert payload.pop("time") >= 0 + payload.pop("inference_id") + assert payload == { "image": {"width": 640, "height": 480}, "predictions": [], } @@ -173,12 +182,13 @@ def infer(request, context): def test_http_response_keeps_serving_package_when_model_is_replaced(monkeypatch): - replacement = PackageModel() - replacement.resolved_model = ResolvedModelMetadata( - model_id="canonical/1", - model_package_id="trtpackage", - backend="trt", - quantization="fp16", + replacement = PackageModel( + ResolvedModelMetadata( + model_id="canonical/1", + model_package_id="trtpackage", + backend="trt", + quantization="fp16", + ) ) registry = SimpleNamespace( get_model=lambda *args, **kwargs: lambda **kwargs: replacement @@ -205,12 +215,10 @@ def infer_from_request(self, request): assert second.json()["resolved_model"]["model_package_id"] == "trtpackage" -def test_http_inference_omits_metadata_with_an_older_model_library(monkeypatch): - from inference_models import entities as model_entities - - monkeypatch.delattr(model_entities, "ResolvedModelMetadata") - monkeypatch.setattr(PackageModel, "resolved_model", None) - client, _ = build_client(monkeypatch) +def test_http_inference_omits_metadata_for_a_model_without_package_identity( + monkeypatch, +): + client, _ = build_client(monkeypatch, model=PackageModel(metadata=None)) response = client.post( "/infer/object_detection", @@ -222,3 +230,76 @@ def test_http_inference_omits_metadata_with_an_older_model_library(monkeypatch): assert response.status_code == 200, response.text assert "resolved_model" not in response.json() + + +def test_direct_adapter_response_reports_its_package(monkeypatch): + from inference.core.entities.requests.inference import ( + InferenceRequestImage, + ObjectDetectionInferenceRequest, + ) + + monkeypatch.setattr(adapters, "USE_INFERENCE_MODELS", True) + response = PackageModel().infer_from_request( + ObjectDetectionInferenceRequest( + id="direct-request", + model_id="test/1", + image=InferenceRequestImage(type="base64", value="image"), + ) + ) + + assert isinstance(response, ObjectDetectionInferenceResponse) + assert response.resolved_model is not None + assert response.resolved_model.model_package_id == "onnxpackage" + + +def test_classification_adapter_reports_package_with_predictions(monkeypatch): + import base64 + + import cv2 + import torch + + from inference.core.entities.requests.inference import ( + ClassificationInferenceRequest, + InferenceRequestImage, + ) + from inference.core.entities.responses.inference import ( + ClassificationInferenceResponse, + ) + from inference_models import ClassificationPrediction + + class ClassificationBackend: + class_names = ["cat", "dog"] + resolved_model = RESOLVED_MODEL + + def pre_process(self, images, **kwargs): + return np.stack(images) + + def forward(self, images, **kwargs): + return torch.tensor([[0.9, 0.1]] * len(images)) + + def post_process(self, predictions, **kwargs): + return ClassificationPrediction( + class_id=predictions.argmax(dim=-1), confidence=predictions + ) + + monkeypatch.setattr(adapters, "USE_INFERENCE_MODELS", True) + monkeypatch.setattr( + adapters.AutoModel, "from_pretrained", lambda **kwargs: ClassificationBackend() + ) + model = adapters.InferenceModelsClassificationAdapter("test/1", api_key="test") + _, image = cv2.imencode(".png", np.zeros((4, 5, 3), dtype=np.uint8)) + + response = model.infer_from_request( + ClassificationInferenceRequest( + id="classification-request", + model_id="test/1", + image=InferenceRequestImage( + type="base64", value=base64.b64encode(image).decode() + ), + ) + ) + + assert isinstance(response, ClassificationInferenceResponse) + assert response.top == "cat" + assert response.resolved_model is not None + assert response.resolved_model.model_package_id == "onnxpackage" From 7378a447104fae108793ca66215861310797e0ac Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 15 Sep 2026 16:55:10 -0230 Subject: [PATCH 04/18] Keep inference-models flag gating at model selection --- .../core/models/inference_models_adapters.py | 3 -- .../http/test_resolved_model_metadata.py | 36 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index f16b927826..e905b4add2 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -58,7 +58,6 @@ GCP_SERVERLESS, MAX_VIDEO_DURATION_SECONDS, RFDETR_ONNX_MAX_RESOLUTION, - USE_INFERENCE_MODELS, VALID_INFERENCE_MODELS_BACKENDS, WORKFLOWS_ASYNC_FUTURE_RESULT_TIMEOUT, ) @@ -274,8 +273,6 @@ def infer_from_request( return responses def _attach_resolved_model_metadata(self, responses: Any) -> None: - if not USE_INFERENCE_MODELS: - return metadata = self.resolved_model if metadata is None: return diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py index e16ab50925..c94a914228 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -1,3 +1,4 @@ +import runpy from types import SimpleNamespace from typing import Optional from unittest.mock import AsyncMock @@ -6,6 +7,7 @@ import pytest from starlette.testclient import TestClient +from inference.core import env from inference.core.entities.responses.inference import ( InferenceResponseImage, ObjectDetectionInferenceResponse, @@ -14,7 +16,10 @@ from inference.core.managers import base as manager_module from inference.core.managers.base import ModelManager from inference.core.models import inference_models_adapters as adapters +from inference.core.registries import roboflow as registry_module from inference.core.registries.base import ModelRegistry +from inference.models import utils as model_utils +from inference.models.yolov8.yolov8_object_detection import YOLOv8ObjectDetection from inference_models.entities import ResolvedModelMetadata from inference_models.utils.content_addressed_artifact_cache import ( NullContentAddressedArtifactCache, @@ -46,8 +51,7 @@ def infer(self, image, **kwargs): return [response.model_copy() for _ in images] -def build_client(monkeypatch, flag=True, model=None, registry=None): - monkeypatch.setattr(adapters, "USE_INFERENCE_MODELS", flag) +def build_client(monkeypatch, flag=True, model=None, registry=None, preload=True): monkeypatch.setattr(manager_module, "USE_INFERENCE_MODELS", flag) monkeypatch.setattr(manager_module, "MODELS_CACHE_AUTH_ENABLED", False) monkeypatch.setattr(manager_module, "DISABLE_INFERENCE_CACHE", True) @@ -67,7 +71,7 @@ def build_client(monkeypatch, flag=True, model=None, registry=None): ) manager = ModelManager( model_registry=registry or ModelRegistry({}), - models={"test/1": model or PackageModel()}, + models={"test/1": model or PackageModel()} if preload else {}, content_addressed_artifact_cache=NullContentAddressedArtifactCache(), ) return TestClient(http_api.HttpInterface(model_manager=manager).app), manager @@ -128,7 +132,25 @@ def test_http_batch_reports_package_for_each_image(monkeypatch): def test_http_inference_omits_metadata_when_flag_is_disabled(monkeypatch): - client, _ = build_client(monkeypatch, flag=False) + monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) + model_types = runpy.run_path(model_utils.__file__)["ROBOFLOW_MODEL_TYPES"] + monkeypatch.setattr( + registry_module, + "get_model_type", + lambda *args, **kwargs: ("object-detection", "yolov8"), + ) + + def initialize_legacy_model(self, model_id, **kwargs): + self.model_id = model_id + + monkeypatch.setattr(YOLOv8ObjectDetection, "__init__", initialize_legacy_model) + monkeypatch.setattr(YOLOv8ObjectDetection, "infer", PackageModel.infer) + client, manager = build_client( + monkeypatch, + flag=False, + registry=registry_module.RoboflowModelRegistry(model_types), + preload=False, + ) response = client.post( "/infer/object_detection", @@ -139,6 +161,7 @@ def test_http_inference_omits_metadata_when_flag_is_disabled(monkeypatch): ) assert response.status_code == 200, response.text + assert isinstance(manager.models()["test/1"], YOLOv8ObjectDetection) payload = response.json() assert payload.pop("time") >= 0 payload.pop("inference_id") @@ -232,13 +255,13 @@ def test_http_inference_omits_metadata_for_a_model_without_package_identity( assert "resolved_model" not in response.json() -def test_direct_adapter_response_reports_its_package(monkeypatch): +def test_direct_adapter_response_reports_its_package_with_flag_disabled(monkeypatch): from inference.core.entities.requests.inference import ( InferenceRequestImage, ObjectDetectionInferenceRequest, ) - monkeypatch.setattr(adapters, "USE_INFERENCE_MODELS", True) + monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) response = PackageModel().infer_from_request( ObjectDetectionInferenceRequest( id="direct-request", @@ -282,7 +305,6 @@ def post_process(self, predictions, **kwargs): class_id=predictions.argmax(dim=-1), confidence=predictions ) - monkeypatch.setattr(adapters, "USE_INFERENCE_MODELS", True) monkeypatch.setattr( adapters.AutoModel, "from_pretrained", lambda **kwargs: ClassificationBackend() ) From 9bcbc613929c2a045399c22c43bdeacbb9584183 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Wed, 16 Sep 2026 11:59:39 -0230 Subject: [PATCH 05/18] Support the pinned inference-models release without metadata --- .github/workflows/unit_tests_inference_x86.yml | 1 - docs/http-resolved-model.md | 5 +++-- inference/core/models/inference_models_adapters.py | 3 +-- .../core/interfaces/http/test_resolved_model_metadata.py | 7 +++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit_tests_inference_x86.yml b/.github/workflows/unit_tests_inference_x86.yml index 02eba2ae58..aea6fd9e3f 100644 --- a/.github/workflows/unit_tests_inference_x86.yml +++ b/.github/workflows/unit_tests_inference_x86.yml @@ -52,7 +52,6 @@ jobs: pip install --upgrade setuptools # Composition-root tests also exercise the CLI entry points. pip install -r requirements/requirements.cli.txt -r requirements/_requirements.txt -r requirements/requirements.cpu.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt -r requirements/requirements.http.txt -r requirements/requirements.transformers.txt - pip install --no-deps ./inference_models - name: 🧪 Unit Tests of Inference timeout-minutes: 30 run: python -m pytest tests/inference/unit_tests tests/unit/core/cache/test_air_gapped.py diff --git a/docs/http-resolved-model.md b/docs/http-resolved-model.md index 5b381852ea..dc1e8aacff 100644 --- a/docs/http-resolved-model.md +++ b/docs/http-resolved-model.md @@ -4,8 +4,9 @@ When `USE_INFERENCE_MODELS=true`, standard computer vision HTTP responses includ `resolved_model` if the loaded model has package metadata. The metadata identifies the model instance that produced that result. -Install an `inference-models` release that provides resolved package metadata. -Pin the server dependency to that release before you deploy the updated server. +To receive metadata, install an `inference-models` release that provides it +and update the server dependency pin. +With an older library, ordinary inference works and omits `resolved_model`. For source-based testing, install this repository's `inference_models/` package into the server environment. diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index e905b4add2..32580b904b 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -104,7 +104,6 @@ MAX_RFDETR_PIPELINE_DEPTH, get_rfdetr_pipeline_depth, ) -from inference_models.entities import ResolvedModelMetadata from inference_models.models.base.action_recognition import ( ActionRecognitionModel, effective_max_frame_side, @@ -262,7 +261,7 @@ class InferenceModelsAdapter(Model): _model: Any @property - def resolved_model(self) -> Optional[ResolvedModelMetadata]: + def resolved_model(self) -> Any: return getattr(self._model, "resolved_model", None) def infer_from_request( diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py index c94a914228..20426cf54c 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -20,13 +20,12 @@ from inference.core.registries.base import ModelRegistry from inference.models import utils as model_utils from inference.models.yolov8.yolov8_object_detection import YOLOv8ObjectDetection -from inference_models.entities import ResolvedModelMetadata from inference_models.utils.content_addressed_artifact_cache import ( NullContentAddressedArtifactCache, ) from inference_sdk import InferenceHTTPClient -RESOLVED_MODEL = ResolvedModelMetadata( +RESOLVED_MODEL = SimpleNamespace( model_id="canonical/1", model_package_id="onnxpackage", backend="onnx", @@ -38,7 +37,7 @@ class PackageModel(adapters.InferenceModelsAdapter): task_type = "object-detection" model_id = "test/1" - def __init__(self, metadata: Optional[ResolvedModelMetadata] = RESOLVED_MODEL): + def __init__(self, metadata: Optional[SimpleNamespace] = RESOLVED_MODEL): self._model = SimpleNamespace() if metadata is not None: self._model.resolved_model = metadata @@ -206,7 +205,7 @@ def infer(request, context): def test_http_response_keeps_serving_package_when_model_is_replaced(monkeypatch): replacement = PackageModel( - ResolvedModelMetadata( + SimpleNamespace( model_id="canonical/1", model_package_id="trtpackage", backend="trt", From 163c5c759cf9b4943c7c14f24d8a24bd0b9729f2 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Wed, 16 Sep 2026 12:44:15 -0230 Subject: [PATCH 06/18] Address resolved metadata review findings --- .../workflows/unit_tests_inference_x86.yml | 1 + .../common/inference_response_entities.py | 1 + inference_models/docs/changelog.md | 6 ++++ .../models/auto_loaders/core.py | 3 ++ .../test_resolved_model_metadata.py | 29 ++++++++++++------- mkdocs.yml | 2 ++ 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unit_tests_inference_x86.yml b/.github/workflows/unit_tests_inference_x86.yml index aea6fd9e3f..02eba2ae58 100644 --- a/.github/workflows/unit_tests_inference_x86.yml +++ b/.github/workflows/unit_tests_inference_x86.yml @@ -52,6 +52,7 @@ jobs: pip install --upgrade setuptools # Composition-root tests also exercise the CLI entry points. pip install -r requirements/requirements.cli.txt -r requirements/_requirements.txt -r requirements/requirements.cpu.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt -r requirements/requirements.http.txt -r requirements/requirements.transformers.txt + pip install --no-deps ./inference_models - name: 🧪 Unit Tests of Inference timeout-minutes: 30 run: python -m pytest tests/inference/unit_tests tests/unit/core/cache/test_air_gapped.py diff --git a/inference/core/workflows/core_steps/common/inference_response_entities.py b/inference/core/workflows/core_steps/common/inference_response_entities.py index 25d7fe23b8..c8c9ce8a93 100644 --- a/inference/core/workflows/core_steps/common/inference_response_entities.py +++ b/inference/core/workflows/core_steps/common/inference_response_entities.py @@ -61,6 +61,7 @@ class InferenceResponse(BaseModel): class ResolvedModel(BaseModel): + model_config = ConfigDict(protected_namespaces=()) model_id: str model_package_id: str backend: str diff --git a/inference_models/docs/changelog.md b/inference_models/docs/changelog.md index ceda421953..f6267bc69f 100644 --- a/inference_models/docs/changelog.md +++ b/inference_models/docs/changelog.md @@ -2,6 +2,12 @@ ## Unreleased +### Added + +- `AutoModel.from_pretrained` exposes `resolved_model` metadata for the loaded + package: canonical model ID, package ID, backend, and quantization. The public + `ResolvedModelMetadata` entity describes this metadata. + ### Fixed - `YOLONasForObjectDetectionTRT` concatenated TRT outputs on the default CUDA stream without ordering against the post-processing stream, which could yield phantom detections under GPU contention. Concatenation now runs on the inference stream and is synchronised before post-processing. diff --git a/inference_models/inference_models/models/auto_loaders/core.py b/inference_models/inference_models/models/auto_loaders/core.py index e1632f8fd1..94b774bf93 100644 --- a/inference_models/inference_models/models/auto_loaders/core.py +++ b/inference_models/inference_models/models/auto_loaders/core.py @@ -2276,6 +2276,9 @@ def invalidate_cache_entry(reason: str) -> None: ) invalidate_cache_entry("cached requested-model identity does not match") return None + if cache_entry.backend_type is None: + invalidate_cache_entry("cached package backend is missing") + return None if not allow_untrusted_packages and cache_entry.trusted_source is not True: verbose_info( message=( diff --git a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py index 69e82c9776..f44f3781f1 100644 --- a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py +++ b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py @@ -36,9 +36,11 @@ def available_onnx_backend(monkeypatch): ) -@pytest.mark.parametrize("cache_has_backend", [True, False]) +@pytest.mark.parametrize( + "missing_cache_field", [None, "backend_type", "canonical_model_id"] +) def test_auto_model_reports_canonical_package_after_fresh_and_cached_loads( - tmp_path, monkeypatch, cache_has_backend + tmp_path, monkeypatch, missing_cache_field ): monkeypatch.setattr(model_cache_paths, "INFERENCE_HOME", str(tmp_path)) monkeypatch.setattr(auto_resolution_cache, "INFERENCE_HOME", str(tmp_path)) @@ -73,16 +75,19 @@ def test_auto_model_reports_canonical_package_after_fresh_and_cached_loads( "quantization": "fp32", } - def unavailable_provider(**kwargs): - raise AssertionError("A cached load must not fetch model metadata") + refreshed_metadata = [] + + def refresh_provider_metadata(**kwargs): + refreshed_metadata.append(kwargs) + return metadata - monkeypatch.setattr(core, "get_model_from_provider", unavailable_provider) + monkeypatch.setattr(core, "get_model_from_provider", refresh_provider_metadata) class PartialMetadataCache(auto_resolution_cache.BaseAutoLoadMetadataCache): def retrieve(self, auto_negotiation_hash): entry = super().retrieve(auto_negotiation_hash) - if entry is not None and not cache_has_backend: - return entry.model_copy(update={"backend_type": None}) + if entry is not None and missing_cache_field is not None: + return entry.model_copy(update={missing_cache_field: None}) return entry second = core.AutoModel.from_pretrained( @@ -93,10 +98,12 @@ def retrieve(self, auto_negotiation_hash): ) assert second is not first second_metadata = getattr(second, "resolved_model", None) - if cache_has_backend: - assert second_metadata == first_metadata - else: - assert second_metadata is None + assert second_metadata == first_metadata + assert len(refreshed_metadata) == (0 if missing_cache_field is None else 1) + + third = core.AutoModel.from_pretrained("alias/1", backend="onnx", device="cpu") + assert getattr(third, "resolved_model", None) == first_metadata + assert len(refreshed_metadata) == (0 if missing_cache_field is None else 1) def test_auto_model_reports_the_successful_fallback_package(tmp_path, monkeypatch): diff --git a/mkdocs.yml b/mkdocs.yml index d62a0fd7e1..e15d1be276 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -37,6 +37,8 @@ nav: - "Models": "https://docs.roboflow.com/models/supported-models" - "Workflows": "https://docs.roboflow.com/workflows" - "Reference": "https://docs.roboflow.com/reference/inference/choosing-the-right-tool" + - "HTTP API": + - "Resolved model metadata": http-resolved-model.md - "Changelog": "https://github.com/roboflow/inference/releases" theme: From 6c091d2a038a31bd937ff73d5abd0474cc899d03 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Thu, 17 Sep 2026 09:57:23 -0230 Subject: [PATCH 07/18] Expose resolved metadata through the base model and API schemas --- docs/http-resolved-model.md | 49 ---- .../entities/responses/action_recognition.py | 5 +- .../core/entities/responses/inference.py | 2 +- inference/core/entities/responses/ocr.py | 5 +- inference/core/entities/responses/sam.py | 8 +- inference/core/entities/responses/sam2.py | 7 +- inference/core/entities/responses/sam3.py | 6 +- inference/core/interfaces/http/http_api.py | 17 ++ inference/core/models/base.py | 30 +- .../core/models/inference_models_adapters.py | 52 +--- .../common/inference_response_entities.py | 40 ++- .../models/clip/clip_inference_models.py | 1 + .../doctr/doctr_model_inference_models.py | 8 +- .../easy_ocr/easy_ocr_inference_models.py | 6 +- .../grounding_dino_inference_models.py | 1 + .../perception_encoder_inference_models.py | 1 + .../sam/segment_anything_inference_models.py | 7 +- .../segment_anything2_inference_models.py | 10 +- .../segment_anything3_inference_models.py | 4 +- .../visual_segmentation_inference_models.py | 19 +- .../models/trocr/trocr_inference_models.py | 1 + mkdocs.yml | 2 - .../http/test_resolved_model_metadata.py | 92 +++++- .../models/test_action_recognition_adapter.py | 20 ++ ...est_specialized_resolved_model_metadata.py | 275 ++++++++++++++++++ 25 files changed, 528 insertions(+), 140 deletions(-) delete mode 100644 docs/http-resolved-model.md create mode 100644 tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py diff --git a/docs/http-resolved-model.md b/docs/http-resolved-model.md deleted file mode 100644 index dc1e8aacff..0000000000 --- a/docs/http-resolved-model.md +++ /dev/null @@ -1,49 +0,0 @@ -# Resolved model metadata - -When `USE_INFERENCE_MODELS=true`, standard computer vision HTTP responses include -`resolved_model` if the loaded model has package metadata. The metadata identifies -the model instance that produced that result. - -To receive metadata, install an `inference-models` release that provides it -and update the server dependency pin. -With an older library, ordinary inference works and omits `resolved_model`. -For source-based testing, install this repository's `inference_models/` package -into the server environment. - -```json -{ - "image": {"width": 640, "height": 480}, - "predictions": [], - "resolved_model": { - "model_id": "project/3", - "model_package_id": "trtpackage", - "backend": "trt", - "quantization": "fp16" - } -} -``` - -| Field | Meaning | -| --- | --- | -| `model_id` | The canonical model ID returned by the weights provider. An input alias can differ. | -| `model_package_id` | The ID of the package that loaded successfully. | -| `backend` | The backend of that package, such as `onnx` or `trt`. | -| `quantization` | The package quantization, such as `fp32`, `fp16`, or `unknown` when unavailable. | - -Quantization describes the package. It does not describe the input tensor dtype -or guarantee that every runtime operation uses that precision. - -The v0 `/{dataset_id}/{version_id}` endpoint and standard v1 `/infer/...` endpoints -return this metadata. Supported tasks are object detection, classification, -instance segmentation, semantic segmentation, and keypoint detection. JSON results -include the metadata. Responses that contain only a rendered image do not. - -Each result in a v1 batch has its own descriptor. The Python HTTP SDK preserves -this field for both API versions, including batches of inference inputs. - -If a preferred package fails to load and another package succeeds, the response -identifies the successful package. A cache reload preserves its package identity. -An unrelated model removal or replacement does not change metadata for an active request. - -When `USE_INFERENCE_MODELS=false`, responses omit `resolved_model`. Models without -package identity, including direct local-path loads, also omit the field. diff --git a/inference/core/entities/responses/action_recognition.py b/inference/core/entities/responses/action_recognition.py index 709123274f..b8a6b856f4 100644 --- a/inference/core/entities/responses/action_recognition.py +++ b/inference/core/entities/responses/action_recognition.py @@ -1,13 +1,14 @@ from typing import List -from pydantic import BaseModel, Field +from pydantic import Field +from inference.core.entities.responses.inference import InferenceResponse from inference.core.workflows.core_steps.models.roboflow.action_recognition.entities import ( # noqa: F401 ActionRecognitionPrediction, ) -class ActionRecognitionInferenceResponse(BaseModel): +class ActionRecognitionInferenceResponse(InferenceResponse): """Classified ranges covering one clip. Frame indices count from the first frame of the submitted clip, so a diff --git a/inference/core/entities/responses/inference.py b/inference/core/entities/responses/inference.py index f98e9b0f01..d16e53e410 100644 --- a/inference/core/entities/responses/inference.py +++ b/inference/core/entities/responses/inference.py @@ -366,7 +366,7 @@ class FaceDetectionPrediction(ObjectDetectionPrediction): landmarks: Union[List[Point], List[Point3D]] -class DepthEstimationResponse(BaseModel): +class DepthEstimationResponse(InferenceResponse): """Response for depth estimation inference. Attributes: diff --git a/inference/core/entities/responses/ocr.py b/inference/core/entities/responses/ocr.py index ea3f9a3c7d..55d9497fc0 100644 --- a/inference/core/entities/responses/ocr.py +++ b/inference/core/entities/responses/ocr.py @@ -1,14 +1,15 @@ from typing import List, Optional -from pydantic import BaseModel, Field +from pydantic import Field from inference.core.entities.responses.inference import ( + InferenceResponse, InferenceResponseImage, ObjectDetectionPrediction, ) -class OCRInferenceResponse(BaseModel): +class OCRInferenceResponse(InferenceResponse): """ OCR Inference response. diff --git a/inference/core/entities/responses/sam.py b/inference/core/entities/responses/sam.py index f455219ca5..8d595a1098 100644 --- a/inference/core/entities/responses/sam.py +++ b/inference/core/entities/responses/sam.py @@ -1,9 +1,11 @@ from typing import Any, List, Union -from pydantic import BaseModel, Field +from pydantic import Field +from inference.core.entities.responses.inference import InferenceResponse -class SamEmbeddingResponse(BaseModel): + +class SamEmbeddingResponse(InferenceResponse): """SAM embedding response. Attributes: @@ -20,7 +22,7 @@ class SamEmbeddingResponse(BaseModel): ) -class SamSegmentationResponse(BaseModel): +class SamSegmentationResponse(InferenceResponse): """SAM segmentation response. Attributes: diff --git a/inference/core/entities/responses/sam2.py b/inference/core/entities/responses/sam2.py index 04730851ae..4afdc1ffa5 100644 --- a/inference/core/entities/responses/sam2.py +++ b/inference/core/entities/responses/sam2.py @@ -1,13 +1,14 @@ from typing import List -from pydantic import BaseModel, Field +from pydantic import Field +from inference.core.entities.responses.inference import InferenceResponse from inference.core.workflows.core_steps.common.segmentation_entities import ( # noqa: F401 Sam2SegmentationPrediction, ) -class Sam2EmbeddingResponse(BaseModel): +class Sam2EmbeddingResponse(InferenceResponse): """SAM embedding response. Attributes: @@ -21,7 +22,7 @@ class Sam2EmbeddingResponse(BaseModel): ) -class Sam2SegmentationResponse(BaseModel): +class Sam2SegmentationResponse(InferenceResponse): predictions: List[Sam2SegmentationPrediction] = Field() time: float = Field( description="The time in seconds it took to produce the segmentation including preprocessing" diff --git a/inference/core/entities/responses/sam3.py b/inference/core/entities/responses/sam3.py index 846f27a83e..cfbc8678a2 100644 --- a/inference/core/entities/responses/sam3.py +++ b/inference/core/entities/responses/sam3.py @@ -2,8 +2,10 @@ from pydantic import BaseModel, Field +from inference.core.entities.responses.inference import InferenceResponse -class Sam3EmbeddingResponse(BaseModel): + +class Sam3EmbeddingResponse(InferenceResponse): image_id: str = Field(description="Image id embeddings are cached to") time: float = Field( description="The time in seconds it took to produce the embeddings including preprocessing" @@ -33,7 +35,7 @@ class Sam3PromptResult(BaseModel): predictions: List[Sam3SegmentationPrediction] = Field() -class Sam3SegmentationResponse(BaseModel): +class Sam3SegmentationResponse(InferenceResponse): prompt_results: List[Sam3PromptResult] = Field() time: float = Field( description="The time in seconds it took to produce the segmentation including preprocessing" diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index 5cead0018b..2fc4e40865 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -3043,6 +3043,7 @@ def healthz(): @app.post( "/clip/embed_image", response_model=ClipEmbeddingResponse, + response_model_exclude_none=True, summary="CLIP Image Embeddings", description="Run the Open AI CLIP model to embed image data.", ) @@ -3089,6 +3090,7 @@ def clip_embed_image( @app.post( "/clip/embed_text", response_model=ClipEmbeddingResponse, + response_model_exclude_none=True, summary="CLIP Text Embeddings", description="Run the Open AI CLIP model to embed text data.", ) @@ -3135,6 +3137,7 @@ def clip_embed_text( @app.post( "/clip/compare", response_model=ClipCompareResponse, + response_model_exclude_none=True, summary="CLIP Compare", description="Run the Open AI CLIP model to compute similarity scores.", ) @@ -3183,6 +3186,7 @@ def clip_compare( @app.post( "/perception_encoder/embed_image", response_model=PerceptionEncoderEmbeddingResponse, + response_model_exclude_none=True, summary="PE Image Embeddings", description="Run the Meta Perception Encoder model to embed image data.", ) @@ -3229,6 +3233,7 @@ def pe_embed_image( @app.post( "/perception_encoder/embed_text", response_model=PerceptionEncoderEmbeddingResponse, + response_model_exclude_none=True, summary="Perception Encoder Text Embeddings", description="Run the Meta Perception Encoder model to embed text data.", ) @@ -3275,6 +3280,7 @@ def pe_embed_text( @app.post( "/perception_encoder/compare", response_model=PerceptionEncoderCompareResponse, + response_model_exclude_none=True, summary="Perception Encoder Compare", description="Run the Meta Perception Encoder model to compute similarity scores.", ) @@ -3523,6 +3529,7 @@ def easy_ocr_retrieve_text( @app.post( "/sam/embed_image", response_model=SamEmbeddingResponse, + response_model_exclude_none=True, summary="SAM Image Embeddings", description="Run the Meta AI Segmant Anything Model to embed image data.", ) @@ -3574,6 +3581,7 @@ def sam_embed_image( @app.post( "/sam/segment_image", response_model=SamSegmentationResponse, + response_model_exclude_none=True, summary="SAM Image Segmentation", description="Run the Meta AI Segmant Anything Model to generate segmenations for image data.", ) @@ -3627,6 +3635,7 @@ def sam_segment_image( @app.post( "/sam2/embed_image", response_model=Sam2EmbeddingResponse, + response_model_exclude_none=True, summary="SAM2 Image Embeddings", description="Run the Meta AI Segment Anything 2 Model to embed image data.", ) @@ -3668,6 +3677,7 @@ def sam2_embed_image( @app.post( "/sam2/segment_image", response_model=Sam2SegmentationResponse, + response_model_exclude_none=True, summary="SAM2 Image Segmentation", description="Run the Meta AI Segment Anything 2 Model to generate segmenations for image data.", ) @@ -3716,6 +3726,7 @@ def sam2_segment_image( @app.post( "/sam3/embed_image", response_model=Sam3EmbeddingResponse, + response_model_exclude_none=True, summary="Seg preview Image Embeddings", description="Run the Model to embed image data.", ) @@ -3762,6 +3773,7 @@ def sam3_embed_image( @app.post( "/sam3/concept_segment", response_model=Sam3SegmentationResponse, + response_model_exclude_none=True, summary="SAM3 PCS (promptable concept segmentation)", description="Run the SAM3 PCS (promptable concept segmentation) to generate segmentations for image data.", ) @@ -3934,6 +3946,7 @@ def sam3_segment_image( @app.post( "/sam3/visual_segment", response_model=Sam2SegmentationResponse, + response_model_exclude_none=True, summary="SAM3 PVS (promptable visual segmentation)", description="Run the SAM3 PVS (promptable visual segmentation) to generate segmentations for image data.", ) @@ -4256,6 +4269,7 @@ def _infer_depth_estimation( depth_data["normalized_depth"] ) return DepthEstimationResponse( + resolved_model=getattr(response, "resolved_model", None), normalized_depth=serialized_depth, depth_map_format=inference_request.depth_map_format, image=depth_data["image"].base64_image, @@ -4264,6 +4278,7 @@ def _infer_depth_estimation( @app.post( "/infer/depth-estimation", response_model=DepthEstimationResponse, + response_model_exclude_none=True, summary="Depth Estimation", description="Run the depth estimation model to generate a depth map.", ) @@ -4302,6 +4317,7 @@ def depth_estimation( @app.post( "/infer/depth-estimation/{model_id:path}", response_model=DepthEstimationResponse, + response_model_exclude_none=True, summary="Depth Estimation with model ID in path", description="Run depth estimation. Model ID is specified in the URL path and can contain slashes.", ) @@ -4340,6 +4356,7 @@ def depth_estimation_with_model_id( @app.post( "/infer/action_recognition", response_model=ActionRecognitionInferenceResponse, + response_model_exclude_none=True, summary="Action Recognition", description=( "Classify the actions in a video clip. The model states " diff --git a/inference/core/models/base.py b/inference/core/models/base.py index 2be7df64a5..8af26d94b4 100644 --- a/inference/core/models/base.py +++ b/inference/core/models/base.py @@ -5,7 +5,11 @@ from inference.core import logger from inference.core.entities.requests.inference import InferenceRequest -from inference.core.entities.responses.inference import InferenceResponse +from inference.core.entities.responses.inference import ( + InferenceResponse, + InstanceSegmentationInferenceResponseDC, + ResolvedModel, +) from inference.core.env import USE_INFERENCE_MODELS from inference.core.models.types import PreprocessReturnMetadata from inference.core.telemetry import set_span_attribute, start_span @@ -106,6 +110,29 @@ class Model(BaseInference): clear_cache(): Clears any cache if necessary. """ + @property + def resolved_model(self) -> Any: + if not USE_INFERENCE_MODELS: + return None + return getattr(getattr(self, "_model", None), "resolved_model", None) + + def _attach_resolved_model_metadata(self, responses: Any) -> None: + metadata = self.resolved_model + if metadata is None: + return + resolved_model = ResolvedModel( + model_id=metadata.model_id, + model_package_id=metadata.model_package_id, + backend=metadata.backend, + quantization=metadata.quantization, + ) + responses = responses if isinstance(responses, list) else [responses] + for response in responses: + if isinstance( + response, (InferenceResponse, InstanceSegmentationInferenceResponseDC) + ): + response.resolved_model = resolved_model + def log(self, m): """Prints the given message. @@ -181,6 +208,7 @@ def infer_from_request( if not isinstance(request.image, list) and len(responses) > 0: responses = responses[0] + self._attach_resolved_model_metadata(responses) return responses def make_response( diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index 32580b904b..7bba44b435 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -28,7 +28,6 @@ ) from inference.core.entities.responses.inference import ( ClassificationInferenceResponse, - CvInferenceResponse, InferenceResponse, InferenceResponseImage, InferenceResponseImageDC, @@ -46,7 +45,6 @@ ObjectDetectionPrediction, Point, PointDC, - ResolvedModel, SemanticSegmentationInferenceResponse, SemanticSegmentationPrediction, ) @@ -257,39 +255,7 @@ def _fixed_input_hw_from_backend(backend: Any) -> Optional[Tuple[int, int]]: return height, width -class InferenceModelsAdapter(Model): - _model: Any - - @property - def resolved_model(self) -> Any: - return getattr(self._model, "resolved_model", None) - - def infer_from_request( - self, request: InferenceRequest - ) -> Union[List[InferenceResponse], InferenceResponse]: - responses = super().infer_from_request(request) - self._attach_resolved_model_metadata(responses) - return responses - - def _attach_resolved_model_metadata(self, responses: Any) -> None: - metadata = self.resolved_model - if metadata is None: - return - resolved_model = ResolvedModel( - model_id=metadata.model_id, - model_package_id=metadata.model_package_id, - backend=metadata.backend, - quantization=metadata.quantization, - ) - responses = responses if isinstance(responses, list) else [responses] - for response in responses: - if isinstance( - response, (CvInferenceResponse, InstanceSegmentationInferenceResponseDC) - ): - response.resolved_model = resolved_model - - -class InferenceModelsObjectDetectionAdapter(InferenceModelsAdapter): +class InferenceModelsObjectDetectionAdapter(Model): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -455,7 +421,7 @@ def draw_predictions( ) -class InferenceModelsInstanceSegmentationAdapter(InferenceModelsAdapter): +class InferenceModelsInstanceSegmentationAdapter(Model): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1168,7 +1134,7 @@ def rle_masks2poly(masks: InstancesRLEMasks) -> List[np.ndarray]: return segments -class InferenceModelsKeyPointsDetectionAdapter(InferenceModelsAdapter): +class InferenceModelsKeyPointsDetectionAdapter(Model): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1389,7 +1355,7 @@ def model_keypoints_to_response( return results -class InferenceModelsClassificationAdapter(InferenceModelsAdapter): +class InferenceModelsClassificationAdapter(Model): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1730,7 +1696,7 @@ def draw_predictions(inference_request, inference_response, class_names: List[st return buffered.getvalue() -class InferenceModelsSemanticSegmentationAdapter(InferenceModelsAdapter): +class InferenceModelsSemanticSegmentationAdapter(Model): def __init__(self, model_id: str, api_key: str = None, **kwargs): super().__init__() @@ -1888,7 +1854,7 @@ def draw_predictions( ) -class InferenceModelsDepthEstimationAdapter(InferenceModelsAdapter): +class InferenceModelsDepthEstimationAdapter(Model): """Serves any `inference_models` DepthEstimationModel (e.g. YOLO26-depth) behind the depth-estimation contract shared with the DepthAnything adapters: per-image min-max-normalized depth plus a viridis @@ -2011,7 +1977,7 @@ def clear_cache(self, delete_from_disk: bool = True) -> None: pass -class InferenceModelsActionRecognitionAdapter(InferenceModelsAdapter): +class InferenceModelsActionRecognitionAdapter(Model): """Serves a clip to an action recognition model, one window at a time. The model declares how a clip is cut and sampled, so a caller never states @@ -2075,12 +2041,14 @@ def infer_from_request( stride=max(1.0, source_fps / window.sample_fps), ) timeline.sort(key=lambda entry: (entry.start_frame_idx, entry.class_id)) - return ActionRecognitionInferenceResponse( + response = ActionRecognitionInferenceResponse( timeline=timeline, source_fps=source_fps, frame_count=frame_count, windows_classified=windows_classified, ) + self._attach_resolved_model_metadata(response) + return response def preprocess(self, *args, **kwargs): raise NotImplementedError( diff --git a/inference/core/workflows/core_steps/common/inference_response_entities.py b/inference/core/workflows/core_steps/common/inference_response_entities.py index c8c9ce8a93..aa657c9d65 100644 --- a/inference/core/workflows/core_steps/common/inference_response_entities.py +++ b/inference/core/workflows/core_steps/common/inference_response_entities.py @@ -37,6 +37,22 @@ class InferenceResponseImage(BaseModel): ) +class ResolvedModel(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_id: str = Field( + description="Canonical model ID returned by the weights provider. It can differ from the requested alias." + ) + model_package_id: str = Field( + description="ID of the package that loaded successfully and produced this result, including after a loading fallback." + ) + backend: str = Field( + description="Backend of the loaded package, such as onnx, trt, or torch." + ) + quantization: str = Field( + description="Package quantization, such as fp32 or fp16, or unknown when unavailable. This does not specify the input tensor dtype or the precision of every runtime operation." + ) + + class InferenceResponse(BaseModel): """Base inference response. @@ -47,6 +63,18 @@ class InferenceResponse(BaseModel): """ model_config = ConfigDict(protected_namespaces=()) + resolved_model: Optional[ResolvedModel] = Field( + default=None, + description="Package identity of the model instance that produced this result. Available when USE_INFERENCE_MODELS=true and the model has a single resolved package. Each batch result carries its own metadata. Models without package identity, including direct local-path loads and multi-package pipelines, omit it. Image-only responses do not contain this field.", + examples=[ + { + "model_id": "project/3", + "model_package_id": "trtpackage", + "backend": "trt", + "quantization": "fp16", + } + ], + ) inference_id: Optional[str] = Field( description="Unique identifier of inference", default=None ) @@ -60,14 +88,6 @@ class InferenceResponse(BaseModel): ) -class ResolvedModel(BaseModel): - model_config = ConfigDict(protected_namespaces=()) - model_id: str - model_package_id: str - backend: str - quantization: str - - class CvInferenceResponse(InferenceResponse): """Computer Vision inference response. @@ -76,10 +96,6 @@ class CvInferenceResponse(InferenceResponse): """ image: Union[List[InferenceResponseImage], InferenceResponseImage] - resolved_model: Optional[ResolvedModel] = Field( - default=None, - description="The model package that produced this result. Present when USE_INFERENCE_MODELS is enabled and the loaded model has package metadata. Quantization describes the package, not every runtime operation.", - ) class WithVisualizationResponse(BaseModel): diff --git a/inference/models/clip/clip_inference_models.py b/inference/models/clip/clip_inference_models.py index 68bcaf63c9..10e784ff9c 100644 --- a/inference/models/clip/clip_inference_models.py +++ b/inference/models/clip/clip_inference_models.py @@ -343,6 +343,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/doctr/doctr_model_inference_models.py b/inference/models/doctr/doctr_model_inference_models.py index f1f96d7af5..151f9e460b 100644 --- a/inference/models/doctr/doctr_model_inference_models.py +++ b/inference/models/doctr/doctr_model_inference_models.py @@ -82,12 +82,14 @@ def infer_from_request( ) -> Union[OCRInferenceResponse, List]: if type(request.image) is list: response = [] - request_copy = copy.copy(request) + request_copy = copy(request) for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) - return response - return self.single_request(request) + else: + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: DoctrOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/easy_ocr/easy_ocr_inference_models.py b/inference/models/easy_ocr/easy_ocr_inference_models.py index 14269988c1..da4f713621 100644 --- a/inference/models/easy_ocr/easy_ocr_inference_models.py +++ b/inference/models/easy_ocr/easy_ocr_inference_models.py @@ -102,8 +102,10 @@ def infer_from_request( for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) - return response - return self.single_request(request) + else: + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: EasyOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/grounding_dino/grounding_dino_inference_models.py b/inference/models/grounding_dino/grounding_dino_inference_models.py index 14782cd4a5..4a3a1faa5c 100644 --- a/inference/models/grounding_dino/grounding_dino_inference_models.py +++ b/inference/models/grounding_dino/grounding_dino_inference_models.py @@ -95,6 +95,7 @@ def infer_from_request( Perform inference based on the details provided in the request, and return the associated responses. """ result = self.infer(**request.dict()) + self._attach_resolved_model_metadata(result) return result @usage_collector("model") diff --git a/inference/models/perception_encoder/perception_encoder_inference_models.py b/inference/models/perception_encoder/perception_encoder_inference_models.py index ae24fa6042..6a96f7c986 100644 --- a/inference/models/perception_encoder/perception_encoder_inference_models.py +++ b/inference/models/perception_encoder/perception_encoder_inference_models.py @@ -287,6 +287,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/sam/segment_anything_inference_models.py b/inference/models/sam/segment_anything_inference_models.py index 66a963ce78..699bd28bae 100644 --- a/inference/models/sam/segment_anything_inference_models.py +++ b/inference/models/sam/segment_anything_inference_models.py @@ -110,16 +110,18 @@ def infer_from_request(self, request: SamInferenceRequest): embedding, _ = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 if request.format == "json": - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=embedding.tolist(), time=inference_time ) else: binary_vector = BytesIO() np.save(binary_vector, embedding) binary_vector.seek(0) - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=binary_vector.getvalue(), time=inference_time ) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, SamSegmentationRequest): masks, low_res_masks = self.segment_image(**request.dict()) if request.format == "json": @@ -143,6 +145,7 @@ def infer_from_request(self, request: SamInferenceRequest): low_res_masks=[m.tolist() for m in low_res_masks], time=perf_counter() - t1, ) + self._attach_resolved_model_metadata(response) return response def embed_image(self, image: Any, image_id: Optional[str] = None, **kwargs): diff --git a/inference/models/sam2/segment_anything2_inference_models.py b/inference/models/sam2/segment_anything2_inference_models.py index 648f72256a..e7e459a620 100644 --- a/inference/models/sam2/segment_anything2_inference_models.py +++ b/inference/models/sam2/segment_anything2_inference_models.py @@ -157,19 +157,21 @@ def infer_from_request(self, request: Sam2InferenceRequest): if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 - return Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + response = Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, Sam2SegmentationRequest): masks, scores, low_resolution_logits = self.segment_image(**request.dict()) if request.format == "json": - return turn_segmentation_results_into_api_response( + response = turn_segmentation_results_into_api_response( masks=masks, scores=scores, mask_threshold=MASK_THRESHOLD, inference_start_timestamp=t1, ) elif request.format == "rle": - return turn_segmentation_results_into_rle_response( + response = turn_segmentation_results_into_rle_response( masks=masks, scores=scores, mask_threshold=0.0, @@ -185,6 +187,8 @@ def infer_from_request(self, request: Sam2InferenceRequest): return binary_data else: raise ValueError(f"Invalid format {request.format}") + self._attach_resolved_model_metadata(response) + return response else: raise ValueError(f"Invalid request type {type(request)}") diff --git a/inference/models/sam3/segment_anything3_inference_models.py b/inference/models/sam3/segment_anything3_inference_models.py index 248a25f1b7..ba09e39f03 100644 --- a/inference/models/sam3/segment_anything3_inference_models.py +++ b/inference/models/sam3/segment_anything3_inference_models.py @@ -90,7 +90,7 @@ def infer_from_request(self, request: Sam3InferenceRequest): record_fixed_model_input_for_request(self, request) t1 = perf_counter() if isinstance(request, Sam3SegmentationRequest): - return self.segment_image( + response = self.segment_image( image=request.image, prompts=request.prompts, output_prob_thresh=request.output_prob_thresh or 0.5, @@ -98,6 +98,8 @@ def infer_from_request(self, request: Sam3InferenceRequest): nms_iou_threshold=request.nms_iou_threshold, inference_start_timestamp=t1, ) + self._attach_resolved_model_metadata(response) + return response raise ValueError(f"Invalid request type {type(request)}") def run_tensor_native_inference(self, **kwargs) -> List[List[Dict]]: diff --git a/inference/models/sam3/visual_segmentation_inference_models.py b/inference/models/sam3/visual_segmentation_inference_models.py index 0b11a9f98e..16a9d92aa1 100644 --- a/inference/models/sam3/visual_segmentation_inference_models.py +++ b/inference/models/sam3/visual_segmentation_inference_models.py @@ -128,27 +128,34 @@ def infer_from_request(self, request: Sam2InferenceRequest): t1 = perf_counter() if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) - return Sam2EmbeddingResponse(time=perf_counter() - t1, image_id=image_id) + response = Sam2EmbeddingResponse( + time=perf_counter() - t1, image_id=image_id + ) + self._attach_resolved_model_metadata(response) + return response if isinstance(request, Sam2SegmentationRequest): masks, scores, low_res_logits = self.segment_image(**request.dict()) if request.format == "json" or request.format == "polygon": - return _build_polygon_response( + response = _build_polygon_response( masks=masks, scores=scores, inference_start_timestamp=t1, ) - if request.format == "rle": - return _build_rle_response( + elif request.format == "rle": + response = _build_rle_response( masks=masks, scores=scores, inference_start_timestamp=t1, ) - if request.format == "binary": + elif request.format == "binary": buf = BytesIO() np.savez_compressed(buf, masks=masks, low_res_masks=low_res_logits) buf.seek(0) return buf.getvalue() - raise ValueError(f"Invalid format {request.format}") + else: + raise ValueError(f"Invalid format {request.format}") + self._attach_resolved_model_metadata(response) + return response raise ValueError(f"Invalid request type {type(request)}") def preproc_image(self, image: InferenceRequestImage): diff --git a/inference/models/trocr/trocr_inference_models.py b/inference/models/trocr/trocr_inference_models.py index 3513d88d9c..3f25ade77a 100644 --- a/inference/models/trocr/trocr_inference_models.py +++ b/inference/models/trocr/trocr_inference_models.py @@ -80,4 +80,5 @@ def infer_from_request( text = self.infer(**request.model_dump()) t2 = perf_counter() response = OCRInferenceResponse(result=text, time=t2 - t1) + self._attach_resolved_model_metadata(response) return response diff --git a/mkdocs.yml b/mkdocs.yml index e15d1be276..d62a0fd7e1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -37,8 +37,6 @@ nav: - "Models": "https://docs.roboflow.com/models/supported-models" - "Workflows": "https://docs.roboflow.com/workflows" - "Reference": "https://docs.roboflow.com/reference/inference/choosing-the-right-tool" - - "HTTP API": - - "Resolved model metadata": http-resolved-model.md - "Changelog": "https://github.com/roboflow/inference/releases" theme: diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py index 20426cf54c..1b546c17cb 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -10,11 +10,13 @@ from inference.core import env from inference.core.entities.responses.inference import ( InferenceResponseImage, + LMMInferenceResponse, ObjectDetectionInferenceResponse, ) from inference.core.interfaces.http import http_api from inference.core.managers import base as manager_module from inference.core.managers.base import ModelManager +from inference.core.models import base as model_module from inference.core.models import inference_models_adapters as adapters from inference.core.registries import roboflow as registry_module from inference.core.registries.base import ModelRegistry @@ -33,7 +35,7 @@ ) -class PackageModel(adapters.InferenceModelsAdapter): +class PackageModel(model_module.Model): task_type = "object-detection" model_id = "test/1" @@ -51,6 +53,7 @@ def infer(self, image, **kwargs): def build_client(monkeypatch, flag=True, model=None, registry=None, preload=True): + monkeypatch.setattr(model_module, "USE_INFERENCE_MODELS", flag) monkeypatch.setattr(manager_module, "USE_INFERENCE_MODELS", flag) monkeypatch.setattr(manager_module, "MODELS_CACHE_AUTH_ENABLED", False) monkeypatch.setattr(manager_module, "DISABLE_INFERENCE_CACHE", True) @@ -254,13 +257,14 @@ def test_http_inference_omits_metadata_for_a_model_without_package_identity( assert "resolved_model" not in response.json() -def test_direct_adapter_response_reports_its_package_with_flag_disabled(monkeypatch): +def test_direct_model_response_omits_package_with_flag_disabled(monkeypatch): from inference.core.entities.requests.inference import ( InferenceRequestImage, ObjectDetectionInferenceRequest, ) monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) + monkeypatch.setattr(model_module, "USE_INFERENCE_MODELS", False) response = PackageModel().infer_from_request( ObjectDetectionInferenceRequest( id="direct-request", @@ -270,11 +274,11 @@ def test_direct_adapter_response_reports_its_package_with_flag_disabled(monkeypa ) assert isinstance(response, ObjectDetectionInferenceResponse) - assert response.resolved_model is not None - assert response.resolved_model.model_package_id == "onnxpackage" + assert response.resolved_model is None def test_classification_adapter_reports_package_with_predictions(monkeypatch): + monkeypatch.setattr(model_module, "USE_INFERENCE_MODELS", True) import base64 import cv2 @@ -324,3 +328,83 @@ def post_process(self, predictions, **kwargs): assert response.top == "cat" assert response.resolved_model is not None assert response.resolved_model.model_package_id == "onnxpackage" + + +def test_lmm_response_reports_package_without_task_specific_adapter(monkeypatch): + class LMMModel(PackageModel): + task_type = "lmm" + + def infer(self, **kwargs): + return [ + LMMInferenceResponse( + image=InferenceResponseImage(width=640, height=480), + response="A cat.", + ) + ] + + client, _ = build_client(monkeypatch, model=LMMModel()) + response = client.post( + "/infer/lmm", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + "prompt": "Describe the image.", + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["response"] == "A cat." + assert response.json()["resolved_model"] == vars(RESOLVED_MODEL) + + +@pytest.mark.parametrize("depth_map_format", ["json", "png8", "png16"]) +def test_depth_response_preserves_package_metadata(monkeypatch, depth_map_format): + class DepthModel(PackageModel): + task_type = "depth-estimation" + + def infer(self, **kwargs): + return [ + LMMInferenceResponse( + image=InferenceResponseImage(width=2, height=2), + response={ + "normalized_depth": np.array([[0.0, 0.25], [0.5, 1.0]]), + "image": SimpleNamespace(base64_image="depth-image"), + }, + ) + ] + + monkeypatch.setattr(http_api, "DEPTH_ESTIMATION_ENABLED", True) + client, _ = build_client(monkeypatch, model=DepthModel()) + response = client.post( + "/infer/depth-estimation", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + "depth_map_format": depth_map_format, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["depth_map_format"] == depth_map_format + assert response.json()["resolved_model"] == vars(RESOLVED_MODEL) + + +def test_openapi_documents_metadata_for_all_inference_response_types(monkeypatch): + monkeypatch.setattr(http_api, "DEPTH_ESTIMATION_ENABLED", True) + monkeypatch.setattr(http_api, "ACTION_RECOGNITION_ENABLED", True) + client, _ = build_client(monkeypatch) + schemas = client.get("/openapi.json").json()["components"]["schemas"] + + for name in ( + "ObjectDetectionInferenceResponse", + "LMMInferenceResponse", + "DepthEstimationResponse", + "ActionRecognitionInferenceResponse", + ): + metadata = schemas[name]["properties"]["resolved_model"] + assert "USE_INFERENCE_MODELS=true" in metadata["description"] + assert metadata["examples"][0]["backend"] == "trt" + assert ( + "Canonical model ID" + in schemas["ResolvedModel"]["properties"]["model_id"]["description"] + ) diff --git a/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py b/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py index 716ee08e29..b2a4a9971b 100644 --- a/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py +++ b/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py @@ -1,4 +1,5 @@ import contextlib +from types import SimpleNamespace from typing import List, Optional from unittest.mock import MagicMock, patch @@ -18,6 +19,8 @@ class _FakeModel: + resolved_model: Optional[SimpleNamespace] = None + def __init__(self, responses, class_names=None, sampling=None): self.responses = list(responses) self._class_names = class_names @@ -94,6 +97,23 @@ def test_window_segments_map_to_clip_frame_indices() -> None: assert response.timeline[0].class_id == 0 +def test_clip_response_reports_the_loaded_package(monkeypatch) -> None: + from inference.core.models import base + + monkeypatch.setattr(base, "USE_INFERENCE_MODELS", True) + model = _FakeModel(responses=[[]], class_names=["walk"]) + model.resolved_model = SimpleNamespace( + model_id="workspace/model", + model_package_id="video-package", + backend="torch", + quantization="fp16", + ) + with _clip(frame_count=3, source_fps=10.0): + response = _adapter(model).infer_from_request(_request()) + + assert response.model_dump()["resolved_model"] == vars(model.resolved_model) + + def test_ranges_of_one_class_merge_across_windows() -> None: model = _FakeModel( responses=[ diff --git a/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py new file mode 100644 index 0000000000..8e2243d598 --- /dev/null +++ b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py @@ -0,0 +1,275 @@ +import importlib +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest + +import inference.models +from inference.core.entities.requests.clip import ClipTextEmbeddingRequest +from inference.core.entities.requests.doctr import DoctrOCRInferenceRequest +from inference.core.entities.requests.easy_ocr import EasyOCRInferenceRequest +from inference.core.entities.requests.groundingdino import GroundingDINOInferenceRequest +from inference.core.entities.requests.inference import InferenceRequestImage +from inference.core.entities.requests.perception_encoder import ( + PerceptionEncoderTextEmbeddingRequest, +) +from inference.core.entities.requests.sam import ( + SamEmbeddingRequest, + SamSegmentationRequest, +) +from inference.core.entities.requests.sam2 import ( + Sam2EmbeddingRequest, + Sam2SegmentationRequest, +) +from inference.core.entities.requests.sam3 import Sam3SegmentationRequest +from inference.core.entities.requests.trocr import TrOCRInferenceRequest +from inference.core.entities.responses.inference import ( + InferenceResponseImage, + ObjectDetectionInferenceResponse, +) +from inference.core.entities.responses.ocr import OCRInferenceResponse +from inference.core.entities.responses.sam3 import Sam3SegmentationResponse + +METADATA = { + "model_id": "foundation/model", + "model_package_id": "package-id", + "backend": "torch", + "quantization": "fp32", +} +IMAGE = InferenceRequestImage(type="base64", value="unused") + + +@pytest.fixture +def load_adapter(monkeypatch): + monkeypatch.setattr("inference.core.models.base.USE_INFERENCE_MODELS", True) + imported_modules = [] + + def load(module_name, class_name): + module_name = f"inference.models.{module_name}_inference_models" + monkeypatch.delitem(sys.modules, module_name, raising=False) + module = importlib.import_module(module_name) + imported_modules.append(module_name) + adapter_class = getattr(module, class_name) + adapter = adapter_class.__new__(adapter_class) + adapter._model = SimpleNamespace(resolved_model=SimpleNamespace(**METADATA)) + if hasattr(module, "record_fixed_model_input_for_request"): + monkeypatch.setattr(module, "record_fixed_model_input_for_request", Mock()) + return adapter + + yield load + for module_name in imported_modules: + sys.modules.pop(module_name, None) + + +@pytest.fixture +def segmentation_backends(monkeypatch): + # Request handling does not need the optional GPU model implementations. + for name in ["sam", "sam2", "sam3"]: + package = ModuleType(f"inference.models.{name}") + package.__path__ = [str(Path(inference.models.__file__).parent / name)] + monkeypatch.setitem(sys.modules, package.__name__, package) + for module_name, class_name in [ + ("inference_models.models.sam.sam_torch", "SAMTorch"), + ("inference_models.models.sam2.sam2_torch", "SAM2Torch"), + ("inference_models.models.sam3.sam3_torch", "SAM3Torch"), + ]: + module = ModuleType(module_name) + setattr(module, class_name, type(class_name, (), {})) + setattr(module, "compute_image_hash", Mock()) + monkeypatch.setitem(sys.modules, module_name, module) + monkeypatch.setitem(sys.modules, "rasterio", ModuleType("rasterio")) + sam2 = ModuleType("sam2") + sam2_utils = ModuleType("sam2.utils") + sam2_misc = ModuleType("sam2.utils.misc") + setattr(sam2, "utils", sam2_utils) + setattr(sam2_utils, "misc", sam2_misc) + for module in [sam2, sam2_utils, sam2_misc]: + monkeypatch.setitem(sys.modules, module.__name__, module) + + +@pytest.mark.parametrize( + "module_name,class_name,request_class", + [ + ("clip.clip", "InferenceModelsClipAdapter", ClipTextEmbeddingRequest), + ( + "perception_encoder.perception_encoder", + "InferenceModelsPerceptionEncoderAdapter", + PerceptionEncoderTextEmbeddingRequest, + ), + ], +) +@pytest.mark.parametrize("enabled", [True, False]) +def test_embedding_request_metadata( + load_adapter, monkeypatch, module_name, class_name, request_class, enabled +): + adapter = load_adapter(module_name, class_name) + monkeypatch.setattr("inference.core.models.base.USE_INFERENCE_MODELS", enabled) + adapter.embed_text = Mock(return_value=np.array([[0.1, 0.2]])) + + response = adapter.infer_from_request(request_class(text="cat")) + + assert response.embeddings == [[0.1, 0.2]] + assert response.model_dump()["resolved_model"] == (METADATA if enabled else None) + + +@pytest.mark.parametrize( + "module_name,class_name,request_class", + [ + ("doctr.doctr_model", "InferenceModelsDocTRAdapter", DoctrOCRInferenceRequest), + ("easy_ocr.easy_ocr", "InferenceModelsEasyOCRAdapter", EasyOCRInferenceRequest), + ], +) +@pytest.mark.parametrize("batched", [True, False]) +def test_ocr_request_metadata( + load_adapter, module_name, class_name, request_class, batched +): + adapter = load_adapter(module_name, class_name) + adapter.single_request = Mock( + side_effect=lambda request: OCRInferenceResponse(result="text", time=0.1) + ) + + response = adapter.infer_from_request( + request_class(image=[IMAGE, IMAGE] if batched else IMAGE) + ) + + responses = response if batched else [response] + assert len(responses) == (2 if batched else 1) + assert all(item.model_dump()["resolved_model"] == METADATA for item in responses) + + +def test_trocr_request_metadata(load_adapter): + adapter = load_adapter("trocr.trocr", "InferenceModelsTrOCRAdapter") + adapter.infer = Mock(return_value="text") + + response = adapter.infer_from_request( + TrOCRInferenceRequest.model_validate({"image": IMAGE}) + ) + + assert response.result == "text" + assert response.model_dump()["resolved_model"] == METADATA + + +def test_grounding_dino_request_metadata(load_adapter): + adapter = load_adapter( + "grounding_dino.grounding_dino", "InferenceModelsGroundingDINOAdapter" + ) + adapter.infer = Mock( + return_value=[ + ObjectDetectionInferenceResponse( + predictions=[], image=InferenceResponseImage(width=8, height=8) + ) + ] + ) + + response = adapter.infer_from_request( + GroundingDINOInferenceRequest.model_validate({"image": IMAGE, "text": ["cat"]}) + ) + + assert response[0].model_dump()["resolved_model"] == METADATA + + +def test_ocr_request_without_backend_metadata(load_adapter): + adapter = load_adapter("trocr.trocr", "InferenceModelsTrOCRAdapter") + adapter._model = SimpleNamespace() + adapter.infer = Mock(return_value="text") + + response = adapter.infer_from_request( + TrOCRInferenceRequest.model_validate({"image": IMAGE}) + ) + + assert "resolved_model" not in response.model_dump(exclude_none=True) + + +@pytest.mark.usefixtures("segmentation_backends") +@pytest.mark.parametrize( + "module_name,class_name,request_class,embedding_result", + [ + ( + "sam.segment_anything", + "InferenceModelsSAMAdapter", + SamEmbeddingRequest, + (np.zeros((1, 1, 1, 1)), None), + ), + ( + "sam2.segment_anything2", + "InferenceModelsSAM2Adapter", + Sam2EmbeddingRequest, + (None, None, "image-id"), + ), + ( + "sam3.visual_segmentation", + "InferenceModelsSAM3InteractiveAdapter", + Sam2EmbeddingRequest, + (None, None, "image-id"), + ), + ], +) +def test_sam_embedding_request_metadata( + load_adapter, module_name, class_name, request_class, embedding_result +): + adapter = load_adapter(module_name, class_name) + adapter.embed_image = Mock(return_value=embedding_result) + + response = adapter.infer_from_request(request_class(image=IMAGE)) + + assert response.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.usefixtures("segmentation_backends") +@pytest.mark.parametrize( + "module_name,class_name", + [ + ("sam2.segment_anything2", "InferenceModelsSAM2Adapter"), + ("sam3.visual_segmentation", "InferenceModelsSAM3InteractiveAdapter"), + ], +) +@pytest.mark.parametrize("format", ["json", "rle", "binary"]) +def test_sam_segmentation_response_formats( + load_adapter, module_name, class_name, format +): + adapter = load_adapter(module_name, class_name) + masks = np.zeros((1, 8, 8), dtype=np.float32) + adapter.segment_image = Mock(return_value=(masks, np.array([0.9]), masks)) + request = Sam2SegmentationRequest.model_construct(image=IMAGE, format=format) + + response = adapter.infer_from_request(request) + + if format == "binary": + assert isinstance(response, bytes) + else: + assert response.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.usefixtures("segmentation_backends") +@pytest.mark.parametrize("format", ["json", "binary"]) +def test_sam_segmentation_metadata(load_adapter, format): + adapter = load_adapter("sam.segment_anything", "InferenceModelsSAMAdapter") + masks = np.zeros((1, 8, 8), dtype=np.float32) + adapter.segment_image = Mock(return_value=(masks, masks)) + + response = adapter.infer_from_request( + SamSegmentationRequest.model_validate({"format": format}) + ) + + if format == "binary": + assert isinstance(response, bytes) + else: + assert response.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.usefixtures("segmentation_backends") +def test_sam3_segmentation_request_metadata(load_adapter): + adapter = load_adapter("sam3.segment_anything3", "InferenceModelsSAM3Adapter") + adapter.segment_image = Mock( + return_value=Sam3SegmentationResponse(prompt_results=[], time=0.1) + ) + request = Sam3SegmentationRequest.model_validate( + {"image": IMAGE, "prompts": [{"text": "cat"}]} + ) + + response = adapter.infer_from_request(request) + + assert response.model_dump()["resolved_model"] == METADATA From da5b56145394ddd7913aa8aee5d8784b9eb24c50 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Thu, 17 Sep 2026 10:12:40 -0230 Subject: [PATCH 08/18] Simplify resolved model response field documentation --- .../common/inference_response_entities.py | 16 ++++------------ .../http/test_resolved_model_metadata.py | 3 +-- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/inference/core/workflows/core_steps/common/inference_response_entities.py b/inference/core/workflows/core_steps/common/inference_response_entities.py index aa657c9d65..01b518173b 100644 --- a/inference/core/workflows/core_steps/common/inference_response_entities.py +++ b/inference/core/workflows/core_steps/common/inference_response_entities.py @@ -63,18 +63,6 @@ class InferenceResponse(BaseModel): """ model_config = ConfigDict(protected_namespaces=()) - resolved_model: Optional[ResolvedModel] = Field( - default=None, - description="Package identity of the model instance that produced this result. Available when USE_INFERENCE_MODELS=true and the model has a single resolved package. Each batch result carries its own metadata. Models without package identity, including direct local-path loads and multi-package pipelines, omit it. Image-only responses do not contain this field.", - examples=[ - { - "model_id": "project/3", - "model_package_id": "trtpackage", - "backend": "trt", - "quantization": "fp16", - } - ], - ) inference_id: Optional[str] = Field( description="Unique identifier of inference", default=None ) @@ -86,6 +74,10 @@ class InferenceResponse(BaseModel): default=None, description="The time in seconds it took to produce the predictions including image preprocessing", ) + resolved_model: Optional[ResolvedModel] = Field( + default=None, + description="Package identity of the model instance that produced this result.", + ) class CvInferenceResponse(InferenceResponse): diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py index 1b546c17cb..0326c47962 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -402,8 +402,7 @@ def test_openapi_documents_metadata_for_all_inference_response_types(monkeypatch "ActionRecognitionInferenceResponse", ): metadata = schemas[name]["properties"]["resolved_model"] - assert "USE_INFERENCE_MODELS=true" in metadata["description"] - assert metadata["examples"][0]["backend"] == "trt" + assert {"$ref": "#/components/schemas/ResolvedModel"} in metadata["anyOf"] assert ( "Canonical model ID" in schemas["ResolvedModel"]["properties"]["model_id"]["description"] From 5388631d9f2a633199847990e116b474bba650df Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Thu, 17 Sep 2026 10:25:41 -0230 Subject: [PATCH 09/18] Patch metadata test flags on the active base model module --- .../models/test_specialized_resolved_model_metadata.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py index 8e2243d598..1356b98407 100644 --- a/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py +++ b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py @@ -44,7 +44,8 @@ @pytest.fixture def load_adapter(monkeypatch): - monkeypatch.setattr("inference.core.models.base.USE_INFERENCE_MODELS", True) + base_module = importlib.import_module("inference.core.models.base") + monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", True) imported_modules = [] def load(module_name, class_name): @@ -106,7 +107,8 @@ def test_embedding_request_metadata( load_adapter, monkeypatch, module_name, class_name, request_class, enabled ): adapter = load_adapter(module_name, class_name) - monkeypatch.setattr("inference.core.models.base.USE_INFERENCE_MODELS", enabled) + base_module = importlib.import_module("inference.core.models.base") + monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", enabled) adapter.embed_text = Mock(return_value=np.array([[0.1, 0.2]])) response = adapter.infer_from_request(request_class(text="cat")) From 904cfef98957afba038290388b9539fab53b9774 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Thu, 17 Sep 2026 11:08:28 -0230 Subject: [PATCH 10/18] Align resolved model metadata with response conventions --- .../common/inference_response_entities.py | 10 +++++++ .../models/auto_loaders/core.py | 29 ++++++++++++------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/inference/core/workflows/core_steps/common/inference_response_entities.py b/inference/core/workflows/core_steps/common/inference_response_entities.py index 01b518173b..01676a4e2c 100644 --- a/inference/core/workflows/core_steps/common/inference_response_entities.py +++ b/inference/core/workflows/core_steps/common/inference_response_entities.py @@ -38,6 +38,15 @@ class InferenceResponseImage(BaseModel): class ResolvedModel(BaseModel): + """Identity of the model package used for inference. + + Attributes: + model_id (str): Canonical model ID returned by the weights provider. + model_package_id (str): ID of the loaded model package. + backend (str): Backend of the loaded package. + quantization (str): Quantization of the loaded package. + """ + model_config = ConfigDict(protected_namespaces=()) model_id: str = Field( description="Canonical model ID returned by the weights provider. It can differ from the requested alias." @@ -60,6 +69,7 @@ class InferenceResponse(BaseModel): inference_id (Optional[str]): Unique identifier of inference frame_id (Optional[int]): The frame id of the image used in inference if the input was a video. time (Optional[float]): The time in seconds it took to produce the predictions including image preprocessing. + resolved_model (Optional[ResolvedModel]): Identity of the model package used for inference. """ model_config = ConfigDict(protected_namespaces=()) diff --git a/inference_models/inference_models/models/auto_loaders/core.py b/inference_models/inference_models/models/auto_loaders/core.py index 94b774bf93..b8a7010920 100644 --- a/inference_models/inference_models/models/auto_loaders/core.py +++ b/inference_models/inference_models/models/auto_loaders/core.py @@ -11,7 +11,18 @@ from contextvars import ContextVar from datetime import datetime from functools import partial -from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + Generator, + List, + Optional, + Set, + Tuple, + Union, + cast, +) import torch from filelock import FileLock @@ -2427,16 +2438,12 @@ def dependency_directory_pointer(model_dir: str) -> None: model_init_kwargs=model_init_kwargs, ), ) - if ( - cache_entry.canonical_model_id is not None - and cache_entry.backend_type is not None - ): - model.resolved_model = ResolvedModelMetadata( - model_id=cache_entry.canonical_model_id, - model_package_id=cache_entry.model_package_id, - backend=cache_entry.backend_type.value, - quantization=package_config.quantization or Quantization.UNKNOWN.value, - ) + model.resolved_model = ResolvedModelMetadata( + model_id=cast(str, cache_entry.canonical_model_id), + model_package_id=cache_entry.model_package_id, + backend=cache_entry.backend_type.value, + quantization=package_config.quantization or Quantization.UNKNOWN.value, + ) if point_model_directory: point_model_directory(model_package_cache_dir) verbose_info( From 4e408c2895b1f41511f010e3cb190c72f594a1ee Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Thu, 17 Sep 2026 11:26:44 -0230 Subject: [PATCH 11/18] Preserve null response fields and report legacy model identity --- inference/core/interfaces/http/http_api.py | 16 -- inference/core/models/base.py | 5 +- .../common/inference_response_entities.py | 31 ++-- inference/models/clip/clip_model.py | 1 + inference/models/doctr/doctr_model.py | 5 +- inference/models/easy_ocr/easy_ocr.py | 5 +- .../models/grounding_dino/grounding_dino.py | 1 + .../perception_encoder/perception_encoder.py | 1 + inference/models/sam/segment_anything.py | 9 +- inference/models/sam2/segment_anything2.py | 12 +- inference/models/sam3/segment_anything3.py | 1 + inference/models/sam3/visual_segmentation.py | 8 +- inference/models/trocr/trocr.py | 1 + inference/models/yolo_world/yolo_world.py | 4 +- .../http/test_resolved_model_metadata.py | 17 +- .../http/test_response_null_compatibility.py | 119 ++++++++++++++ .../test_legacy_resolved_model_metadata.py | 155 ++++++++++++++++++ 17 files changed, 345 insertions(+), 46 deletions(-) create mode 100644 tests/inference/unit_tests/core/interfaces/http/test_response_null_compatibility.py create mode 100644 tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index 2fc4e40865..1f1ddeda68 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -3043,7 +3043,6 @@ def healthz(): @app.post( "/clip/embed_image", response_model=ClipEmbeddingResponse, - response_model_exclude_none=True, summary="CLIP Image Embeddings", description="Run the Open AI CLIP model to embed image data.", ) @@ -3090,7 +3089,6 @@ def clip_embed_image( @app.post( "/clip/embed_text", response_model=ClipEmbeddingResponse, - response_model_exclude_none=True, summary="CLIP Text Embeddings", description="Run the Open AI CLIP model to embed text data.", ) @@ -3137,7 +3135,6 @@ def clip_embed_text( @app.post( "/clip/compare", response_model=ClipCompareResponse, - response_model_exclude_none=True, summary="CLIP Compare", description="Run the Open AI CLIP model to compute similarity scores.", ) @@ -3186,7 +3183,6 @@ def clip_compare( @app.post( "/perception_encoder/embed_image", response_model=PerceptionEncoderEmbeddingResponse, - response_model_exclude_none=True, summary="PE Image Embeddings", description="Run the Meta Perception Encoder model to embed image data.", ) @@ -3233,7 +3229,6 @@ def pe_embed_image( @app.post( "/perception_encoder/embed_text", response_model=PerceptionEncoderEmbeddingResponse, - response_model_exclude_none=True, summary="Perception Encoder Text Embeddings", description="Run the Meta Perception Encoder model to embed text data.", ) @@ -3280,7 +3275,6 @@ def pe_embed_text( @app.post( "/perception_encoder/compare", response_model=PerceptionEncoderCompareResponse, - response_model_exclude_none=True, summary="Perception Encoder Compare", description="Run the Meta Perception Encoder model to compute similarity scores.", ) @@ -3529,7 +3523,6 @@ def easy_ocr_retrieve_text( @app.post( "/sam/embed_image", response_model=SamEmbeddingResponse, - response_model_exclude_none=True, summary="SAM Image Embeddings", description="Run the Meta AI Segmant Anything Model to embed image data.", ) @@ -3581,7 +3574,6 @@ def sam_embed_image( @app.post( "/sam/segment_image", response_model=SamSegmentationResponse, - response_model_exclude_none=True, summary="SAM Image Segmentation", description="Run the Meta AI Segmant Anything Model to generate segmenations for image data.", ) @@ -3635,7 +3627,6 @@ def sam_segment_image( @app.post( "/sam2/embed_image", response_model=Sam2EmbeddingResponse, - response_model_exclude_none=True, summary="SAM2 Image Embeddings", description="Run the Meta AI Segment Anything 2 Model to embed image data.", ) @@ -3677,7 +3668,6 @@ def sam2_embed_image( @app.post( "/sam2/segment_image", response_model=Sam2SegmentationResponse, - response_model_exclude_none=True, summary="SAM2 Image Segmentation", description="Run the Meta AI Segment Anything 2 Model to generate segmenations for image data.", ) @@ -3726,7 +3716,6 @@ def sam2_segment_image( @app.post( "/sam3/embed_image", response_model=Sam3EmbeddingResponse, - response_model_exclude_none=True, summary="Seg preview Image Embeddings", description="Run the Model to embed image data.", ) @@ -3773,7 +3762,6 @@ def sam3_embed_image( @app.post( "/sam3/concept_segment", response_model=Sam3SegmentationResponse, - response_model_exclude_none=True, summary="SAM3 PCS (promptable concept segmentation)", description="Run the SAM3 PCS (promptable concept segmentation) to generate segmentations for image data.", ) @@ -3946,7 +3934,6 @@ def sam3_segment_image( @app.post( "/sam3/visual_segment", response_model=Sam2SegmentationResponse, - response_model_exclude_none=True, summary="SAM3 PVS (promptable visual segmentation)", description="Run the SAM3 PVS (promptable visual segmentation) to generate segmentations for image data.", ) @@ -4278,7 +4265,6 @@ def _infer_depth_estimation( @app.post( "/infer/depth-estimation", response_model=DepthEstimationResponse, - response_model_exclude_none=True, summary="Depth Estimation", description="Run the depth estimation model to generate a depth map.", ) @@ -4317,7 +4303,6 @@ def depth_estimation( @app.post( "/infer/depth-estimation/{model_id:path}", response_model=DepthEstimationResponse, - response_model_exclude_none=True, summary="Depth Estimation with model ID in path", description="Run depth estimation. Model ID is specified in the URL path and can contain slashes.", ) @@ -4356,7 +4341,6 @@ def depth_estimation_with_model_id( @app.post( "/infer/action_recognition", response_model=ActionRecognitionInferenceResponse, - response_model_exclude_none=True, summary="Action Recognition", description=( "Classify the actions in a video clip. The model states " diff --git a/inference/core/models/base.py b/inference/core/models/base.py index 8af26d94b4..14909c28df 100644 --- a/inference/core/models/base.py +++ b/inference/core/models/base.py @@ -113,7 +113,10 @@ class Model(BaseInference): @property def resolved_model(self) -> Any: if not USE_INFERENCE_MODELS: - return None + model_id = getattr(self, "endpoint", None) or getattr( + self, "model_id", None + ) + return ResolvedModel(model_id=model_id) if model_id is not None else None return getattr(getattr(self, "_model", None), "resolved_model", None) def _attach_resolved_model_metadata(self, responses: Any) -> None: diff --git a/inference/core/workflows/core_steps/common/inference_response_entities.py b/inference/core/workflows/core_steps/common/inference_response_entities.py index 01676a4e2c..c4e15077ab 100644 --- a/inference/core/workflows/core_steps/common/inference_response_entities.py +++ b/inference/core/workflows/core_steps/common/inference_response_entities.py @@ -38,27 +38,30 @@ class InferenceResponseImage(BaseModel): class ResolvedModel(BaseModel): - """Identity of the model package used for inference. + """Identity of the model and available package details used for inference. Attributes: - model_id (str): Canonical model ID returned by the weights provider. - model_package_id (str): ID of the loaded model package. - backend (str): Backend of the loaded package. - quantization (str): Quantization of the loaded package. + model_id (str): ID of the loaded model, using the canonical ID when available. + model_package_id (Optional[str]): ID of the loaded model package. + backend (Optional[str]): Backend of the loaded package. + quantization (Optional[str]): Quantization of the loaded package. """ model_config = ConfigDict(protected_namespaces=()) model_id: str = Field( - description="Canonical model ID returned by the weights provider. It can differ from the requested alias." + description="ID of the loaded model, using the canonical ID when available." ) - model_package_id: str = Field( - description="ID of the package that loaded successfully and produced this result, including after a loading fallback." + model_package_id: Optional[str] = Field( + default=None, + description="ID of the package that loaded successfully and produced this result, including after a loading fallback.", ) - backend: str = Field( - description="Backend of the loaded package, such as onnx, trt, or torch." + backend: Optional[str] = Field( + default=None, + description="Backend of the loaded package, such as onnx, trt, or torch.", ) - quantization: str = Field( - description="Package quantization, such as fp32 or fp16, or unknown when unavailable. This does not specify the input tensor dtype or the precision of every runtime operation." + quantization: Optional[str] = Field( + default=None, + description="Package quantization, such as fp32 or fp16, or unknown when unavailable. This does not specify the input tensor dtype or the precision of every runtime operation.", ) @@ -69,7 +72,7 @@ class InferenceResponse(BaseModel): inference_id (Optional[str]): Unique identifier of inference frame_id (Optional[int]): The frame id of the image used in inference if the input was a video. time (Optional[float]): The time in seconds it took to produce the predictions including image preprocessing. - resolved_model (Optional[ResolvedModel]): Identity of the model package used for inference. + resolved_model (Optional[ResolvedModel]): Model identity and available package details for this result. """ model_config = ConfigDict(protected_namespaces=()) @@ -86,7 +89,7 @@ class InferenceResponse(BaseModel): ) resolved_model: Optional[ResolvedModel] = Field( default=None, - description="Package identity of the model instance that produced this result.", + description="Model identity and available package details for this result.", ) diff --git a/inference/models/clip/clip_model.py b/inference/models/clip/clip_model.py index 4c0e5d5370..96affa5e37 100644 --- a/inference/models/clip/clip_model.py +++ b/inference/models/clip/clip_model.py @@ -335,6 +335,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/doctr/doctr_model.py b/inference/models/doctr/doctr_model.py index 490583938c..dfdd26332f 100644 --- a/inference/models/doctr/doctr_model.py +++ b/inference/models/doctr/doctr_model.py @@ -104,8 +104,11 @@ def infer_from_request( for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) + self._attach_resolved_model_metadata(response) return response - return self.single_request(request) + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: DoctrOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/easy_ocr/easy_ocr.py b/inference/models/easy_ocr/easy_ocr.py index 25432c4cc0..0461e2e2ed 100644 --- a/inference/models/easy_ocr/easy_ocr.py +++ b/inference/models/easy_ocr/easy_ocr.py @@ -127,8 +127,11 @@ def infer_from_request( for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) + self._attach_resolved_model_metadata(response) return response - return self.single_request(request) + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: EasyOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/grounding_dino/grounding_dino.py b/inference/models/grounding_dino/grounding_dino.py index fbe4daca10..cf73785d76 100644 --- a/inference/models/grounding_dino/grounding_dino.py +++ b/inference/models/grounding_dino/grounding_dino.py @@ -151,6 +151,7 @@ def infer_from_request( Perform inference based on the details provided in the request, and return the associated responses. """ result = self.infer(**request.dict()) + self._attach_resolved_model_metadata(result) return result @usage_collector("model") diff --git a/inference/models/perception_encoder/perception_encoder.py b/inference/models/perception_encoder/perception_encoder.py index cc77d97231..ee661fc114 100644 --- a/inference/models/perception_encoder/perception_encoder.py +++ b/inference/models/perception_encoder/perception_encoder.py @@ -301,6 +301,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/sam/segment_anything.py b/inference/models/sam/segment_anything.py index c1e8279ba2..024b0a47f3 100644 --- a/inference/models/sam/segment_anything.py +++ b/inference/models/sam/segment_anything.py @@ -141,16 +141,20 @@ def infer_from_request(self, request: SamInferenceRequest): embedding, _ = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 if request.format == "json": - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=embedding.tolist(), time=inference_time ) + self._attach_resolved_model_metadata(response) + return response elif request.format == "binary": binary_vector = BytesIO() np.save(binary_vector, embedding) binary_vector.seek(0) - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=binary_vector.getvalue(), time=inference_time ) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, SamSegmentationRequest): masks, low_res_masks = self.segment_image(**request.dict()) if request.format == "json": @@ -174,6 +178,7 @@ def infer_from_request(self, request: SamInferenceRequest): low_res_masks=[m.tolist() for m in low_res_masks], time=perf_counter() - t1, ) + self._attach_resolved_model_metadata(response) return response def preproc_image(self, image: InferenceRequestImage): diff --git a/inference/models/sam2/segment_anything2.py b/inference/models/sam2/segment_anything2.py index 8b52dcc603..b3553730da 100644 --- a/inference/models/sam2/segment_anything2.py +++ b/inference/models/sam2/segment_anything2.py @@ -196,24 +196,30 @@ def infer_from_request(self, request: Sam2InferenceRequest): if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 - return Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + response = Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, Sam2SegmentationRequest): masks, scores, low_resolution_logits = self.segment_image(**request.dict()) if request.format == "json": - return turn_segmentation_results_into_api_response( + response = turn_segmentation_results_into_api_response( masks=masks, scores=scores, mask_threshold=0.0, inference_start_timestamp=t1, ) + self._attach_resolved_model_metadata(response) + return response elif request.format == "rle": - return turn_segmentation_results_into_rle_response( + response = turn_segmentation_results_into_rle_response( masks=masks, scores=scores, mask_threshold=0.0, inference_start_timestamp=t1, ) + self._attach_resolved_model_metadata(response) + return response elif request.format == "binary": binary_vector = BytesIO() np.savez_compressed( diff --git a/inference/models/sam3/segment_anything3.py b/inference/models/sam3/segment_anything3.py index 91018a4f9a..277fcfc4e2 100644 --- a/inference/models/sam3/segment_anything3.py +++ b/inference/models/sam3/segment_anything3.py @@ -526,6 +526,7 @@ def infer_from_request(self, request: Sam3InferenceRequest): nms_iou_threshold=request.nms_iou_threshold, ) # segment_image now returns either bytes or a response model + self._attach_resolved_model_metadata(result) return result else: raise ValueError(f"Invalid request type {type(request)}") diff --git a/inference/models/sam3/visual_segmentation.py b/inference/models/sam3/visual_segmentation.py index 090540a588..b64f030932 100644 --- a/inference/models/sam3/visual_segmentation.py +++ b/inference/models/sam3/visual_segmentation.py @@ -211,14 +211,18 @@ def infer_from_request(self, request: Sam2InferenceRequest): if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 - return Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + response = Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, Sam2SegmentationRequest): masks, scores, low_resolution_logits = self.segment_image(**request.dict()) predictions = _masks_to_predictions(masks, scores, request.format) - return Sam2SegmentationResponse( + response = Sam2SegmentationResponse( time=perf_counter() - t1, predictions=predictions, ) + self._attach_resolved_model_metadata(response) + return response else: raise ValueError(f"Invalid request type {type(request)}") diff --git a/inference/models/trocr/trocr.py b/inference/models/trocr/trocr.py index feed2177c0..1e1edc574b 100644 --- a/inference/models/trocr/trocr.py +++ b/inference/models/trocr/trocr.py @@ -82,4 +82,5 @@ def infer_from_request( text = self.infer(**request.model_dump()) t2 = perf_counter() response = OCRInferenceResponse(result=text[0], time=t2 - t1) + self._attach_resolved_model_metadata(response) return response diff --git a/inference/models/yolo_world/yolo_world.py b/inference/models/yolo_world/yolo_world.py index 355cea21fe..076af83b24 100644 --- a/inference/models/yolo_world/yolo_world.py +++ b/inference/models/yolo_world/yolo_world.py @@ -82,7 +82,9 @@ def infer_from_request( Perform inference based on the details provided in the request, and return the associated responses. """ with self._state_lock: - return self.infer(**request.dict()) + response = self.infer(**request.dict()) + self._attach_resolved_model_metadata(response) + return response @usage_collector("model") def infer( diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py index 0326c47962..7a66c50cf3 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -133,7 +133,7 @@ def test_http_batch_reports_package_for_each_image(monkeypatch): ] -def test_http_inference_omits_metadata_when_flag_is_disabled(monkeypatch): +def test_http_inference_reports_model_id_when_flag_is_disabled(monkeypatch): monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) model_types = runpy.run_path(model_utils.__file__)["ROBOFLOW_MODEL_TYPES"] monkeypatch.setattr( @@ -143,7 +143,7 @@ def test_http_inference_omits_metadata_when_flag_is_disabled(monkeypatch): ) def initialize_legacy_model(self, model_id, **kwargs): - self.model_id = model_id + self.endpoint = model_id monkeypatch.setattr(YOLOv8ObjectDetection, "__init__", initialize_legacy_model) monkeypatch.setattr(YOLOv8ObjectDetection, "infer", PackageModel.infer) @@ -170,6 +170,7 @@ def initialize_legacy_model(self, model_id, **kwargs): assert payload == { "image": {"width": 640, "height": 480}, "predictions": [], + "resolved_model": {"model_id": "test/1"}, } @@ -257,7 +258,7 @@ def test_http_inference_omits_metadata_for_a_model_without_package_identity( assert "resolved_model" not in response.json() -def test_direct_model_response_omits_package_with_flag_disabled(monkeypatch): +def test_direct_model_response_reports_model_id_with_flag_disabled(monkeypatch): from inference.core.entities.requests.inference import ( InferenceRequestImage, ObjectDetectionInferenceRequest, @@ -274,7 +275,13 @@ def test_direct_model_response_omits_package_with_flag_disabled(monkeypatch): ) assert isinstance(response, ObjectDetectionInferenceResponse) - assert response.resolved_model is None + assert response.resolved_model is not None + assert response.resolved_model.model_dump() == { + "model_id": "test/1", + "model_package_id": None, + "backend": None, + "quantization": None, + } def test_classification_adapter_reports_package_with_predictions(monkeypatch): @@ -404,6 +411,6 @@ def test_openapi_documents_metadata_for_all_inference_response_types(monkeypatch metadata = schemas[name]["properties"]["resolved_model"] assert {"$ref": "#/components/schemas/ResolvedModel"} in metadata["anyOf"] assert ( - "Canonical model ID" + "canonical ID" in schemas["ResolvedModel"]["properties"]["model_id"]["description"] ) diff --git a/tests/inference/unit_tests/core/interfaces/http/test_response_null_compatibility.py b/tests/inference/unit_tests/core/interfaces/http/test_response_null_compatibility.py new file mode 100644 index 0000000000..6fa19196d7 --- /dev/null +++ b/tests/inference/unit_tests/core/interfaces/http/test_response_null_compatibility.py @@ -0,0 +1,119 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import numpy as np +import pytest +from starlette.testclient import TestClient + +from inference.core.entities.responses.clip import ClipCompareResponse +from inference.core.entities.responses.perception_encoder import ( + PerceptionEncoderCompareResponse, +) +from inference.core.entities.responses.sam3 import ( + Sam3PromptEcho, + Sam3PromptResult, + Sam3SegmentationResponse, +) +from inference.core.interfaces.http import http_api + + +@pytest.fixture +def client_and_manager(monkeypatch): + for name in ( + "CORE_MODELS_ENABLED", + "CORE_MODEL_CLIP_ENABLED", + "CORE_MODEL_PE_ENABLED", + "CORE_MODEL_SAM3_ENABLED", + "DEPTH_ESTIMATION_ENABLED", + ): + monkeypatch.setattr(http_api, name, True) + monkeypatch.setattr(http_api, "LAMBDA", False) + monkeypatch.setattr(http_api, "GCP_SERVERLESS", False) + monkeypatch.setattr(http_api, "DEDICATED_DEPLOYMENT_WORKSPACE_URL", None) + monkeypatch.setattr(http_api, "InferenceInstrumentator", MagicMock()) + monkeypatch.setattr( + http_api.usage_collector, "async_push_usage_payloads", AsyncMock() + ) + manager = MagicMock() + manager.pingback = None + return TestClient(http_api.HttpInterface(model_manager=manager).app), manager + + +@pytest.mark.parametrize( + "endpoint,response_type", + [ + ("/clip/compare", ClipCompareResponse), + ("/perception_encoder/compare", PerceptionEncoderCompareResponse), + ], +) +def test_embedding_comparison_keeps_existing_null_fields( + client_and_manager, endpoint, response_type +): + client, manager = client_and_manager + manager.infer_from_request_sync.return_value = response_type(similarity=[0.5]) + + response = client.post( + endpoint, + json={"subject": "cat", "subject_type": "text", "prompt": "animal"}, + ) + + assert response.status_code == 200, response.text + payload = response.json() + for field in ("parent_id", "time", "frame_id", "inference_id"): + assert field in payload + assert payload[field] is None + + +def test_sam3_keeps_null_prompt_echo_fields(client_and_manager): + client, manager = client_and_manager + manager.infer_from_request_sync.return_value = Sam3SegmentationResponse( + prompt_results=[ + Sam3PromptResult( + prompt_index=0, + echo=Sam3PromptEcho(prompt_index=0, type="text", text="cat"), + predictions=[], + ) + ], + time=0.1, + ) + + response = client.post( + "/sam3/concept_segment", + json={ + "image": {"type": "base64", "value": "image"}, + "prompts": [{"type": "text", "text": "cat"}], + }, + ) + + assert response.status_code == 200, response.text + echo = response.json()["prompt_results"][0]["echo"] + assert "num_boxes" in echo + assert echo["num_boxes"] is None + + +@pytest.mark.parametrize( + "endpoint", + ["/infer/depth-estimation", "/infer/depth-estimation/test/1"], +) +def test_depth_estimation_keeps_null_image(client_and_manager, endpoint): + client, manager = client_and_manager + manager.infer_from_request_sync.return_value = SimpleNamespace( + response={ + "normalized_depth": np.zeros((1, 1)), + "image": SimpleNamespace(base64_image=None), + } + ) + + response = client.post( + endpoint, + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + "depth_map_format": "json", + }, + ) + + assert response.status_code == 200, response.text + payload = response.json() + assert "image" in payload + assert payload["image"] is None diff --git a/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py new file mode 100644 index 0000000000..deb6efc954 --- /dev/null +++ b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py @@ -0,0 +1,155 @@ +import importlib +import sys +from pathlib import Path +from threading import Lock +from types import ModuleType +from unittest.mock import Mock + +import numpy as np +import pytest + +import inference.models +from inference.core.entities.requests.clip import ClipTextEmbeddingRequest +from inference.core.entities.requests.doctr import DoctrOCRInferenceRequest +from inference.core.entities.requests.easy_ocr import EasyOCRInferenceRequest +from inference.core.entities.requests.inference import InferenceRequestImage +from inference.core.entities.requests.sam import ( + SamEmbeddingRequest, + SamSegmentationRequest, +) +from inference.core.entities.responses.ocr import OCRInferenceResponse + +IMAGE = InferenceRequestImage(type="base64", value="unused") +MODEL_ID = "foundation/model" +METADATA = { + "model_id": MODEL_ID, + "model_package_id": None, + "backend": None, + "quantization": None, +} + + +@pytest.fixture +def load_legacy_model(monkeypatch): + base_module = importlib.import_module("inference.core.models.base") + monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", False) + imported_modules = [] + + def load(module_name, class_name): + package_name = module_name.split(".")[0] + package = ModuleType(f"inference.models.{package_name}") + package.__path__ = [str(Path(inference.models.__file__).parent / package_name)] + monkeypatch.setitem(sys.modules, package.__name__, package) + module_name = f"inference.models.{module_name}" + monkeypatch.delitem(sys.modules, module_name, raising=False) + module = importlib.import_module(module_name) + imported_modules.append(module_name) + model_class = getattr(module, class_name) + model = model_class.__new__(model_class) + model.endpoint = MODEL_ID + return model + + yield load + for module_name in imported_modules: + sys.modules.pop(module_name, None) + + +def test_legacy_clip_request_metadata(load_legacy_model, monkeypatch): + monkeypatch.setitem(sys.modules, "clip", ModuleType("clip")) + model = load_legacy_model("clip.clip_model", "Clip") + model.embed_text = Mock(return_value=np.array([[0.1, 0.2]])) + + response = model.infer_from_request( + ClipTextEmbeddingRequest(id="test-request", model_id=None, text="cat") + ) + + assert response.embeddings == [[0.1, 0.2]] + assert response.model_dump()["resolved_model"] == METADATA + assert response.time >= 0 + + +@pytest.mark.parametrize( + "module_name,class_name,request_class,batched", + [ + ("doctr.doctr_model", "DocTR", DoctrOCRInferenceRequest, False), + ("easy_ocr.easy_ocr", "EasyOCR", EasyOCRInferenceRequest, False), + ("easy_ocr.easy_ocr", "EasyOCR", EasyOCRInferenceRequest, True), + ], +) +def test_legacy_ocr_request_metadata( + load_legacy_model, monkeypatch, module_name, class_name, request_class, batched +): + for module_name_to_stub, attributes in [ + ("doctr.io", ["DocumentFile"]), + ("doctr.models", ["crnn_vgg16_bn", "db_resnet50", "ocr_predictor"]), + ("easyocr", []), + ]: + module = ModuleType(module_name_to_stub) + for attribute in attributes: + setattr(module, attribute, Mock()) + monkeypatch.setitem(sys.modules, module_name_to_stub, module) + model = load_legacy_model(module_name, class_name) + model.single_request = Mock( + side_effect=lambda request: OCRInferenceResponse(result="text", time=0.1) + ) + + response = model.infer_from_request( + request_class(image=[IMAGE, IMAGE] if batched else IMAGE) + ) + + responses = response if isinstance(response, list) else [response] + assert len(responses) == (2 if batched else 1) + for result in responses: + assert result.result == "text" + assert result.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.parametrize("response_format", ["json", "binary"]) +def test_legacy_sam_embedding_metadata(load_legacy_model, monkeypatch, response_format): + for module_name in ["rasterio", "rasterio.features", "segment_anything"]: + module = ModuleType(module_name) + if module_name == "segment_anything": + setattr(module, "SamPredictor", Mock()) + setattr(module, "sam_model_registry", {}) + monkeypatch.setitem(sys.modules, module_name, module) + model = load_legacy_model("sam.segment_anything", "SegmentAnything") + model._state_lock = Lock() + model.embed_image = Mock(return_value=(np.array([[0.1, 0.2]]), None)) + + response = model.infer_from_request( + SamEmbeddingRequest( + id="test-request", model_id=None, image=IMAGE, format=response_format + ) + ) + + assert response.resolved_model.model_dump() == METADATA + if response_format == "binary": + assert isinstance(response.embeddings, bytes) + else: + assert response.embeddings == [[0.1, 0.2]] + + +def test_legacy_sam_binary_segmentation_preserves_payload( + load_legacy_model, monkeypatch +): + for module_name in ["rasterio", "rasterio.features", "segment_anything"]: + module = ModuleType(module_name) + if module_name == "segment_anything": + setattr(module, "SamPredictor", Mock()) + setattr(module, "sam_model_registry", {}) + monkeypatch.setitem(sys.modules, module_name, module) + model = load_legacy_model("sam.segment_anything", "SegmentAnything") + model._state_lock = Lock() + model.segment_image = Mock(return_value=(np.zeros((1, 2, 2)), np.zeros((1, 2, 2)))) + + response = model.infer_from_request( + SamSegmentationRequest( + id="test-request", + model_id=None, + embeddings=None, + image=IMAGE, + format="binary", + ) + ) + + assert isinstance(response, bytes) From 01f0d929934353d17f0952ae735672065246e889 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Thu, 17 Sep 2026 11:32:08 -0230 Subject: [PATCH 12/18] Remove unreachable flag-off adapter test cases --- .../models/test_specialized_resolved_model_metadata.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py index 1356b98407..54128017fe 100644 --- a/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py +++ b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py @@ -102,19 +102,16 @@ def segmentation_backends(monkeypatch): ), ], ) -@pytest.mark.parametrize("enabled", [True, False]) def test_embedding_request_metadata( - load_adapter, monkeypatch, module_name, class_name, request_class, enabled + load_adapter, module_name, class_name, request_class ): adapter = load_adapter(module_name, class_name) - base_module = importlib.import_module("inference.core.models.base") - monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", enabled) adapter.embed_text = Mock(return_value=np.array([[0.1, 0.2]])) response = adapter.infer_from_request(request_class(text="cat")) assert response.embeddings == [[0.1, 0.2]] - assert response.model_dump()["resolved_model"] == (METADATA if enabled else None) + assert response.model_dump()["resolved_model"] == METADATA @pytest.mark.parametrize( From f8c8c6e4b94f734d1056e697e10d0884a2f5e539 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Thu, 17 Sep 2026 14:23:59 -0230 Subject: [PATCH 13/18] Report legacy model identity with inference models enabled --- inference/core/models/base.py | 11 +++++------ .../models/test_legacy_resolved_model_metadata.py | 8 +++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/inference/core/models/base.py b/inference/core/models/base.py index 14909c28df..547da9f601 100644 --- a/inference/core/models/base.py +++ b/inference/core/models/base.py @@ -112,12 +112,11 @@ class Model(BaseInference): @property def resolved_model(self) -> Any: - if not USE_INFERENCE_MODELS: - model_id = getattr(self, "endpoint", None) or getattr( - self, "model_id", None - ) - return ResolvedModel(model_id=model_id) if model_id is not None else None - return getattr(getattr(self, "_model", None), "resolved_model", None) + model = getattr(self, "_model", None) + if USE_INFERENCE_MODELS and model is not None: + return getattr(model, "resolved_model", None) + model_id = getattr(self, "endpoint", None) or getattr(self, "model_id", None) + return ResolvedModel(model_id=model_id) if model_id is not None else None def _attach_resolved_model_metadata(self, responses: Any) -> None: metadata = self.resolved_model diff --git a/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py index deb6efc954..89954efd57 100644 --- a/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py +++ b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py @@ -29,10 +29,12 @@ } -@pytest.fixture -def load_legacy_model(monkeypatch): +@pytest.fixture( + params=[False, True], ids=["legacy-default", "inference-models-default"] +) +def load_legacy_model(monkeypatch, request): base_module = importlib.import_module("inference.core.models.base") - monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", False) + monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", request.param) imported_modules = [] def load(module_name, class_name): From ee460097894c5f9c2a5d35c8e3aef9c79ce2cdb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 18 Sep 2026 16:50:06 +0200 Subject: [PATCH 14/18] Report resolved model metadata for legacy classification and wrapped Cosmos reasoners Two paths returned no `resolved_model`, and no test noticed. Legacy classification. `ClassificationBaseOnnxRoboflowInferenceModel` overrides `infer_from_request` without calling `super()` and never attached the metadata. With USE_INFERENCE_MODELS=False, YOLOv8, ResNet, ViT and DINOv3 classifiers returned nothing while object detection on the same server returned `{"model_id": ...}`. The override now attaches it before returning, as `Model.infer_from_request` does. The only legacy-mode HTTP test used object detection; classification is now covered through `/infer/classification`, a batch request, and the legacy route, plus unit cases for all four classes, single-label and multi-label, single image and list. Cosmos action recognition. The auto-loader stamps `resolved_model` on the object it returns. For a fine-tune that is a bare `Cosmos3EdgeReasoner`, which `Cosmos3EdgeActionRecognition.from_reasoner` then wraps in a new object, so the adapter found no metadata on `self._model`. The wrapper now takes the reasoner's `resolved_model` in `__init__`, which covers every `from_reasoner` return path. A loader that returns the wrapper itself still overwrites the attribute after construction. The adapter test used a fake wrapper; a new one goes through the real `_as_action_recognition_model` path with a real reasoner instance. Each new test set fails without its one-line production change. Not changed here: PP-OCR, sam3_3d and the stub model still return no `resolved_model`. Co-Authored-By: Claude Fable 5.1 --- inference/core/models/classification_base.py | 1 + inference_models/docs/changelog.md | 3 + .../cosmos3/cosmos3_action_recognition.py | 4 + .../models/test_cosmos3_action_recognition.py | 74 ++++++++++ .../http/test_resolved_model_metadata.py | 126 ++++++++++++++++++ .../models/test_action_recognition_adapter.py | 42 ++++++ .../test_legacy_resolved_model_metadata.py | 69 +++++++++- 7 files changed, 318 insertions(+), 1 deletion(-) diff --git a/inference/core/models/classification_base.py b/inference/core/models/classification_base.py index 5ff55327eb..ed37e98c7f 100644 --- a/inference/core/models/classification_base.py +++ b/inference/core/models/classification_base.py @@ -297,6 +297,7 @@ def infer_from_request( if not isinstance(request.image, list): responses = responses[0] + self._attach_resolved_model_metadata(responses) return responses def make_response( diff --git a/inference_models/docs/changelog.md b/inference_models/docs/changelog.md index f6267bc69f..7f25d1748a 100644 --- a/inference_models/docs/changelog.md +++ b/inference_models/docs/changelog.md @@ -8,6 +8,9 @@ package: canonical model ID, package ID, backend, and quantization. The public `ResolvedModelMetadata` entity describes this metadata. +- `Cosmos3EdgeActionRecognition` wrapped over an already loaded reasoner + (`from_reasoner`) exposes that reasoner's `resolved_model`. + ### Fixed - `YOLONasForObjectDetectionTRT` concatenated TRT outputs on the default CUDA stream without ordering against the post-processing stream, which could yield phantom detections under GPU contention. Concatenation now runs on the inference stream and is synchronised before post-processing. diff --git a/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py b/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py index 0cab71028c..56fd70bd94 100644 --- a/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py +++ b/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py @@ -310,6 +310,10 @@ def __init__( video_sampling: Optional[VideoSampling] = None, ): self._reasoner = reasoner + # The auto-loader stamps the package on the object it returns. When that + # was the reasoner, carry it over; a loader that returns this wrapper + # overwrites the attribute after construction. + self.resolved_model = getattr(reasoner, "resolved_model", None) self._class_names = class_names self._video_sampling = video_sampling or VideoSampling() diff --git a/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py b/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py index 28496f9f74..38dcf8736e 100644 --- a/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py +++ b/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py @@ -5,6 +5,7 @@ import pytest import torch +from inference_models.entities import ResolvedModelMetadata from inference_models.errors import CorruptedModelPackageError from inference_models.models.base.action_recognition import ( SLIDING_WINDOW_MODE, @@ -377,6 +378,79 @@ def test_from_pretrained_reads_the_recorded_sampling(monkeypatch, tmp_path) -> N assert all(frame.shape == (56, 100, 3) for frame in sent_frames) +_RESOLVED_MODEL = ResolvedModelMetadata( + model_id="workspace/actions/1", + model_package_id="hf-package", + backend="hugging-face", + quantization="bf16", +) + + +def _loaded_reasoner(package_dir=None, class_names=(), resolved_model=_RESOLVED_MODEL): + """A reasoner as the auto-loader hands it over: stamped after loading.""" + reasoner = _FakeReasoner(tokenizer=_FakeTokenizer(class_names=class_names)) + reasoner.package_dir = package_dir + if resolved_model is not None: + reasoner.resolved_model = resolved_model + return reasoner + + +def test_from_reasoner_over_a_hosted_base_exposes_the_reasoner_package() -> None: + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=_loaded_reasoner()) + + assert wrapper.class_names is None + assert wrapper.resolved_model is _RESOLVED_MODEL + + +def test_from_reasoner_over_a_fine_tune_exposes_the_reasoner_package(tmp_path) -> None: + (tmp_path / "class_names.txt").write_text("walking\nrunning\n") + reasoner = _loaded_reasoner( + package_dir=str(tmp_path), class_names=["walking", "running"] + ) + + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=reasoner) + + assert wrapper.class_names == ["walking", "running"] + assert wrapper.resolved_model is _RESOLVED_MODEL + + +def test_from_reasoner_over_a_base_package_exposes_the_reasoner_package( + tmp_path, +) -> None: + reasoner = _loaded_reasoner(package_dir=str(tmp_path)) + + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=reasoner) + + assert wrapper.class_names is None + assert wrapper.resolved_model is _RESOLVED_MODEL + + +def test_a_reasoner_loaded_outside_the_auto_loader_resolves_to_nothing() -> None: + reasoner = _loaded_reasoner(resolved_model=None) + + from_reasoner = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=reasoner) + constructed = Cosmos3EdgeActionRecognition(reasoner=reasoner) + + assert not hasattr(reasoner, "resolved_model") + assert from_reasoner.resolved_model is None + assert constructed.resolved_model is None + + +def test_the_package_the_loader_stamps_on_the_wrapper_wins() -> None: + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=_loaded_reasoner()) + stamped_by_loader = ResolvedModelMetadata( + model_id="workspace/actions/2", + model_package_id="trt-package", + backend="trt", + quantization="fp16", + ) + + wrapper.resolved_model = stamped_by_loader + + assert wrapper.resolved_model is stamped_by_loader + assert wrapper._reasoner.resolved_model is _RESOLVED_MODEL + + def test_fine_tune_parser_accepts_trailing_end_token(monkeypatch) -> None: tokenizer = _FakeTokenizer(class_names=["walking", "running"]) reasoner = _FakeReasoner( diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py index 7a66c50cf3..4703bf07ba 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -9,8 +9,10 @@ from inference.core import env from inference.core.entities.responses.inference import ( + ClassificationInferenceResponse, InferenceResponseImage, LMMInferenceResponse, + MultiLabelClassificationInferenceResponse, ObjectDetectionInferenceResponse, ) from inference.core.interfaces.http import http_api @@ -21,6 +23,10 @@ from inference.core.registries import roboflow as registry_module from inference.core.registries.base import ModelRegistry from inference.models import utils as model_utils +from inference.models.dinov3.dinov3_classification import DinoV3Classification +from inference.models.resnet.resnet_classification import ResNetClassification +from inference.models.vit.vit_classification import VitClassification +from inference.models.yolov8.yolov8_classification import YOLOv8Classification from inference.models.yolov8.yolov8_object_detection import YOLOv8ObjectDetection from inference_models.utils.content_addressed_artifact_cache import ( NullContentAddressedArtifactCache, @@ -174,6 +180,126 @@ def initialize_legacy_model(self, model_id, **kwargs): } +def _legacy_classification_infer(multi_label): + def infer(self, image, **kwargs): + image_metadata = InferenceResponseImage(width=640, height=480) + if multi_label: + response = MultiLabelClassificationInferenceResponse( + image=image_metadata, + predictions={"cat": {"confidence": 0.9, "class_id": 0}}, + predicted_classes=["cat"], + ) + else: + response = ClassificationInferenceResponse( + image=image_metadata, + predictions=[{"class": "cat", "class_id": 0, "confidence": 0.9}], + top="cat", + confidence=0.9, + ) + images = image if isinstance(image, list) else [image] + return [response.model_copy() for _ in images] + + return infer + + +def _build_legacy_classification_client( + monkeypatch, model_type, model_class, multi_label +): + monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) + model_types = runpy.run_path(model_utils.__file__)["ROBOFLOW_MODEL_TYPES"] + monkeypatch.setattr( + registry_module, + "get_model_type", + lambda *args, **kwargs: ("classification", model_type), + ) + + def initialize_legacy_model(self, model_id, **kwargs): + self.endpoint = model_id + + monkeypatch.setattr(model_class, "__init__", initialize_legacy_model) + monkeypatch.setattr(model_class, "infer", _legacy_classification_infer(multi_label)) + return build_client( + monkeypatch, + flag=False, + registry=registry_module.RoboflowModelRegistry(model_types), + preload=False, + ) + + +@pytest.mark.parametrize("multi_label", [False, True], ids=["single", "multi"]) +@pytest.mark.parametrize( + "model_type,model_class", + [ + ("yolov8", YOLOv8Classification), + ("resnet18", ResNetClassification), + ("vit", VitClassification), + ("dinov3", DinoV3Classification), + ], +) +def test_http_classification_reports_model_id_when_flag_is_disabled( + monkeypatch, model_type, model_class, multi_label +): + client, manager = _build_legacy_classification_client( + monkeypatch, model_type, model_class, multi_label + ) + + response = client.post( + "/infer/classification", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert type(manager.models()["test/1"]) is model_class + payload = response.json() + assert payload["predicted_classes" if multi_label else "top"] + assert payload["resolved_model"] == {"model_id": "test/1"} + + +def test_http_classification_batch_reports_model_id_when_flag_is_disabled( + monkeypatch, +): + client, _ = _build_legacy_classification_client( + monkeypatch, "yolov8", YOLOv8Classification, multi_label=False + ) + + response = client.post( + "/infer/classification", + json={ + "model_id": "test/1", + "image": [ + {"type": "base64", "value": "first"}, + {"type": "base64", "value": "second"}, + ], + }, + ) + + assert response.status_code == 200, response.text + assert [item["resolved_model"] for item in response.json()] == [ + {"model_id": "test/1"}, + {"model_id": "test/1"}, + ] + + +def test_legacy_route_classification_reports_model_id_when_flag_is_disabled( + monkeypatch, +): + client, _ = _build_legacy_classification_client( + monkeypatch, "yolov8", YOLOv8Classification, multi_label=False + ) + + response = client.post( + "/test/1", + content="image", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["resolved_model"] == {"model_id": "test/1"} + + @pytest.mark.parametrize("api_version", ["v0", "v1"]) def test_sdk_preserves_http_package_metadata(monkeypatch, requests_mock, api_version): server, _ = build_client(monkeypatch) diff --git a/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py b/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py index b2a4a9971b..63b7da74c0 100644 --- a/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py +++ b/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py @@ -114,6 +114,48 @@ def test_clip_response_reports_the_loaded_package(monkeypatch) -> None: assert response.model_dump()["resolved_model"] == vars(model.resolved_model) +def test_clip_response_reports_the_package_of_a_wrapped_reasoner(monkeypatch) -> None: + from inference.core.models import base + from inference.core.models import inference_models_adapters as adapters + from inference_models.entities import ResolvedModelMetadata + from inference_models.models.cosmos3.cosmos3_action_recognition import ( + Cosmos3EdgeActionRecognition, + ) + from inference_models.models.cosmos3.cosmos3_reasoner_hf import Cosmos3EdgeReasoner + + monkeypatch.setattr(base, "USE_INFERENCE_MODELS", True) + # The registry hands a fine-tune over as a bare reasoner, and the + # auto-loader stamps the package on that reasoner, not on the wrapper. + reasoner = Cosmos3EdgeReasoner.__new__(Cosmos3EdgeReasoner) + reasoner._processor = SimpleNamespace(tokenizer=None) + reasoner.package_dir = None + reasoner.prompt_video = MagicMock(return_value="") + reasoner.resolved_model = ResolvedModelMetadata( + model_id="workspace/model", + model_package_id="video-package", + backend="hugging-face", + quantization="bf16", + ) + + with patch.object(adapters, "AutoModel") as auto_model: + auto_model.from_pretrained.return_value = reasoner + adapter = adapters.InferenceModelsActionRecognitionAdapter( + model_id="workspace/model", api_key="key" + ) + with _clip(frame_count=3, source_fps=10.0): + response = adapter.infer_from_request(_request()) + + assert isinstance(adapter._model, Cosmos3EdgeActionRecognition) + assert adapter._model._reasoner is reasoner + assert reasoner.prompt_video.call_count == 1 + assert response.model_dump()["resolved_model"] == { + "model_id": "workspace/model", + "model_package_id": "video-package", + "backend": "hugging-face", + "quantization": "bf16", + } + + def test_ranges_of_one_class_merge_across_windows() -> None: model = _FakeModel( responses=[ diff --git a/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py index 89954efd57..4eed21cfca 100644 --- a/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py +++ b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py @@ -12,12 +12,20 @@ from inference.core.entities.requests.clip import ClipTextEmbeddingRequest from inference.core.entities.requests.doctr import DoctrOCRInferenceRequest from inference.core.entities.requests.easy_ocr import EasyOCRInferenceRequest -from inference.core.entities.requests.inference import InferenceRequestImage +from inference.core.entities.requests.inference import ( + ClassificationInferenceRequest, + InferenceRequestImage, +) from inference.core.entities.requests.sam import ( SamEmbeddingRequest, SamSegmentationRequest, ) +from inference.core.entities.responses.inference import ( + ClassificationInferenceResponse, + MultiLabelClassificationInferenceResponse, +) from inference.core.entities.responses.ocr import OCRInferenceResponse +from inference.core.models.types import PreprocessReturnMetadata IMAGE = InferenceRequestImage(type="base64", value="unused") MODEL_ID = "foundation/model" @@ -106,6 +114,65 @@ def test_legacy_ocr_request_metadata( assert result.model_dump()["resolved_model"] == METADATA +@pytest.mark.parametrize("batched", [False, True], ids=["single-image", "image-list"]) +@pytest.mark.parametrize( + "multiclass,response_class", + [ + (False, ClassificationInferenceResponse), + (True, MultiLabelClassificationInferenceResponse), + ], + ids=["single-label", "multi-label"], +) +@pytest.mark.parametrize( + "module_name,class_name", + [ + ("yolov8.yolov8_classification", "YOLOv8Classification"), + ("resnet.resnet_classification", "ResNetClassification"), + ("vit.vit_classification", "VitClassification"), + ("dinov3.dinov3_classification", "DinoV3Classification"), + ], +) +def test_legacy_classification_request_metadata( + load_legacy_model, module_name, class_name, multiclass, response_class, batched +): + batch_size = 2 if batched else 1 + model = load_legacy_model(module_name, class_name) + model.multiclass = multiclass + model.class_names = ["cat", "dog"] + model.batching_enabled = False + model.batch_size = 1 + + def preprocess(image, **kwargs): + images_count = len(image) if isinstance(image, list) else 1 + return ( + np.zeros((images_count, 3, 2, 2), dtype=np.float32), + PreprocessReturnMetadata({"img_dims": [(2, 2)] * images_count}), + ) + + model.preprocess = Mock(side_effect=preprocess) + model.predict = Mock( + side_effect=lambda img_in, **kwargs: ( + np.array([[[0.9, 0.1]]] * img_in.shape[0], dtype=np.float32), + ) + ) + + response = model.infer_from_request( + ClassificationInferenceRequest( + id="test-request", + model_id=MODEL_ID, + image=[IMAGE, IMAGE] if batched else IMAGE, + ) + ) + + assert isinstance(response, list) == batched + responses = response if batched else [response] + assert len(responses) == batch_size + for result in responses: + assert isinstance(result, response_class) + assert result.resolved_model.model_dump() == METADATA + assert result.model_dump()["resolved_model"] == METADATA + + @pytest.mark.parametrize("response_format", ["json", "binary"]) def test_legacy_sam_embedding_metadata(load_legacy_model, monkeypatch, response_format): for module_name in ["rasterio", "rasterio.features", "segment_anything"]: From 9fbafd014dc5b4fb1ffec530b3d9a488ac8ca7a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 18 Sep 2026 16:52:38 +0200 Subject: [PATCH 15/18] Bump dependencies --- inference_models/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference_models/pyproject.toml b/inference_models/pyproject.toml index 2be9e2a74c..590c60d6e2 100644 --- a/inference_models/pyproject.toml +++ b/inference_models/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "inference-models" -version = "0.37.1" +version = "0.37.2" description = "The new inference engine for Computer Vision models" readme = "README.md" requires-python = ">=3.10,<3.13" From 23df2d78c20ae11fb2c9b61ecc2419efa2f95081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 18 Sep 2026 17:51:06 +0200 Subject: [PATCH 16/18] Move inference-models changelog entries under 0.37.2 The package version on this branch is 0.37.2, so the entries that sat under "Unreleased" ship in that release: `resolved_model` metadata from `AutoModel.from_pretrained`, the same metadata on a `Cosmos3EdgeActionRecognition` wrapped over a loaded reasoner, and the YOLO-NAS TRT stream ordering fix. An empty "Unreleased" section stays on top, as after earlier releases. Also correct a heading. The `GitPython` lower-bound entry was added by the commit that bumped 0.37.0 to 0.37.1 (#2972) but was titled `0.37.0`, which left two `0.37.0` sections and no `0.37.1`. Co-Authored-By: Claude Fable 5.1 --- inference_models/docs/changelog.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/inference_models/docs/changelog.md b/inference_models/docs/changelog.md index 7f25d1748a..8465076610 100644 --- a/inference_models/docs/changelog.md +++ b/inference_models/docs/changelog.md @@ -2,6 +2,10 @@ ## Unreleased +--- + +## `0.37.2` + ### Added - `AutoModel.from_pretrained` exposes `resolved_model` metadata for the loaded @@ -17,7 +21,7 @@ --- -## `0.37.0` +## `0.37.1` ### Fixed From c07dc149be8ec3fbff95ccd88653f27939374ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 18 Sep 2026 17:59:42 +0200 Subject: [PATCH 17/18] Pin inference-models 0.37.2 in the inference requirements `inference_models` is 0.37.2 on this branch and that version is published on PyPI. The server reports `resolved_model` package metadata only when it runs that release, so move the pin in every requirements file that carries it: cpu, gpu, gpu.cu13, jetson and vino. Also set the package's own version in `inference_models/uv.lock` to 0.37.2, as the 0.37.1 bump (#2972) did. Only that line changes; the lock was not regenerated. Co-Authored-By: Claude Fable 5.1 --- inference_models/uv.lock | 2 +- requirements/requirements.cpu.txt | 2 +- requirements/requirements.gpu.cu13.txt | 2 +- requirements/requirements.gpu.txt | 2 +- requirements/requirements.jetson.txt | 2 +- requirements/requirements.vino.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/inference_models/uv.lock b/inference_models/uv.lock index c63c2eaab1..8382f7b0cb 100644 --- a/inference_models/uv.lock +++ b/inference_models/uv.lock @@ -1044,7 +1044,7 @@ wheels = [ [[package]] name = "inference-models" -version = "0.37.1" +version = "0.37.2" source = { virtual = "." } dependencies = [ { name = "accelerate" }, diff --git a/requirements/requirements.cpu.txt b/requirements/requirements.cpu.txt index 516a75eb36..ba385628ee 100644 --- a/requirements/requirements.cpu.txt +++ b/requirements/requirements.cpu.txt @@ -1,3 +1,3 @@ onnxruntime>=1.15.1,<1.22.0 nvidia-ml-py<13.0.0 -inference-models[torch-cpu,onnx-cpu]~=0.37.1 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu,onnx-cpu]~=0.37.2 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.gpu.cu13.txt b/requirements/requirements.gpu.cu13.txt index 8a72a5e5b7..aa8a46d9a0 100644 --- a/requirements/requirements.gpu.cu13.txt +++ b/requirements/requirements.gpu.cu13.txt @@ -4,6 +4,6 @@ # is a CUDA 12 build), so the Dockerfile installs it in a dedicated step from # Microsoft's onnxruntime-cuda-13 feed. A plain pin here would silently # resolve to the CUDA 12 wheel. -inference-models[torch-cu130]~=0.37.1 +inference-models[torch-cu130]~=0.37.2 pynvvideocodec~=2.1.0; platform_machine == "x86_64" onnxruntime-gpu>=1.27.0,<1.30.0 \ No newline at end of file diff --git a/requirements/requirements.gpu.txt b/requirements/requirements.gpu.txt index bbfc239e21..ff4e238c89 100644 --- a/requirements/requirements.gpu.txt +++ b/requirements/requirements.gpu.txt @@ -1,3 +1,3 @@ onnxruntime-gpu>=1.15.1,<1.22.0 -inference-models[torch-cu124,onnx-cu12]~=0.37.1 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cu124,onnx-cu12]~=0.37.2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt pynvvideocodec~=2.1.0; platform_machine == "x86_64" diff --git a/requirements/requirements.jetson.txt b/requirements/requirements.jetson.txt index 1d4ba27850..1f7ce72a93 100644 --- a/requirements/requirements.jetson.txt +++ b/requirements/requirements.jetson.txt @@ -1,3 +1,3 @@ pypdfium2>=4.11.0,<5.0.0 PyYAML~=6.0.0 -inference-models~=0.37.1 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models~=0.37.2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.vino.txt b/requirements/requirements.vino.txt index ff26f4e7e9..92a4cb311d 100644 --- a/requirements/requirements.vino.txt +++ b/requirements/requirements.vino.txt @@ -1,2 +1,2 @@ onnxruntime-openvino>=1.15.0,<1.22.0 -inference-models[torch-cpu]~=0.37.1 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu]~=0.37.2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt From 013ec330aea5b3502c64d23f43bd2bb519000ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 18 Sep 2026 18:12:27 +0200 Subject: [PATCH 18/18] Make linters happy --- inference/core/workflows/__init__.py | 1 + inference/enterprise/workflows/__init__.py | 1 + .../core_steps/visualizations/common/fonts/__init__.py | 1 - .../tests/stub_plugins/scalar_only_block_plugin/__init__.py | 1 - .../unit_tests/core_steps/common/test_deserializers.py | 4 +--- .../introspection/plugin_duplicated_identifiers/__init__.py | 6 +----- .../plugin_with_multiple_versions_of_blocks/__init__.py | 6 +----- .../introspection/plugin_with_valid_blocks/__init__.py | 6 +----- 8 files changed, 6 insertions(+), 20 deletions(-) diff --git a/inference/core/workflows/__init__.py b/inference/core/workflows/__init__.py index bb869d6bcb..ed2f9d87b2 100644 --- a/inference/core/workflows/__init__.py +++ b/inference/core/workflows/__init__.py @@ -4,6 +4,7 @@ installs the compat finder and rebinds this module in ``sys.modules`` to the canonical package so the two names share one object. """ + from inference._workflows_compat import alias_legacy_root as _alias_legacy_root _alias_legacy_root(__name__) diff --git a/inference/enterprise/workflows/__init__.py b/inference/enterprise/workflows/__init__.py index ef4fb09ce6..58427406b4 100644 --- a/inference/enterprise/workflows/__init__.py +++ b/inference/enterprise/workflows/__init__.py @@ -6,6 +6,7 @@ but it installs the compat finder so child imports of ``enterprise_blocks`` resolve to canonical modules. """ + from inference._workflows_compat import install as _install _install() diff --git a/workflows/roboflow_workflows/core_steps/visualizations/common/fonts/__init__.py b/workflows/roboflow_workflows/core_steps/visualizations/common/fonts/__init__.py index 9ce1fb8269..e524779718 100644 --- a/workflows/roboflow_workflows/core_steps/visualizations/common/fonts/__init__.py +++ b/workflows/roboflow_workflows/core_steps/visualizations/common/fonts/__init__.py @@ -25,7 +25,6 @@ from typing import Set from filelock import FileLock - from roboflow_workflows.core_steps.visualizations.common.fonts.downloader import ( FontDownloadError, compute_file_sha256, diff --git a/workflows/tests/stub_plugins/scalar_only_block_plugin/__init__.py b/workflows/tests/stub_plugins/scalar_only_block_plugin/__init__.py index 14d359a293..123e411fa0 100644 --- a/workflows/tests/stub_plugins/scalar_only_block_plugin/__init__.py +++ b/workflows/tests/stub_plugins/scalar_only_block_plugin/__init__.py @@ -7,7 +7,6 @@ from typing import List, Literal, Optional, Type, Union from pydantic import Field - from roboflow_workflows.execution_engine.entities.base import OutputDefinition from roboflow_workflows.execution_engine.entities.types import Selector from roboflow_workflows.prototypes.block import ( diff --git a/workflows/tests/unit_tests/core_steps/common/test_deserializers.py b/workflows/tests/unit_tests/core_steps/common/test_deserializers.py index d71ebc0df2..7848637327 100644 --- a/workflows/tests/unit_tests/core_steps/common/test_deserializers.py +++ b/workflows/tests/unit_tests/core_steps/common/test_deserializers.py @@ -23,9 +23,7 @@ deserialize_timestamp, deserialize_zone_kind, ) -from roboflow_workflows.core_steps.common.serializers import ( - serialise_sv_detections, -) +from roboflow_workflows.core_steps.common.serializers import serialise_sv_detections from roboflow_workflows.errors import RuntimeInputError from roboflow_workflows.execution_engine.constants import IMAGE_DIMENSIONS_KEY from roboflow_workflows.execution_engine.entities.base import ( diff --git a/workflows/tests/unit_tests/execution_engine/introspection/plugin_duplicated_identifiers/__init__.py b/workflows/tests/unit_tests/execution_engine/introspection/plugin_duplicated_identifiers/__init__.py index ca7db6187a..d4b0eb1021 100644 --- a/workflows/tests/unit_tests/execution_engine/introspection/plugin_duplicated_identifiers/__init__.py +++ b/workflows/tests/unit_tests/execution_engine/introspection/plugin_duplicated_identifiers/__init__.py @@ -1,7 +1,6 @@ from typing import Any, Dict, List, Literal, Tuple, Type, Union from pydantic import Field - from roboflow_workflows.execution_engine.entities.base import OutputDefinition from roboflow_workflows.execution_engine.entities.types import ( Kind, @@ -9,10 +8,7 @@ WorkflowParameterSelector, ) from roboflow_workflows.execution_engine.v1.entities import FlowControl -from roboflow_workflows.prototypes.block import ( - WorkflowBlock, - WorkflowBlockManifest, -) +from roboflow_workflows.prototypes.block import WorkflowBlock, WorkflowBlockManifest MY_KIND_1 = Kind(name="1") MY_KIND_2 = Kind(name="2") diff --git a/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_multiple_versions_of_blocks/__init__.py b/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_multiple_versions_of_blocks/__init__.py index 36e059311a..075186b22b 100644 --- a/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_multiple_versions_of_blocks/__init__.py +++ b/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_multiple_versions_of_blocks/__init__.py @@ -1,7 +1,6 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union from pydantic import ConfigDict, Field - from roboflow_workflows.execution_engine.entities.base import OutputDefinition from roboflow_workflows.execution_engine.entities.types import ( Kind, @@ -9,10 +8,7 @@ WorkflowParameterSelector, ) from roboflow_workflows.execution_engine.v1.entities import FlowControl -from roboflow_workflows.prototypes.block import ( - WorkflowBlock, - WorkflowBlockManifest, -) +from roboflow_workflows.prototypes.block import WorkflowBlock, WorkflowBlockManifest MY_KIND_1 = Kind(name="1") MY_KIND_2 = Kind(name="2") diff --git a/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_valid_blocks/__init__.py b/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_valid_blocks/__init__.py index e50b18931e..4330477ee6 100644 --- a/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_valid_blocks/__init__.py +++ b/workflows/tests/unit_tests/execution_engine/introspection/plugin_with_valid_blocks/__init__.py @@ -1,7 +1,6 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union from pydantic import Field - from roboflow_workflows.execution_engine.entities.base import OutputDefinition from roboflow_workflows.execution_engine.entities.types import ( Kind, @@ -9,10 +8,7 @@ WorkflowParameterSelector, ) from roboflow_workflows.execution_engine.v1.entities import FlowControl -from roboflow_workflows.prototypes.block import ( - WorkflowBlock, - WorkflowBlockManifest, -) +from roboflow_workflows.prototypes.block import WorkflowBlock, WorkflowBlockManifest MY_KIND_1 = Kind(name="1") MY_KIND_2 = Kind(name="2")