diff --git a/inference/core/entities/responses/action_recognition.py b/inference/core/entities/responses/action_recognition.py index 709123274f..b8a6b856f4 100644 --- a/inference/core/entities/responses/action_recognition.py +++ b/inference/core/entities/responses/action_recognition.py @@ -1,13 +1,14 @@ from typing import List -from pydantic import BaseModel, Field +from pydantic import Field +from inference.core.entities.responses.inference import InferenceResponse from inference.core.workflows.core_steps.models.roboflow.action_recognition.entities import ( # noqa: F401 ActionRecognitionPrediction, ) -class ActionRecognitionInferenceResponse(BaseModel): +class ActionRecognitionInferenceResponse(InferenceResponse): """Classified ranges covering one clip. Frame indices count from the first frame of the submitted clip, so a diff --git a/inference/core/entities/responses/inference.py b/inference/core/entities/responses/inference.py index 8b339ffccf..d16e53e410 100644 --- a/inference/core/entities/responses/inference.py +++ b/inference/core/entities/responses/inference.py @@ -10,6 +10,7 @@ InferenceResponse, InferenceResponseImage, InstanceSegmentationInferenceResponse, + ResolvedModel, WithVisualizationResponse, ) from inference.core.workflows.core_steps.common.segmentation_entities import ( # noqa: F401 @@ -232,6 +233,7 @@ class InstanceSegmentationInferenceResponseDC: frame_id: object = None time: object = None visualization: object = None + resolved_model: Optional[ResolvedModel] = None # Internal stream-pipeline fast path: lets workflow execution carry a # response future through Model.infer_from_request without blocking the # inference thread. `_is_response_dc_to_dict` intentionally ignores it. @@ -283,6 +285,8 @@ def _is_response_dc_to_dict(r: InstanceSegmentationInferenceResponseDC) -> dict: d["time"] = r.time if r.visualization is not None: d["visualization"] = r.visualization + if r.resolved_model is not None: + d["resolved_model"] = r.resolved_model.model_dump() return d @@ -362,7 +366,7 @@ class FaceDetectionPrediction(ObjectDetectionPrediction): landmarks: Union[List[Point], List[Point3D]] -class DepthEstimationResponse(BaseModel): +class DepthEstimationResponse(InferenceResponse): """Response for depth estimation inference. Attributes: diff --git a/inference/core/entities/responses/ocr.py b/inference/core/entities/responses/ocr.py index ea3f9a3c7d..55d9497fc0 100644 --- a/inference/core/entities/responses/ocr.py +++ b/inference/core/entities/responses/ocr.py @@ -1,14 +1,15 @@ from typing import List, Optional -from pydantic import BaseModel, Field +from pydantic import Field from inference.core.entities.responses.inference import ( + InferenceResponse, InferenceResponseImage, ObjectDetectionPrediction, ) -class OCRInferenceResponse(BaseModel): +class OCRInferenceResponse(InferenceResponse): """ OCR Inference response. diff --git a/inference/core/entities/responses/sam.py b/inference/core/entities/responses/sam.py index f455219ca5..8d595a1098 100644 --- a/inference/core/entities/responses/sam.py +++ b/inference/core/entities/responses/sam.py @@ -1,9 +1,11 @@ from typing import Any, List, Union -from pydantic import BaseModel, Field +from pydantic import Field +from inference.core.entities.responses.inference import InferenceResponse -class SamEmbeddingResponse(BaseModel): + +class SamEmbeddingResponse(InferenceResponse): """SAM embedding response. Attributes: @@ -20,7 +22,7 @@ class SamEmbeddingResponse(BaseModel): ) -class SamSegmentationResponse(BaseModel): +class SamSegmentationResponse(InferenceResponse): """SAM segmentation response. Attributes: diff --git a/inference/core/entities/responses/sam2.py b/inference/core/entities/responses/sam2.py index 04730851ae..4afdc1ffa5 100644 --- a/inference/core/entities/responses/sam2.py +++ b/inference/core/entities/responses/sam2.py @@ -1,13 +1,14 @@ from typing import List -from pydantic import BaseModel, Field +from pydantic import Field +from inference.core.entities.responses.inference import InferenceResponse from inference.core.workflows.core_steps.common.segmentation_entities import ( # noqa: F401 Sam2SegmentationPrediction, ) -class Sam2EmbeddingResponse(BaseModel): +class Sam2EmbeddingResponse(InferenceResponse): """SAM embedding response. Attributes: @@ -21,7 +22,7 @@ class Sam2EmbeddingResponse(BaseModel): ) -class Sam2SegmentationResponse(BaseModel): +class Sam2SegmentationResponse(InferenceResponse): predictions: List[Sam2SegmentationPrediction] = Field() time: float = Field( description="The time in seconds it took to produce the segmentation including preprocessing" diff --git a/inference/core/entities/responses/sam3.py b/inference/core/entities/responses/sam3.py index 846f27a83e..cfbc8678a2 100644 --- a/inference/core/entities/responses/sam3.py +++ b/inference/core/entities/responses/sam3.py @@ -2,8 +2,10 @@ from pydantic import BaseModel, Field +from inference.core.entities.responses.inference import InferenceResponse -class Sam3EmbeddingResponse(BaseModel): + +class Sam3EmbeddingResponse(InferenceResponse): image_id: str = Field(description="Image id embeddings are cached to") time: float = Field( description="The time in seconds it took to produce the embeddings including preprocessing" @@ -33,7 +35,7 @@ class Sam3PromptResult(BaseModel): predictions: List[Sam3SegmentationPrediction] = Field() -class Sam3SegmentationResponse(BaseModel): +class Sam3SegmentationResponse(InferenceResponse): prompt_results: List[Sam3PromptResult] = Field() time: float = Field( description="The time in seconds it took to produce the segmentation including preprocessing" diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index 5cead0018b..1f1ddeda68 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -4256,6 +4256,7 @@ def _infer_depth_estimation( depth_data["normalized_depth"] ) return DepthEstimationResponse( + resolved_model=getattr(response, "resolved_model", None), normalized_depth=serialized_depth, depth_map_format=inference_request.depth_map_format, image=depth_data["image"].base64_image, diff --git a/inference/core/models/base.py b/inference/core/models/base.py index 2be7df64a5..547da9f601 100644 --- a/inference/core/models/base.py +++ b/inference/core/models/base.py @@ -5,7 +5,11 @@ from inference.core import logger from inference.core.entities.requests.inference import InferenceRequest -from inference.core.entities.responses.inference import InferenceResponse +from inference.core.entities.responses.inference import ( + InferenceResponse, + InstanceSegmentationInferenceResponseDC, + ResolvedModel, +) from inference.core.env import USE_INFERENCE_MODELS from inference.core.models.types import PreprocessReturnMetadata from inference.core.telemetry import set_span_attribute, start_span @@ -106,6 +110,31 @@ class Model(BaseInference): clear_cache(): Clears any cache if necessary. """ + @property + def resolved_model(self) -> Any: + model = getattr(self, "_model", None) + if USE_INFERENCE_MODELS and model is not None: + return getattr(model, "resolved_model", None) + model_id = getattr(self, "endpoint", None) or getattr(self, "model_id", None) + return ResolvedModel(model_id=model_id) if model_id is not None else None + + def _attach_resolved_model_metadata(self, responses: Any) -> None: + metadata = self.resolved_model + if metadata is None: + return + resolved_model = ResolvedModel( + model_id=metadata.model_id, + model_package_id=metadata.model_package_id, + backend=metadata.backend, + quantization=metadata.quantization, + ) + responses = responses if isinstance(responses, list) else [responses] + for response in responses: + if isinstance( + response, (InferenceResponse, InstanceSegmentationInferenceResponseDC) + ): + response.resolved_model = resolved_model + def log(self, m): """Prints the given message. @@ -181,6 +210,7 @@ def infer_from_request( if not isinstance(request.image, list) and len(responses) > 0: responses = responses[0] + self._attach_resolved_model_metadata(responses) return responses def make_response( diff --git a/inference/core/models/classification_base.py b/inference/core/models/classification_base.py index 5ff55327eb..ed37e98c7f 100644 --- a/inference/core/models/classification_base.py +++ b/inference/core/models/classification_base.py @@ -297,6 +297,7 @@ def infer_from_request( if not isinstance(request.image, list): responses = responses[0] + self._attach_resolved_model_metadata(responses) return responses def make_response( diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index dbf0735870..7bba44b435 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -1508,6 +1508,7 @@ def infer_from_request( if not isinstance(request.image, list): responses = responses[0] + self._attach_resolved_model_metadata(responses) return responses @@ -2040,12 +2041,14 @@ def infer_from_request( stride=max(1.0, source_fps / window.sample_fps), ) timeline.sort(key=lambda entry: (entry.start_frame_idx, entry.class_id)) - return ActionRecognitionInferenceResponse( + response = ActionRecognitionInferenceResponse( timeline=timeline, source_fps=source_fps, frame_count=frame_count, windows_classified=windows_classified, ) + self._attach_resolved_model_metadata(response) + return response def preprocess(self, *args, **kwargs): raise NotImplementedError( diff --git a/inference/core/workflows/__init__.py b/inference/core/workflows/__init__.py index bb869d6bcb..ed2f9d87b2 100644 --- a/inference/core/workflows/__init__.py +++ b/inference/core/workflows/__init__.py @@ -4,6 +4,7 @@ installs the compat finder and rebinds this module in ``sys.modules`` to the canonical package so the two names share one object. """ + from inference._workflows_compat import alias_legacy_root as _alias_legacy_root _alias_legacy_root(__name__) diff --git a/inference/enterprise/workflows/__init__.py b/inference/enterprise/workflows/__init__.py index ef4fb09ce6..58427406b4 100644 --- a/inference/enterprise/workflows/__init__.py +++ b/inference/enterprise/workflows/__init__.py @@ -6,6 +6,7 @@ but it installs the compat finder so child imports of ``enterprise_blocks`` resolve to canonical modules. """ + from inference._workflows_compat import install as _install _install() diff --git a/inference/models/clip/clip_inference_models.py b/inference/models/clip/clip_inference_models.py index 68bcaf63c9..10e784ff9c 100644 --- a/inference/models/clip/clip_inference_models.py +++ b/inference/models/clip/clip_inference_models.py @@ -343,6 +343,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/clip/clip_model.py b/inference/models/clip/clip_model.py index 4c0e5d5370..96affa5e37 100644 --- a/inference/models/clip/clip_model.py +++ b/inference/models/clip/clip_model.py @@ -335,6 +335,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/doctr/doctr_model.py b/inference/models/doctr/doctr_model.py index 490583938c..dfdd26332f 100644 --- a/inference/models/doctr/doctr_model.py +++ b/inference/models/doctr/doctr_model.py @@ -104,8 +104,11 @@ def infer_from_request( for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) + self._attach_resolved_model_metadata(response) return response - return self.single_request(request) + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: DoctrOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/doctr/doctr_model_inference_models.py b/inference/models/doctr/doctr_model_inference_models.py index f1f96d7af5..151f9e460b 100644 --- a/inference/models/doctr/doctr_model_inference_models.py +++ b/inference/models/doctr/doctr_model_inference_models.py @@ -82,12 +82,14 @@ def infer_from_request( ) -> Union[OCRInferenceResponse, List]: if type(request.image) is list: response = [] - request_copy = copy.copy(request) + request_copy = copy(request) for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) - return response - return self.single_request(request) + else: + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: DoctrOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/easy_ocr/easy_ocr.py b/inference/models/easy_ocr/easy_ocr.py index 25432c4cc0..0461e2e2ed 100644 --- a/inference/models/easy_ocr/easy_ocr.py +++ b/inference/models/easy_ocr/easy_ocr.py @@ -127,8 +127,11 @@ def infer_from_request( for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) + self._attach_resolved_model_metadata(response) return response - return self.single_request(request) + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: EasyOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/easy_ocr/easy_ocr_inference_models.py b/inference/models/easy_ocr/easy_ocr_inference_models.py index 14269988c1..da4f713621 100644 --- a/inference/models/easy_ocr/easy_ocr_inference_models.py +++ b/inference/models/easy_ocr/easy_ocr_inference_models.py @@ -102,8 +102,10 @@ def infer_from_request( for image in request.image: request_copy.image = image response.append(self.single_request(request=request_copy)) - return response - return self.single_request(request) + else: + response = self.single_request(request) + self._attach_resolved_model_metadata(response) + return response def single_request(self, request: EasyOCRInferenceRequest) -> OCRInferenceResponse: t1 = perf_counter() diff --git a/inference/models/grounding_dino/grounding_dino.py b/inference/models/grounding_dino/grounding_dino.py index fbe4daca10..cf73785d76 100644 --- a/inference/models/grounding_dino/grounding_dino.py +++ b/inference/models/grounding_dino/grounding_dino.py @@ -151,6 +151,7 @@ def infer_from_request( Perform inference based on the details provided in the request, and return the associated responses. """ result = self.infer(**request.dict()) + self._attach_resolved_model_metadata(result) return result @usage_collector("model") diff --git a/inference/models/grounding_dino/grounding_dino_inference_models.py b/inference/models/grounding_dino/grounding_dino_inference_models.py index 14782cd4a5..4a3a1faa5c 100644 --- a/inference/models/grounding_dino/grounding_dino_inference_models.py +++ b/inference/models/grounding_dino/grounding_dino_inference_models.py @@ -95,6 +95,7 @@ def infer_from_request( Perform inference based on the details provided in the request, and return the associated responses. """ result = self.infer(**request.dict()) + self._attach_resolved_model_metadata(result) return result @usage_collector("model") diff --git a/inference/models/perception_encoder/perception_encoder.py b/inference/models/perception_encoder/perception_encoder.py index cc77d97231..ee661fc114 100644 --- a/inference/models/perception_encoder/perception_encoder.py +++ b/inference/models/perception_encoder/perception_encoder.py @@ -301,6 +301,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/perception_encoder/perception_encoder_inference_models.py b/inference/models/perception_encoder/perception_encoder_inference_models.py index ae24fa6042..6a96f7c986 100644 --- a/inference/models/perception_encoder/perception_encoder_inference_models.py +++ b/inference/models/perception_encoder/perception_encoder_inference_models.py @@ -287,6 +287,7 @@ def infer_from_request( data = infer_func(**request.dict()) response = make_response_func(data) response.time = perf_counter() - t1 + self._attach_resolved_model_metadata(response) return response def make_response(self, embeddings, *args, **kwargs) -> InferenceResponse: diff --git a/inference/models/sam/segment_anything.py b/inference/models/sam/segment_anything.py index c1e8279ba2..024b0a47f3 100644 --- a/inference/models/sam/segment_anything.py +++ b/inference/models/sam/segment_anything.py @@ -141,16 +141,20 @@ def infer_from_request(self, request: SamInferenceRequest): embedding, _ = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 if request.format == "json": - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=embedding.tolist(), time=inference_time ) + self._attach_resolved_model_metadata(response) + return response elif request.format == "binary": binary_vector = BytesIO() np.save(binary_vector, embedding) binary_vector.seek(0) - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=binary_vector.getvalue(), time=inference_time ) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, SamSegmentationRequest): masks, low_res_masks = self.segment_image(**request.dict()) if request.format == "json": @@ -174,6 +178,7 @@ def infer_from_request(self, request: SamInferenceRequest): low_res_masks=[m.tolist() for m in low_res_masks], time=perf_counter() - t1, ) + self._attach_resolved_model_metadata(response) return response def preproc_image(self, image: InferenceRequestImage): diff --git a/inference/models/sam/segment_anything_inference_models.py b/inference/models/sam/segment_anything_inference_models.py index 66a963ce78..699bd28bae 100644 --- a/inference/models/sam/segment_anything_inference_models.py +++ b/inference/models/sam/segment_anything_inference_models.py @@ -110,16 +110,18 @@ def infer_from_request(self, request: SamInferenceRequest): embedding, _ = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 if request.format == "json": - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=embedding.tolist(), time=inference_time ) else: binary_vector = BytesIO() np.save(binary_vector, embedding) binary_vector.seek(0) - return SamEmbeddingResponse( + response = SamEmbeddingResponse( embeddings=binary_vector.getvalue(), time=inference_time ) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, SamSegmentationRequest): masks, low_res_masks = self.segment_image(**request.dict()) if request.format == "json": @@ -143,6 +145,7 @@ def infer_from_request(self, request: SamInferenceRequest): low_res_masks=[m.tolist() for m in low_res_masks], time=perf_counter() - t1, ) + self._attach_resolved_model_metadata(response) return response def embed_image(self, image: Any, image_id: Optional[str] = None, **kwargs): diff --git a/inference/models/sam2/segment_anything2.py b/inference/models/sam2/segment_anything2.py index 8b52dcc603..b3553730da 100644 --- a/inference/models/sam2/segment_anything2.py +++ b/inference/models/sam2/segment_anything2.py @@ -196,24 +196,30 @@ def infer_from_request(self, request: Sam2InferenceRequest): if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 - return Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + response = Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, Sam2SegmentationRequest): masks, scores, low_resolution_logits = self.segment_image(**request.dict()) if request.format == "json": - return turn_segmentation_results_into_api_response( + response = turn_segmentation_results_into_api_response( masks=masks, scores=scores, mask_threshold=0.0, inference_start_timestamp=t1, ) + self._attach_resolved_model_metadata(response) + return response elif request.format == "rle": - return turn_segmentation_results_into_rle_response( + response = turn_segmentation_results_into_rle_response( masks=masks, scores=scores, mask_threshold=0.0, inference_start_timestamp=t1, ) + self._attach_resolved_model_metadata(response) + return response elif request.format == "binary": binary_vector = BytesIO() np.savez_compressed( diff --git a/inference/models/sam2/segment_anything2_inference_models.py b/inference/models/sam2/segment_anything2_inference_models.py index 648f72256a..e7e459a620 100644 --- a/inference/models/sam2/segment_anything2_inference_models.py +++ b/inference/models/sam2/segment_anything2_inference_models.py @@ -157,19 +157,21 @@ def infer_from_request(self, request: Sam2InferenceRequest): if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 - return Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + response = Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, Sam2SegmentationRequest): masks, scores, low_resolution_logits = self.segment_image(**request.dict()) if request.format == "json": - return turn_segmentation_results_into_api_response( + response = turn_segmentation_results_into_api_response( masks=masks, scores=scores, mask_threshold=MASK_THRESHOLD, inference_start_timestamp=t1, ) elif request.format == "rle": - return turn_segmentation_results_into_rle_response( + response = turn_segmentation_results_into_rle_response( masks=masks, scores=scores, mask_threshold=0.0, @@ -185,6 +187,8 @@ def infer_from_request(self, request: Sam2InferenceRequest): return binary_data else: raise ValueError(f"Invalid format {request.format}") + self._attach_resolved_model_metadata(response) + return response else: raise ValueError(f"Invalid request type {type(request)}") diff --git a/inference/models/sam3/segment_anything3.py b/inference/models/sam3/segment_anything3.py index 91018a4f9a..277fcfc4e2 100644 --- a/inference/models/sam3/segment_anything3.py +++ b/inference/models/sam3/segment_anything3.py @@ -526,6 +526,7 @@ def infer_from_request(self, request: Sam3InferenceRequest): nms_iou_threshold=request.nms_iou_threshold, ) # segment_image now returns either bytes or a response model + self._attach_resolved_model_metadata(result) return result else: raise ValueError(f"Invalid request type {type(request)}") diff --git a/inference/models/sam3/segment_anything3_inference_models.py b/inference/models/sam3/segment_anything3_inference_models.py index 248a25f1b7..ba09e39f03 100644 --- a/inference/models/sam3/segment_anything3_inference_models.py +++ b/inference/models/sam3/segment_anything3_inference_models.py @@ -90,7 +90,7 @@ def infer_from_request(self, request: Sam3InferenceRequest): record_fixed_model_input_for_request(self, request) t1 = perf_counter() if isinstance(request, Sam3SegmentationRequest): - return self.segment_image( + response = self.segment_image( image=request.image, prompts=request.prompts, output_prob_thresh=request.output_prob_thresh or 0.5, @@ -98,6 +98,8 @@ def infer_from_request(self, request: Sam3InferenceRequest): nms_iou_threshold=request.nms_iou_threshold, inference_start_timestamp=t1, ) + self._attach_resolved_model_metadata(response) + return response raise ValueError(f"Invalid request type {type(request)}") def run_tensor_native_inference(self, **kwargs) -> List[List[Dict]]: diff --git a/inference/models/sam3/visual_segmentation.py b/inference/models/sam3/visual_segmentation.py index 090540a588..b64f030932 100644 --- a/inference/models/sam3/visual_segmentation.py +++ b/inference/models/sam3/visual_segmentation.py @@ -211,14 +211,18 @@ def infer_from_request(self, request: Sam2InferenceRequest): if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) inference_time = perf_counter() - t1 - return Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + response = Sam2EmbeddingResponse(time=inference_time, image_id=image_id) + self._attach_resolved_model_metadata(response) + return response elif isinstance(request, Sam2SegmentationRequest): masks, scores, low_resolution_logits = self.segment_image(**request.dict()) predictions = _masks_to_predictions(masks, scores, request.format) - return Sam2SegmentationResponse( + response = Sam2SegmentationResponse( time=perf_counter() - t1, predictions=predictions, ) + self._attach_resolved_model_metadata(response) + return response else: raise ValueError(f"Invalid request type {type(request)}") diff --git a/inference/models/sam3/visual_segmentation_inference_models.py b/inference/models/sam3/visual_segmentation_inference_models.py index 0b11a9f98e..16a9d92aa1 100644 --- a/inference/models/sam3/visual_segmentation_inference_models.py +++ b/inference/models/sam3/visual_segmentation_inference_models.py @@ -128,27 +128,34 @@ def infer_from_request(self, request: Sam2InferenceRequest): t1 = perf_counter() if isinstance(request, Sam2EmbeddingRequest): _, _, image_id = self.embed_image(**request.dict()) - return Sam2EmbeddingResponse(time=perf_counter() - t1, image_id=image_id) + response = Sam2EmbeddingResponse( + time=perf_counter() - t1, image_id=image_id + ) + self._attach_resolved_model_metadata(response) + return response if isinstance(request, Sam2SegmentationRequest): masks, scores, low_res_logits = self.segment_image(**request.dict()) if request.format == "json" or request.format == "polygon": - return _build_polygon_response( + response = _build_polygon_response( masks=masks, scores=scores, inference_start_timestamp=t1, ) - if request.format == "rle": - return _build_rle_response( + elif request.format == "rle": + response = _build_rle_response( masks=masks, scores=scores, inference_start_timestamp=t1, ) - if request.format == "binary": + elif request.format == "binary": buf = BytesIO() np.savez_compressed(buf, masks=masks, low_res_masks=low_res_logits) buf.seek(0) return buf.getvalue() - raise ValueError(f"Invalid format {request.format}") + else: + raise ValueError(f"Invalid format {request.format}") + self._attach_resolved_model_metadata(response) + return response raise ValueError(f"Invalid request type {type(request)}") def preproc_image(self, image: InferenceRequestImage): diff --git a/inference/models/trocr/trocr.py b/inference/models/trocr/trocr.py index feed2177c0..1e1edc574b 100644 --- a/inference/models/trocr/trocr.py +++ b/inference/models/trocr/trocr.py @@ -82,4 +82,5 @@ def infer_from_request( text = self.infer(**request.model_dump()) t2 = perf_counter() response = OCRInferenceResponse(result=text[0], time=t2 - t1) + self._attach_resolved_model_metadata(response) return response diff --git a/inference/models/trocr/trocr_inference_models.py b/inference/models/trocr/trocr_inference_models.py index 3513d88d9c..3f25ade77a 100644 --- a/inference/models/trocr/trocr_inference_models.py +++ b/inference/models/trocr/trocr_inference_models.py @@ -80,4 +80,5 @@ def infer_from_request( text = self.infer(**request.model_dump()) t2 = perf_counter() response = OCRInferenceResponse(result=text, time=t2 - t1) + self._attach_resolved_model_metadata(response) return response diff --git a/inference/models/yolo_world/yolo_world.py b/inference/models/yolo_world/yolo_world.py index 355cea21fe..076af83b24 100644 --- a/inference/models/yolo_world/yolo_world.py +++ b/inference/models/yolo_world/yolo_world.py @@ -82,7 +82,9 @@ def infer_from_request( Perform inference based on the details provided in the request, and return the associated responses. """ with self._state_lock: - return self.infer(**request.dict()) + response = self.infer(**request.dict()) + self._attach_resolved_model_metadata(response) + return response @usage_collector("model") def infer( diff --git a/inference_models/docs/changelog.md b/inference_models/docs/changelog.md index ceda421953..8465076610 100644 --- a/inference_models/docs/changelog.md +++ b/inference_models/docs/changelog.md @@ -2,13 +2,26 @@ ## Unreleased +--- + +## `0.37.2` + +### Added + +- `AutoModel.from_pretrained` exposes `resolved_model` metadata for the loaded + package: canonical model ID, package ID, backend, and quantization. The public + `ResolvedModelMetadata` entity describes this metadata. + +- `Cosmos3EdgeActionRecognition` wrapped over an already loaded reasoner + (`from_reasoner`) exposes that reasoner's `resolved_model`. + ### Fixed - `YOLONasForObjectDetectionTRT` concatenated TRT outputs on the default CUDA stream without ordering against the post-processing stream, which could yield phantom detections under GPU contention. Concatenation now runs on the inference stream and is synchronised before post-processing. --- -## `0.37.0` +## `0.37.1` ### Fixed diff --git a/inference_models/inference_models/entities.py b/inference_models/inference_models/entities.py index dd3278118f..6f2a86ead8 100644 --- a/inference_models/inference_models/entities.py +++ b/inference_models/inference_models/entities.py @@ -1,6 +1,15 @@ from collections import namedtuple +from dataclasses import dataclass from typing import Literal, Union ImageDimensions = namedtuple("ImageDimensions", ["height", "width"]) ColorFormat = Literal["rgb", "bgr"] Confidence = Union[float, Literal["best", "default"]] + + +@dataclass(frozen=True) +class ResolvedModelMetadata: + model_id: str + model_package_id: str + backend: str + quantization: str diff --git a/inference_models/inference_models/models/auto_loaders/core.py b/inference_models/inference_models/models/auto_loaders/core.py index ccc1ac105f..b8a7010920 100644 --- a/inference_models/inference_models/models/auto_loaders/core.py +++ b/inference_models/inference_models/models/auto_loaders/core.py @@ -11,7 +11,18 @@ from contextvars import ContextVar from datetime import datetime from functools import partial -from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + Generator, + List, + Optional, + Set, + Tuple, + Union, + cast, +) import torch from filelock import FileLock @@ -26,6 +37,7 @@ OFFLINE_MODE_WARM_UP, ROBOFLOW_API_KEY, ) +from inference_models.entities import ResolvedModelMetadata from inference_models.errors import ( CorruptedModelPackageError, DirectLocalStorageAccessError, @@ -1671,6 +1683,10 @@ def from_pretrained( - TextImageEmbeddingModel: For vision-language embeddings (CLIP, etc.) - OpenVocabularyObjectDetectionModel: For open-vocabulary detection + Models loaded from registered packages expose `resolved_model` with the + canonical model ID, package ID, backend, and package quantization. + Direct local-path loads do not infer a package ID. + Raises: UnauthorizedModelAccessError: If API key is invalid or model access is denied. ModelPackageNotFoundError: If no compatible model package is found for your @@ -2271,6 +2287,9 @@ def invalidate_cache_entry(reason: str) -> None: ) invalidate_cache_entry("cached requested-model identity does not match") return None + if cache_entry.backend_type is None: + invalidate_cache_entry("cached package backend is missing") + return None if not allow_untrusted_packages and cache_entry.trusted_source is not True: verbose_info( message=( @@ -2419,6 +2438,12 @@ def dependency_directory_pointer(model_dir: str) -> None: model_init_kwargs=model_init_kwargs, ), ) + model.resolved_model = ResolvedModelMetadata( + model_id=cast(str, cache_entry.canonical_model_id), + model_package_id=cache_entry.model_package_id, + backend=cache_entry.backend_type.value, + quantization=package_config.quantization or Quantization.UNKNOWN.value, + ) if point_model_directory: point_model_directory(model_package_cache_dir) verbose_info( @@ -3216,6 +3241,12 @@ def initialize_model( model_init_kwargs=model_init_kwargs, ), ) + model.resolved_model = ResolvedModelMetadata( + model_id=model_id, + model_package_id=model_package.package_id, + backend=model_package.backend.value, + quantization=quantization, + ) if OFFLINE_MODE: # Read-only leg: nothing was downloaded and the manifest is not # rewritten, so there is no mutation to guard. diff --git a/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py b/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py index 0cab71028c..56fd70bd94 100644 --- a/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py +++ b/inference_models/inference_models/models/cosmos3/cosmos3_action_recognition.py @@ -310,6 +310,10 @@ def __init__( video_sampling: Optional[VideoSampling] = None, ): self._reasoner = reasoner + # The auto-loader stamps the package on the object it returns. When that + # was the reasoner, carry it over; a loader that returns this wrapper + # overwrites the attribute after construction. + self.resolved_model = getattr(reasoner, "resolved_model", None) self._class_names = class_names self._video_sampling = video_sampling or VideoSampling() diff --git a/inference_models/pyproject.toml b/inference_models/pyproject.toml index 2be9e2a74c..590c60d6e2 100644 --- a/inference_models/pyproject.toml +++ b/inference_models/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "inference-models" -version = "0.37.1" +version = "0.37.2" description = "The new inference engine for Computer Vision models" readme = "README.md" requires-python = ">=3.10,<3.13" diff --git a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py new file mode 100644 index 0000000000..f44f3781f1 --- /dev/null +++ b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py @@ -0,0 +1,157 @@ +from dataclasses import asdict, replace +from types import SimpleNamespace + +import pytest +from packaging.version import Version + +from inference_models.entities import ResolvedModelMetadata +from inference_models.models.auto_loaders import ( + auto_negotiation, + auto_resolution_cache, + core, + model_cache_paths, +) +from inference_models.models.auto_loaders.entities import BackendType +from inference_models.weights_providers.entities import ( + ModelMetadata, + ModelPackageMetadata, + ONNXPackageDetails, + Quantization, +) + + +@pytest.fixture(autouse=True) +def available_onnx_backend(monkeypatch): + runtime = replace( + core.x_ray_runtime_environment(), + onnxruntime_version=Version("1.20.1"), + available_onnx_execution_providers={"CPUExecutionProvider"}, + ) + monkeypatch.setattr(core, "x_ray_runtime_environment", lambda: runtime) + monkeypatch.setattr(auto_negotiation, "x_ray_runtime_environment", lambda: runtime) + monkeypatch.setattr( + auto_negotiation, + "get_selected_onnx_execution_providers", + lambda: ["CPUExecutionProvider"], + ) + + +@pytest.mark.parametrize( + "missing_cache_field", [None, "backend_type", "canonical_model_id"] +) +def test_auto_model_reports_canonical_package_after_fresh_and_cached_loads( + tmp_path, monkeypatch, missing_cache_field +): + monkeypatch.setattr(model_cache_paths, "INFERENCE_HOME", str(tmp_path)) + monkeypatch.setattr(auto_resolution_cache, "INFERENCE_HOME", str(tmp_path)) + package = ModelPackageMetadata( + package_id="onnxpackage", + backend=BackendType.ONNX, + quantization=Quantization.FP32, + package_artefacts=[], + onnx_package_details=ONNXPackageDetails(opset=17), + trusted_source=True, + ) + metadata = ModelMetadata( + model_id="canonical/1", + model_architecture="yolov8", + task_type="object-detection", + model_packages=[package], + ) + monkeypatch.setattr(core, "get_model_from_provider", lambda **kwargs: metadata) + model_class = SimpleNamespace( + from_pretrained=lambda *args, **kwargs: SimpleNamespace() + ) + monkeypatch.setattr(core, "resolve_model_class", lambda **kwargs: model_class) + + first = core.AutoModel.from_pretrained("alias/1", backend="onnx", device="cpu") + + first_metadata = getattr(first, "resolved_model", None) + assert isinstance(first_metadata, ResolvedModelMetadata) + assert asdict(first_metadata) == { + "model_id": "canonical/1", + "model_package_id": "onnxpackage", + "backend": "onnx", + "quantization": "fp32", + } + + refreshed_metadata = [] + + def refresh_provider_metadata(**kwargs): + refreshed_metadata.append(kwargs) + return metadata + + monkeypatch.setattr(core, "get_model_from_provider", refresh_provider_metadata) + + class PartialMetadataCache(auto_resolution_cache.BaseAutoLoadMetadataCache): + def retrieve(self, auto_negotiation_hash): + entry = super().retrieve(auto_negotiation_hash) + if entry is not None and missing_cache_field is not None: + return entry.model_copy(update={missing_cache_field: None}) + return entry + + second = core.AutoModel.from_pretrained( + "alias/1", + backend="onnx", + device="cpu", + auto_resolution_cache=PartialMetadataCache(file_lock_acquire_timeout=1), + ) + assert second is not first + second_metadata = getattr(second, "resolved_model", None) + assert second_metadata == first_metadata + assert len(refreshed_metadata) == (0 if missing_cache_field is None else 1) + + third = core.AutoModel.from_pretrained("alias/1", backend="onnx", device="cpu") + assert getattr(third, "resolved_model", None) == first_metadata + assert len(refreshed_metadata) == (0 if missing_cache_field is None else 1) + + +def test_auto_model_reports_the_successful_fallback_package(tmp_path, monkeypatch): + monkeypatch.setattr(model_cache_paths, "INFERENCE_HOME", str(tmp_path)) + monkeypatch.setattr(auto_resolution_cache, "INFERENCE_HOME", str(tmp_path)) + packages = [ + ModelPackageMetadata( + package_id=package_id, + backend=BackendType.ONNX, + quantization=quantization, + package_artefacts=[], + onnx_package_details=ONNXPackageDetails(opset=17), + trusted_source=True, + ) + for package_id, quantization in [ + ("brokenfp16", Quantization.FP16), + ("workingfp32", Quantization.FP32), + ] + ] + metadata = ModelMetadata( + model_id="canonical/1", + model_architecture="yolov8", + task_type="object-detection", + model_packages=packages, + ) + monkeypatch.setattr(core, "get_model_from_provider", lambda **kwargs: metadata) + attempted_packages = [] + + def load_package(package_path, **kwargs): + package_id = package_path.rsplit("/", 1)[-1] + attempted_packages.append(package_id) + if package_id == "brokenfp16": + raise RuntimeError("Unsupported package") + return SimpleNamespace() + + model_class = SimpleNamespace(from_pretrained=load_package) + monkeypatch.setattr(core, "resolve_model_class", lambda **kwargs: model_class) + + model = core.AutoModel.from_pretrained( + "alias/1", backend="onnx", quantization=["fp16", "fp32"], device="cpu" + ) + + assert attempted_packages == ["brokenfp16", "workingfp32"] + resolved_model = getattr(model, "resolved_model", None) + assert isinstance(resolved_model, ResolvedModelMetadata) + assert asdict(resolved_model) == { + "model_id": "canonical/1", + "model_package_id": "workingfp32", + "backend": "onnx", + "quantization": "fp32", + } diff --git a/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py b/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py index 28496f9f74..38dcf8736e 100644 --- a/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py +++ b/inference_models/tests/unit_tests/models/test_cosmos3_action_recognition.py @@ -5,6 +5,7 @@ import pytest import torch +from inference_models.entities import ResolvedModelMetadata from inference_models.errors import CorruptedModelPackageError from inference_models.models.base.action_recognition import ( SLIDING_WINDOW_MODE, @@ -377,6 +378,79 @@ def test_from_pretrained_reads_the_recorded_sampling(monkeypatch, tmp_path) -> N assert all(frame.shape == (56, 100, 3) for frame in sent_frames) +_RESOLVED_MODEL = ResolvedModelMetadata( + model_id="workspace/actions/1", + model_package_id="hf-package", + backend="hugging-face", + quantization="bf16", +) + + +def _loaded_reasoner(package_dir=None, class_names=(), resolved_model=_RESOLVED_MODEL): + """A reasoner as the auto-loader hands it over: stamped after loading.""" + reasoner = _FakeReasoner(tokenizer=_FakeTokenizer(class_names=class_names)) + reasoner.package_dir = package_dir + if resolved_model is not None: + reasoner.resolved_model = resolved_model + return reasoner + + +def test_from_reasoner_over_a_hosted_base_exposes_the_reasoner_package() -> None: + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=_loaded_reasoner()) + + assert wrapper.class_names is None + assert wrapper.resolved_model is _RESOLVED_MODEL + + +def test_from_reasoner_over_a_fine_tune_exposes_the_reasoner_package(tmp_path) -> None: + (tmp_path / "class_names.txt").write_text("walking\nrunning\n") + reasoner = _loaded_reasoner( + package_dir=str(tmp_path), class_names=["walking", "running"] + ) + + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=reasoner) + + assert wrapper.class_names == ["walking", "running"] + assert wrapper.resolved_model is _RESOLVED_MODEL + + +def test_from_reasoner_over_a_base_package_exposes_the_reasoner_package( + tmp_path, +) -> None: + reasoner = _loaded_reasoner(package_dir=str(tmp_path)) + + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=reasoner) + + assert wrapper.class_names is None + assert wrapper.resolved_model is _RESOLVED_MODEL + + +def test_a_reasoner_loaded_outside_the_auto_loader_resolves_to_nothing() -> None: + reasoner = _loaded_reasoner(resolved_model=None) + + from_reasoner = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=reasoner) + constructed = Cosmos3EdgeActionRecognition(reasoner=reasoner) + + assert not hasattr(reasoner, "resolved_model") + assert from_reasoner.resolved_model is None + assert constructed.resolved_model is None + + +def test_the_package_the_loader_stamps_on_the_wrapper_wins() -> None: + wrapper = Cosmos3EdgeActionRecognition.from_reasoner(reasoner=_loaded_reasoner()) + stamped_by_loader = ResolvedModelMetadata( + model_id="workspace/actions/2", + model_package_id="trt-package", + backend="trt", + quantization="fp16", + ) + + wrapper.resolved_model = stamped_by_loader + + assert wrapper.resolved_model is stamped_by_loader + assert wrapper._reasoner.resolved_model is _RESOLVED_MODEL + + def test_fine_tune_parser_accepts_trailing_end_token(monkeypatch) -> None: tokenizer = _FakeTokenizer(class_names=["walking", "running"]) reasoner = _FakeReasoner( diff --git a/inference_models/uv.lock b/inference_models/uv.lock index c63c2eaab1..8382f7b0cb 100644 --- a/inference_models/uv.lock +++ b/inference_models/uv.lock @@ -1044,7 +1044,7 @@ wheels = [ [[package]] name = "inference-models" -version = "0.37.1" +version = "0.37.2" source = { virtual = "." } dependencies = [ { name = "accelerate" }, diff --git a/requirements/requirements.cpu.txt b/requirements/requirements.cpu.txt index 516a75eb36..ba385628ee 100644 --- a/requirements/requirements.cpu.txt +++ b/requirements/requirements.cpu.txt @@ -1,3 +1,3 @@ onnxruntime>=1.15.1,<1.22.0 nvidia-ml-py<13.0.0 -inference-models[torch-cpu,onnx-cpu]~=0.37.1 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu,onnx-cpu]~=0.37.2 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.gpu.cu13.txt b/requirements/requirements.gpu.cu13.txt index 8a72a5e5b7..aa8a46d9a0 100644 --- a/requirements/requirements.gpu.cu13.txt +++ b/requirements/requirements.gpu.cu13.txt @@ -4,6 +4,6 @@ # is a CUDA 12 build), so the Dockerfile installs it in a dedicated step from # Microsoft's onnxruntime-cuda-13 feed. A plain pin here would silently # resolve to the CUDA 12 wheel. -inference-models[torch-cu130]~=0.37.1 +inference-models[torch-cu130]~=0.37.2 pynvvideocodec~=2.1.0; platform_machine == "x86_64" onnxruntime-gpu>=1.27.0,<1.30.0 \ No newline at end of file diff --git a/requirements/requirements.gpu.txt b/requirements/requirements.gpu.txt index bbfc239e21..ff4e238c89 100644 --- a/requirements/requirements.gpu.txt +++ b/requirements/requirements.gpu.txt @@ -1,3 +1,3 @@ onnxruntime-gpu>=1.15.1,<1.22.0 -inference-models[torch-cu124,onnx-cu12]~=0.37.1 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cu124,onnx-cu12]~=0.37.2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt pynvvideocodec~=2.1.0; platform_machine == "x86_64" diff --git a/requirements/requirements.jetson.txt b/requirements/requirements.jetson.txt index 1d4ba27850..1f7ce72a93 100644 --- a/requirements/requirements.jetson.txt +++ b/requirements/requirements.jetson.txt @@ -1,3 +1,3 @@ pypdfium2>=4.11.0,<5.0.0 PyYAML~=6.0.0 -inference-models~=0.37.1 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models~=0.37.2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.vino.txt b/requirements/requirements.vino.txt index ff26f4e7e9..92a4cb311d 100644 --- a/requirements/requirements.vino.txt +++ b/requirements/requirements.vino.txt @@ -1,2 +1,2 @@ onnxruntime-openvino>=1.15.0,<1.22.0 -inference-models[torch-cpu]~=0.37.1 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu]~=0.37.2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py new file mode 100644 index 0000000000..4703bf07ba --- /dev/null +++ b/tests/inference/unit_tests/core/interfaces/http/test_resolved_model_metadata.py @@ -0,0 +1,542 @@ +import runpy +from types import SimpleNamespace +from typing import Optional +from unittest.mock import AsyncMock + +import numpy as np +import pytest +from starlette.testclient import TestClient + +from inference.core import env +from inference.core.entities.responses.inference import ( + ClassificationInferenceResponse, + InferenceResponseImage, + LMMInferenceResponse, + MultiLabelClassificationInferenceResponse, + ObjectDetectionInferenceResponse, +) +from inference.core.interfaces.http import http_api +from inference.core.managers import base as manager_module +from inference.core.managers.base import ModelManager +from inference.core.models import base as model_module +from inference.core.models import inference_models_adapters as adapters +from inference.core.registries import roboflow as registry_module +from inference.core.registries.base import ModelRegistry +from inference.models import utils as model_utils +from inference.models.dinov3.dinov3_classification import DinoV3Classification +from inference.models.resnet.resnet_classification import ResNetClassification +from inference.models.vit.vit_classification import VitClassification +from inference.models.yolov8.yolov8_classification import YOLOv8Classification +from inference.models.yolov8.yolov8_object_detection import YOLOv8ObjectDetection +from inference_models.utils.content_addressed_artifact_cache import ( + NullContentAddressedArtifactCache, +) +from inference_sdk import InferenceHTTPClient + +RESOLVED_MODEL = SimpleNamespace( + model_id="canonical/1", + model_package_id="onnxpackage", + backend="onnx", + quantization="fp32", +) + + +class PackageModel(model_module.Model): + task_type = "object-detection" + model_id = "test/1" + + def __init__(self, metadata: Optional[SimpleNamespace] = RESOLVED_MODEL): + self._model = SimpleNamespace() + if metadata is not None: + self._model.resolved_model = metadata + + def infer(self, image, **kwargs): + response = ObjectDetectionInferenceResponse( + image=InferenceResponseImage(width=640, height=480), predictions=[] + ) + images = image if isinstance(image, list) else [image] + return [response.model_copy() for _ in images] + + +def build_client(monkeypatch, flag=True, model=None, registry=None, preload=True): + monkeypatch.setattr(model_module, "USE_INFERENCE_MODELS", flag) + monkeypatch.setattr(manager_module, "USE_INFERENCE_MODELS", flag) + monkeypatch.setattr(manager_module, "MODELS_CACHE_AUTH_ENABLED", False) + monkeypatch.setattr(manager_module, "DISABLE_INFERENCE_CACHE", True) + monkeypatch.setattr(http_api, "USE_INFERENCE_MODELS", flag) + monkeypatch.setattr(http_api, "MAX_INFERENCE_MODELS_CACHE_SIZE_MB", 0) + monkeypatch.setattr(http_api, "GCP_SERVERLESS", False) + monkeypatch.setattr(http_api, "LAMBDA", False) + monkeypatch.setattr( + http_api, + "InferenceInstrumentator", + lambda *args, **kwargs: SimpleNamespace( + set_stream_manager_client=lambda client: None + ), + ) + monkeypatch.setattr( + http_api.usage_collector, "async_push_usage_payloads", AsyncMock() + ) + manager = ModelManager( + model_registry=registry or ModelRegistry({}), + models={"test/1": model or PackageModel()} if preload else {}, + content_addressed_artifact_cache=NullContentAddressedArtifactCache(), + ) + return TestClient(http_api.HttpInterface(model_manager=manager).app), manager + + +def test_http_inference_reports_the_serving_package(monkeypatch): + client, _ = build_client(monkeypatch) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["resolved_model"] == { + "model_id": "canonical/1", + "model_package_id": "onnxpackage", + "backend": "onnx", + "quantization": "fp32", + } + + +def test_legacy_http_inference_reports_the_serving_package(monkeypatch): + client, _ = build_client(monkeypatch) + + response = client.post( + "/test/1", + content="image", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["resolved_model"]["model_package_id"] == "onnxpackage" + + +def test_http_batch_reports_package_for_each_image(monkeypatch): + client, _ = build_client(monkeypatch) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": [ + {"type": "base64", "value": "first"}, + {"type": "base64", "value": "second"}, + ], + }, + ) + + assert response.status_code == 200, response.text + assert [item["resolved_model"]["model_package_id"] for item in response.json()] == [ + "onnxpackage", + "onnxpackage", + ] + + +def test_http_inference_reports_model_id_when_flag_is_disabled(monkeypatch): + monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) + model_types = runpy.run_path(model_utils.__file__)["ROBOFLOW_MODEL_TYPES"] + monkeypatch.setattr( + registry_module, + "get_model_type", + lambda *args, **kwargs: ("object-detection", "yolov8"), + ) + + def initialize_legacy_model(self, model_id, **kwargs): + self.endpoint = model_id + + monkeypatch.setattr(YOLOv8ObjectDetection, "__init__", initialize_legacy_model) + monkeypatch.setattr(YOLOv8ObjectDetection, "infer", PackageModel.infer) + client, manager = build_client( + monkeypatch, + flag=False, + registry=registry_module.RoboflowModelRegistry(model_types), + preload=False, + ) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert isinstance(manager.models()["test/1"], YOLOv8ObjectDetection) + payload = response.json() + assert payload.pop("time") >= 0 + payload.pop("inference_id") + assert payload == { + "image": {"width": 640, "height": 480}, + "predictions": [], + "resolved_model": {"model_id": "test/1"}, + } + + +def _legacy_classification_infer(multi_label): + def infer(self, image, **kwargs): + image_metadata = InferenceResponseImage(width=640, height=480) + if multi_label: + response = MultiLabelClassificationInferenceResponse( + image=image_metadata, + predictions={"cat": {"confidence": 0.9, "class_id": 0}}, + predicted_classes=["cat"], + ) + else: + response = ClassificationInferenceResponse( + image=image_metadata, + predictions=[{"class": "cat", "class_id": 0, "confidence": 0.9}], + top="cat", + confidence=0.9, + ) + images = image if isinstance(image, list) else [image] + return [response.model_copy() for _ in images] + + return infer + + +def _build_legacy_classification_client( + monkeypatch, model_type, model_class, multi_label +): + monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) + model_types = runpy.run_path(model_utils.__file__)["ROBOFLOW_MODEL_TYPES"] + monkeypatch.setattr( + registry_module, + "get_model_type", + lambda *args, **kwargs: ("classification", model_type), + ) + + def initialize_legacy_model(self, model_id, **kwargs): + self.endpoint = model_id + + monkeypatch.setattr(model_class, "__init__", initialize_legacy_model) + monkeypatch.setattr(model_class, "infer", _legacy_classification_infer(multi_label)) + return build_client( + monkeypatch, + flag=False, + registry=registry_module.RoboflowModelRegistry(model_types), + preload=False, + ) + + +@pytest.mark.parametrize("multi_label", [False, True], ids=["single", "multi"]) +@pytest.mark.parametrize( + "model_type,model_class", + [ + ("yolov8", YOLOv8Classification), + ("resnet18", ResNetClassification), + ("vit", VitClassification), + ("dinov3", DinoV3Classification), + ], +) +def test_http_classification_reports_model_id_when_flag_is_disabled( + monkeypatch, model_type, model_class, multi_label +): + client, manager = _build_legacy_classification_client( + monkeypatch, model_type, model_class, multi_label + ) + + response = client.post( + "/infer/classification", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert type(manager.models()["test/1"]) is model_class + payload = response.json() + assert payload["predicted_classes" if multi_label else "top"] + assert payload["resolved_model"] == {"model_id": "test/1"} + + +def test_http_classification_batch_reports_model_id_when_flag_is_disabled( + monkeypatch, +): + client, _ = _build_legacy_classification_client( + monkeypatch, "yolov8", YOLOv8Classification, multi_label=False + ) + + response = client.post( + "/infer/classification", + json={ + "model_id": "test/1", + "image": [ + {"type": "base64", "value": "first"}, + {"type": "base64", "value": "second"}, + ], + }, + ) + + assert response.status_code == 200, response.text + assert [item["resolved_model"] for item in response.json()] == [ + {"model_id": "test/1"}, + {"model_id": "test/1"}, + ] + + +def test_legacy_route_classification_reports_model_id_when_flag_is_disabled( + monkeypatch, +): + client, _ = _build_legacy_classification_client( + monkeypatch, "yolov8", YOLOv8Classification, multi_label=False + ) + + response = client.post( + "/test/1", + content="image", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["resolved_model"] == {"model_id": "test/1"} + + +@pytest.mark.parametrize("api_version", ["v0", "v1"]) +def test_sdk_preserves_http_package_metadata(monkeypatch, requests_mock, api_version): + server, _ = build_client(monkeypatch) + sdk = InferenceHTTPClient(api_url="http://testserver", api_key="test") + getattr(sdk, f"select_api_{api_version}")() + + def infer(request, context): + response = server.post( + request.path_url, + content=request.body, + headers={"Content-Type": request.headers["Content-Type"]}, + ) + context.status_code = response.status_code + return response.json() + + requests_mock.get( + "http://testserver/model/registry", + json=server.get("/model/registry").json(), + ) + endpoint = "/test/1" if api_version == "v0" else "/infer/object_detection" + requests_mock.post(f"http://testserver{endpoint}", json=infer) + + response = sdk.infer(np.zeros((480, 640, 3), dtype=np.uint8), model_id="test/1") + + assert isinstance(response, dict) + assert response["resolved_model"] == { + "model_id": "canonical/1", + "model_package_id": "onnxpackage", + "backend": "onnx", + "quantization": "fp32", + } + + +def test_http_response_keeps_serving_package_when_model_is_replaced(monkeypatch): + replacement = PackageModel( + SimpleNamespace( + model_id="canonical/1", + model_package_id="trtpackage", + backend="trt", + quantization="fp16", + ) + ) + registry = SimpleNamespace( + get_model=lambda *args, **kwargs: lambda **kwargs: replacement + ) + + class EvictedModel(PackageModel): + def infer_from_request(self, request): + manager.remove("test/1", delete_from_disk=False) + manager.add_model("test/1", api_key="test") + return super().infer_from_request(request) + + client, manager = build_client(monkeypatch, model=EvictedModel(), registry=registry) + request = { + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + } + + first = client.post("/infer/object_detection", json=request) + second = client.post("/infer/object_detection", json=request) + + assert first.status_code == 200, first.text + assert second.status_code == 200, second.text + assert first.json()["resolved_model"]["model_package_id"] == "onnxpackage" + assert second.json()["resolved_model"]["model_package_id"] == "trtpackage" + + +def test_http_inference_omits_metadata_for_a_model_without_package_identity( + monkeypatch, +): + client, _ = build_client(monkeypatch, model=PackageModel(metadata=None)) + + response = client.post( + "/infer/object_detection", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + }, + ) + + assert response.status_code == 200, response.text + assert "resolved_model" not in response.json() + + +def test_direct_model_response_reports_model_id_with_flag_disabled(monkeypatch): + from inference.core.entities.requests.inference import ( + InferenceRequestImage, + ObjectDetectionInferenceRequest, + ) + + monkeypatch.setattr(env, "USE_INFERENCE_MODELS", False) + monkeypatch.setattr(model_module, "USE_INFERENCE_MODELS", False) + response = PackageModel().infer_from_request( + ObjectDetectionInferenceRequest( + id="direct-request", + model_id="test/1", + image=InferenceRequestImage(type="base64", value="image"), + ) + ) + + assert isinstance(response, ObjectDetectionInferenceResponse) + assert response.resolved_model is not None + assert response.resolved_model.model_dump() == { + "model_id": "test/1", + "model_package_id": None, + "backend": None, + "quantization": None, + } + + +def test_classification_adapter_reports_package_with_predictions(monkeypatch): + monkeypatch.setattr(model_module, "USE_INFERENCE_MODELS", True) + import base64 + + import cv2 + import torch + + from inference.core.entities.requests.inference import ( + ClassificationInferenceRequest, + InferenceRequestImage, + ) + from inference.core.entities.responses.inference import ( + ClassificationInferenceResponse, + ) + from inference_models import ClassificationPrediction + + class ClassificationBackend: + class_names = ["cat", "dog"] + resolved_model = RESOLVED_MODEL + + def pre_process(self, images, **kwargs): + return np.stack(images) + + def forward(self, images, **kwargs): + return torch.tensor([[0.9, 0.1]] * len(images)) + + def post_process(self, predictions, **kwargs): + return ClassificationPrediction( + class_id=predictions.argmax(dim=-1), confidence=predictions + ) + + monkeypatch.setattr( + adapters.AutoModel, "from_pretrained", lambda **kwargs: ClassificationBackend() + ) + model = adapters.InferenceModelsClassificationAdapter("test/1", api_key="test") + _, image = cv2.imencode(".png", np.zeros((4, 5, 3), dtype=np.uint8)) + + response = model.infer_from_request( + ClassificationInferenceRequest( + id="classification-request", + model_id="test/1", + image=InferenceRequestImage( + type="base64", value=base64.b64encode(image).decode() + ), + ) + ) + + assert isinstance(response, ClassificationInferenceResponse) + assert response.top == "cat" + assert response.resolved_model is not None + assert response.resolved_model.model_package_id == "onnxpackage" + + +def test_lmm_response_reports_package_without_task_specific_adapter(monkeypatch): + class LMMModel(PackageModel): + task_type = "lmm" + + def infer(self, **kwargs): + return [ + LMMInferenceResponse( + image=InferenceResponseImage(width=640, height=480), + response="A cat.", + ) + ] + + client, _ = build_client(monkeypatch, model=LMMModel()) + response = client.post( + "/infer/lmm", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + "prompt": "Describe the image.", + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["response"] == "A cat." + assert response.json()["resolved_model"] == vars(RESOLVED_MODEL) + + +@pytest.mark.parametrize("depth_map_format", ["json", "png8", "png16"]) +def test_depth_response_preserves_package_metadata(monkeypatch, depth_map_format): + class DepthModel(PackageModel): + task_type = "depth-estimation" + + def infer(self, **kwargs): + return [ + LMMInferenceResponse( + image=InferenceResponseImage(width=2, height=2), + response={ + "normalized_depth": np.array([[0.0, 0.25], [0.5, 1.0]]), + "image": SimpleNamespace(base64_image="depth-image"), + }, + ) + ] + + monkeypatch.setattr(http_api, "DEPTH_ESTIMATION_ENABLED", True) + client, _ = build_client(monkeypatch, model=DepthModel()) + response = client.post( + "/infer/depth-estimation", + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + "depth_map_format": depth_map_format, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["depth_map_format"] == depth_map_format + assert response.json()["resolved_model"] == vars(RESOLVED_MODEL) + + +def test_openapi_documents_metadata_for_all_inference_response_types(monkeypatch): + monkeypatch.setattr(http_api, "DEPTH_ESTIMATION_ENABLED", True) + monkeypatch.setattr(http_api, "ACTION_RECOGNITION_ENABLED", True) + client, _ = build_client(monkeypatch) + schemas = client.get("/openapi.json").json()["components"]["schemas"] + + for name in ( + "ObjectDetectionInferenceResponse", + "LMMInferenceResponse", + "DepthEstimationResponse", + "ActionRecognitionInferenceResponse", + ): + metadata = schemas[name]["properties"]["resolved_model"] + assert {"$ref": "#/components/schemas/ResolvedModel"} in metadata["anyOf"] + assert ( + "canonical ID" + in schemas["ResolvedModel"]["properties"]["model_id"]["description"] + ) diff --git a/tests/inference/unit_tests/core/interfaces/http/test_response_null_compatibility.py b/tests/inference/unit_tests/core/interfaces/http/test_response_null_compatibility.py new file mode 100644 index 0000000000..6fa19196d7 --- /dev/null +++ b/tests/inference/unit_tests/core/interfaces/http/test_response_null_compatibility.py @@ -0,0 +1,119 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import numpy as np +import pytest +from starlette.testclient import TestClient + +from inference.core.entities.responses.clip import ClipCompareResponse +from inference.core.entities.responses.perception_encoder import ( + PerceptionEncoderCompareResponse, +) +from inference.core.entities.responses.sam3 import ( + Sam3PromptEcho, + Sam3PromptResult, + Sam3SegmentationResponse, +) +from inference.core.interfaces.http import http_api + + +@pytest.fixture +def client_and_manager(monkeypatch): + for name in ( + "CORE_MODELS_ENABLED", + "CORE_MODEL_CLIP_ENABLED", + "CORE_MODEL_PE_ENABLED", + "CORE_MODEL_SAM3_ENABLED", + "DEPTH_ESTIMATION_ENABLED", + ): + monkeypatch.setattr(http_api, name, True) + monkeypatch.setattr(http_api, "LAMBDA", False) + monkeypatch.setattr(http_api, "GCP_SERVERLESS", False) + monkeypatch.setattr(http_api, "DEDICATED_DEPLOYMENT_WORKSPACE_URL", None) + monkeypatch.setattr(http_api, "InferenceInstrumentator", MagicMock()) + monkeypatch.setattr( + http_api.usage_collector, "async_push_usage_payloads", AsyncMock() + ) + manager = MagicMock() + manager.pingback = None + return TestClient(http_api.HttpInterface(model_manager=manager).app), manager + + +@pytest.mark.parametrize( + "endpoint,response_type", + [ + ("/clip/compare", ClipCompareResponse), + ("/perception_encoder/compare", PerceptionEncoderCompareResponse), + ], +) +def test_embedding_comparison_keeps_existing_null_fields( + client_and_manager, endpoint, response_type +): + client, manager = client_and_manager + manager.infer_from_request_sync.return_value = response_type(similarity=[0.5]) + + response = client.post( + endpoint, + json={"subject": "cat", "subject_type": "text", "prompt": "animal"}, + ) + + assert response.status_code == 200, response.text + payload = response.json() + for field in ("parent_id", "time", "frame_id", "inference_id"): + assert field in payload + assert payload[field] is None + + +def test_sam3_keeps_null_prompt_echo_fields(client_and_manager): + client, manager = client_and_manager + manager.infer_from_request_sync.return_value = Sam3SegmentationResponse( + prompt_results=[ + Sam3PromptResult( + prompt_index=0, + echo=Sam3PromptEcho(prompt_index=0, type="text", text="cat"), + predictions=[], + ) + ], + time=0.1, + ) + + response = client.post( + "/sam3/concept_segment", + json={ + "image": {"type": "base64", "value": "image"}, + "prompts": [{"type": "text", "text": "cat"}], + }, + ) + + assert response.status_code == 200, response.text + echo = response.json()["prompt_results"][0]["echo"] + assert "num_boxes" in echo + assert echo["num_boxes"] is None + + +@pytest.mark.parametrize( + "endpoint", + ["/infer/depth-estimation", "/infer/depth-estimation/test/1"], +) +def test_depth_estimation_keeps_null_image(client_and_manager, endpoint): + client, manager = client_and_manager + manager.infer_from_request_sync.return_value = SimpleNamespace( + response={ + "normalized_depth": np.zeros((1, 1)), + "image": SimpleNamespace(base64_image=None), + } + ) + + response = client.post( + endpoint, + json={ + "model_id": "test/1", + "image": {"type": "base64", "value": "image"}, + "depth_map_format": "json", + }, + ) + + assert response.status_code == 200, response.text + payload = response.json() + assert "image" in payload + assert payload["image"] is None diff --git a/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py b/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py index 716ee08e29..63b7da74c0 100644 --- a/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py +++ b/tests/inference/unit_tests/core/models/test_action_recognition_adapter.py @@ -1,4 +1,5 @@ import contextlib +from types import SimpleNamespace from typing import List, Optional from unittest.mock import MagicMock, patch @@ -18,6 +19,8 @@ class _FakeModel: + resolved_model: Optional[SimpleNamespace] = None + def __init__(self, responses, class_names=None, sampling=None): self.responses = list(responses) self._class_names = class_names @@ -94,6 +97,65 @@ def test_window_segments_map_to_clip_frame_indices() -> None: assert response.timeline[0].class_id == 0 +def test_clip_response_reports_the_loaded_package(monkeypatch) -> None: + from inference.core.models import base + + monkeypatch.setattr(base, "USE_INFERENCE_MODELS", True) + model = _FakeModel(responses=[[]], class_names=["walk"]) + model.resolved_model = SimpleNamespace( + model_id="workspace/model", + model_package_id="video-package", + backend="torch", + quantization="fp16", + ) + with _clip(frame_count=3, source_fps=10.0): + response = _adapter(model).infer_from_request(_request()) + + assert response.model_dump()["resolved_model"] == vars(model.resolved_model) + + +def test_clip_response_reports_the_package_of_a_wrapped_reasoner(monkeypatch) -> None: + from inference.core.models import base + from inference.core.models import inference_models_adapters as adapters + from inference_models.entities import ResolvedModelMetadata + from inference_models.models.cosmos3.cosmos3_action_recognition import ( + Cosmos3EdgeActionRecognition, + ) + from inference_models.models.cosmos3.cosmos3_reasoner_hf import Cosmos3EdgeReasoner + + monkeypatch.setattr(base, "USE_INFERENCE_MODELS", True) + # The registry hands a fine-tune over as a bare reasoner, and the + # auto-loader stamps the package on that reasoner, not on the wrapper. + reasoner = Cosmos3EdgeReasoner.__new__(Cosmos3EdgeReasoner) + reasoner._processor = SimpleNamespace(tokenizer=None) + reasoner.package_dir = None + reasoner.prompt_video = MagicMock(return_value="") + reasoner.resolved_model = ResolvedModelMetadata( + model_id="workspace/model", + model_package_id="video-package", + backend="hugging-face", + quantization="bf16", + ) + + with patch.object(adapters, "AutoModel") as auto_model: + auto_model.from_pretrained.return_value = reasoner + adapter = adapters.InferenceModelsActionRecognitionAdapter( + model_id="workspace/model", api_key="key" + ) + with _clip(frame_count=3, source_fps=10.0): + response = adapter.infer_from_request(_request()) + + assert isinstance(adapter._model, Cosmos3EdgeActionRecognition) + assert adapter._model._reasoner is reasoner + assert reasoner.prompt_video.call_count == 1 + assert response.model_dump()["resolved_model"] == { + "model_id": "workspace/model", + "model_package_id": "video-package", + "backend": "hugging-face", + "quantization": "bf16", + } + + def test_ranges_of_one_class_merge_across_windows() -> None: model = _FakeModel( responses=[ diff --git a/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py new file mode 100644 index 0000000000..4eed21cfca --- /dev/null +++ b/tests/inference/unit_tests/models/test_legacy_resolved_model_metadata.py @@ -0,0 +1,224 @@ +import importlib +import sys +from pathlib import Path +from threading import Lock +from types import ModuleType +from unittest.mock import Mock + +import numpy as np +import pytest + +import inference.models +from inference.core.entities.requests.clip import ClipTextEmbeddingRequest +from inference.core.entities.requests.doctr import DoctrOCRInferenceRequest +from inference.core.entities.requests.easy_ocr import EasyOCRInferenceRequest +from inference.core.entities.requests.inference import ( + ClassificationInferenceRequest, + InferenceRequestImage, +) +from inference.core.entities.requests.sam import ( + SamEmbeddingRequest, + SamSegmentationRequest, +) +from inference.core.entities.responses.inference import ( + ClassificationInferenceResponse, + MultiLabelClassificationInferenceResponse, +) +from inference.core.entities.responses.ocr import OCRInferenceResponse +from inference.core.models.types import PreprocessReturnMetadata + +IMAGE = InferenceRequestImage(type="base64", value="unused") +MODEL_ID = "foundation/model" +METADATA = { + "model_id": MODEL_ID, + "model_package_id": None, + "backend": None, + "quantization": None, +} + + +@pytest.fixture( + params=[False, True], ids=["legacy-default", "inference-models-default"] +) +def load_legacy_model(monkeypatch, request): + base_module = importlib.import_module("inference.core.models.base") + monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", request.param) + imported_modules = [] + + def load(module_name, class_name): + package_name = module_name.split(".")[0] + package = ModuleType(f"inference.models.{package_name}") + package.__path__ = [str(Path(inference.models.__file__).parent / package_name)] + monkeypatch.setitem(sys.modules, package.__name__, package) + module_name = f"inference.models.{module_name}" + monkeypatch.delitem(sys.modules, module_name, raising=False) + module = importlib.import_module(module_name) + imported_modules.append(module_name) + model_class = getattr(module, class_name) + model = model_class.__new__(model_class) + model.endpoint = MODEL_ID + return model + + yield load + for module_name in imported_modules: + sys.modules.pop(module_name, None) + + +def test_legacy_clip_request_metadata(load_legacy_model, monkeypatch): + monkeypatch.setitem(sys.modules, "clip", ModuleType("clip")) + model = load_legacy_model("clip.clip_model", "Clip") + model.embed_text = Mock(return_value=np.array([[0.1, 0.2]])) + + response = model.infer_from_request( + ClipTextEmbeddingRequest(id="test-request", model_id=None, text="cat") + ) + + assert response.embeddings == [[0.1, 0.2]] + assert response.model_dump()["resolved_model"] == METADATA + assert response.time >= 0 + + +@pytest.mark.parametrize( + "module_name,class_name,request_class,batched", + [ + ("doctr.doctr_model", "DocTR", DoctrOCRInferenceRequest, False), + ("easy_ocr.easy_ocr", "EasyOCR", EasyOCRInferenceRequest, False), + ("easy_ocr.easy_ocr", "EasyOCR", EasyOCRInferenceRequest, True), + ], +) +def test_legacy_ocr_request_metadata( + load_legacy_model, monkeypatch, module_name, class_name, request_class, batched +): + for module_name_to_stub, attributes in [ + ("doctr.io", ["DocumentFile"]), + ("doctr.models", ["crnn_vgg16_bn", "db_resnet50", "ocr_predictor"]), + ("easyocr", []), + ]: + module = ModuleType(module_name_to_stub) + for attribute in attributes: + setattr(module, attribute, Mock()) + monkeypatch.setitem(sys.modules, module_name_to_stub, module) + model = load_legacy_model(module_name, class_name) + model.single_request = Mock( + side_effect=lambda request: OCRInferenceResponse(result="text", time=0.1) + ) + + response = model.infer_from_request( + request_class(image=[IMAGE, IMAGE] if batched else IMAGE) + ) + + responses = response if isinstance(response, list) else [response] + assert len(responses) == (2 if batched else 1) + for result in responses: + assert result.result == "text" + assert result.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.parametrize("batched", [False, True], ids=["single-image", "image-list"]) +@pytest.mark.parametrize( + "multiclass,response_class", + [ + (False, ClassificationInferenceResponse), + (True, MultiLabelClassificationInferenceResponse), + ], + ids=["single-label", "multi-label"], +) +@pytest.mark.parametrize( + "module_name,class_name", + [ + ("yolov8.yolov8_classification", "YOLOv8Classification"), + ("resnet.resnet_classification", "ResNetClassification"), + ("vit.vit_classification", "VitClassification"), + ("dinov3.dinov3_classification", "DinoV3Classification"), + ], +) +def test_legacy_classification_request_metadata( + load_legacy_model, module_name, class_name, multiclass, response_class, batched +): + batch_size = 2 if batched else 1 + model = load_legacy_model(module_name, class_name) + model.multiclass = multiclass + model.class_names = ["cat", "dog"] + model.batching_enabled = False + model.batch_size = 1 + + def preprocess(image, **kwargs): + images_count = len(image) if isinstance(image, list) else 1 + return ( + np.zeros((images_count, 3, 2, 2), dtype=np.float32), + PreprocessReturnMetadata({"img_dims": [(2, 2)] * images_count}), + ) + + model.preprocess = Mock(side_effect=preprocess) + model.predict = Mock( + side_effect=lambda img_in, **kwargs: ( + np.array([[[0.9, 0.1]]] * img_in.shape[0], dtype=np.float32), + ) + ) + + response = model.infer_from_request( + ClassificationInferenceRequest( + id="test-request", + model_id=MODEL_ID, + image=[IMAGE, IMAGE] if batched else IMAGE, + ) + ) + + assert isinstance(response, list) == batched + responses = response if batched else [response] + assert len(responses) == batch_size + for result in responses: + assert isinstance(result, response_class) + assert result.resolved_model.model_dump() == METADATA + assert result.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.parametrize("response_format", ["json", "binary"]) +def test_legacy_sam_embedding_metadata(load_legacy_model, monkeypatch, response_format): + for module_name in ["rasterio", "rasterio.features", "segment_anything"]: + module = ModuleType(module_name) + if module_name == "segment_anything": + setattr(module, "SamPredictor", Mock()) + setattr(module, "sam_model_registry", {}) + monkeypatch.setitem(sys.modules, module_name, module) + model = load_legacy_model("sam.segment_anything", "SegmentAnything") + model._state_lock = Lock() + model.embed_image = Mock(return_value=(np.array([[0.1, 0.2]]), None)) + + response = model.infer_from_request( + SamEmbeddingRequest( + id="test-request", model_id=None, image=IMAGE, format=response_format + ) + ) + + assert response.resolved_model.model_dump() == METADATA + if response_format == "binary": + assert isinstance(response.embeddings, bytes) + else: + assert response.embeddings == [[0.1, 0.2]] + + +def test_legacy_sam_binary_segmentation_preserves_payload( + load_legacy_model, monkeypatch +): + for module_name in ["rasterio", "rasterio.features", "segment_anything"]: + module = ModuleType(module_name) + if module_name == "segment_anything": + setattr(module, "SamPredictor", Mock()) + setattr(module, "sam_model_registry", {}) + monkeypatch.setitem(sys.modules, module_name, module) + model = load_legacy_model("sam.segment_anything", "SegmentAnything") + model._state_lock = Lock() + model.segment_image = Mock(return_value=(np.zeros((1, 2, 2)), np.zeros((1, 2, 2)))) + + response = model.infer_from_request( + SamSegmentationRequest( + id="test-request", + model_id=None, + embeddings=None, + image=IMAGE, + format="binary", + ) + ) + + assert isinstance(response, bytes) diff --git a/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py new file mode 100644 index 0000000000..54128017fe --- /dev/null +++ b/tests/inference/unit_tests/models/test_specialized_resolved_model_metadata.py @@ -0,0 +1,274 @@ +import importlib +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest + +import inference.models +from inference.core.entities.requests.clip import ClipTextEmbeddingRequest +from inference.core.entities.requests.doctr import DoctrOCRInferenceRequest +from inference.core.entities.requests.easy_ocr import EasyOCRInferenceRequest +from inference.core.entities.requests.groundingdino import GroundingDINOInferenceRequest +from inference.core.entities.requests.inference import InferenceRequestImage +from inference.core.entities.requests.perception_encoder import ( + PerceptionEncoderTextEmbeddingRequest, +) +from inference.core.entities.requests.sam import ( + SamEmbeddingRequest, + SamSegmentationRequest, +) +from inference.core.entities.requests.sam2 import ( + Sam2EmbeddingRequest, + Sam2SegmentationRequest, +) +from inference.core.entities.requests.sam3 import Sam3SegmentationRequest +from inference.core.entities.requests.trocr import TrOCRInferenceRequest +from inference.core.entities.responses.inference import ( + InferenceResponseImage, + ObjectDetectionInferenceResponse, +) +from inference.core.entities.responses.ocr import OCRInferenceResponse +from inference.core.entities.responses.sam3 import Sam3SegmentationResponse + +METADATA = { + "model_id": "foundation/model", + "model_package_id": "package-id", + "backend": "torch", + "quantization": "fp32", +} +IMAGE = InferenceRequestImage(type="base64", value="unused") + + +@pytest.fixture +def load_adapter(monkeypatch): + base_module = importlib.import_module("inference.core.models.base") + monkeypatch.setattr(base_module, "USE_INFERENCE_MODELS", True) + imported_modules = [] + + def load(module_name, class_name): + module_name = f"inference.models.{module_name}_inference_models" + monkeypatch.delitem(sys.modules, module_name, raising=False) + module = importlib.import_module(module_name) + imported_modules.append(module_name) + adapter_class = getattr(module, class_name) + adapter = adapter_class.__new__(adapter_class) + adapter._model = SimpleNamespace(resolved_model=SimpleNamespace(**METADATA)) + if hasattr(module, "record_fixed_model_input_for_request"): + monkeypatch.setattr(module, "record_fixed_model_input_for_request", Mock()) + return adapter + + yield load + for module_name in imported_modules: + sys.modules.pop(module_name, None) + + +@pytest.fixture +def segmentation_backends(monkeypatch): + # Request handling does not need the optional GPU model implementations. + for name in ["sam", "sam2", "sam3"]: + package = ModuleType(f"inference.models.{name}") + package.__path__ = [str(Path(inference.models.__file__).parent / name)] + monkeypatch.setitem(sys.modules, package.__name__, package) + for module_name, class_name in [ + ("inference_models.models.sam.sam_torch", "SAMTorch"), + ("inference_models.models.sam2.sam2_torch", "SAM2Torch"), + ("inference_models.models.sam3.sam3_torch", "SAM3Torch"), + ]: + module = ModuleType(module_name) + setattr(module, class_name, type(class_name, (), {})) + setattr(module, "compute_image_hash", Mock()) + monkeypatch.setitem(sys.modules, module_name, module) + monkeypatch.setitem(sys.modules, "rasterio", ModuleType("rasterio")) + sam2 = ModuleType("sam2") + sam2_utils = ModuleType("sam2.utils") + sam2_misc = ModuleType("sam2.utils.misc") + setattr(sam2, "utils", sam2_utils) + setattr(sam2_utils, "misc", sam2_misc) + for module in [sam2, sam2_utils, sam2_misc]: + monkeypatch.setitem(sys.modules, module.__name__, module) + + +@pytest.mark.parametrize( + "module_name,class_name,request_class", + [ + ("clip.clip", "InferenceModelsClipAdapter", ClipTextEmbeddingRequest), + ( + "perception_encoder.perception_encoder", + "InferenceModelsPerceptionEncoderAdapter", + PerceptionEncoderTextEmbeddingRequest, + ), + ], +) +def test_embedding_request_metadata( + load_adapter, module_name, class_name, request_class +): + adapter = load_adapter(module_name, class_name) + adapter.embed_text = Mock(return_value=np.array([[0.1, 0.2]])) + + response = adapter.infer_from_request(request_class(text="cat")) + + assert response.embeddings == [[0.1, 0.2]] + assert response.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.parametrize( + "module_name,class_name,request_class", + [ + ("doctr.doctr_model", "InferenceModelsDocTRAdapter", DoctrOCRInferenceRequest), + ("easy_ocr.easy_ocr", "InferenceModelsEasyOCRAdapter", EasyOCRInferenceRequest), + ], +) +@pytest.mark.parametrize("batched", [True, False]) +def test_ocr_request_metadata( + load_adapter, module_name, class_name, request_class, batched +): + adapter = load_adapter(module_name, class_name) + adapter.single_request = Mock( + side_effect=lambda request: OCRInferenceResponse(result="text", time=0.1) + ) + + response = adapter.infer_from_request( + request_class(image=[IMAGE, IMAGE] if batched else IMAGE) + ) + + responses = response if batched else [response] + assert len(responses) == (2 if batched else 1) + assert all(item.model_dump()["resolved_model"] == METADATA for item in responses) + + +def test_trocr_request_metadata(load_adapter): + adapter = load_adapter("trocr.trocr", "InferenceModelsTrOCRAdapter") + adapter.infer = Mock(return_value="text") + + response = adapter.infer_from_request( + TrOCRInferenceRequest.model_validate({"image": IMAGE}) + ) + + assert response.result == "text" + assert response.model_dump()["resolved_model"] == METADATA + + +def test_grounding_dino_request_metadata(load_adapter): + adapter = load_adapter( + "grounding_dino.grounding_dino", "InferenceModelsGroundingDINOAdapter" + ) + adapter.infer = Mock( + return_value=[ + ObjectDetectionInferenceResponse( + predictions=[], image=InferenceResponseImage(width=8, height=8) + ) + ] + ) + + response = adapter.infer_from_request( + GroundingDINOInferenceRequest.model_validate({"image": IMAGE, "text": ["cat"]}) + ) + + assert response[0].model_dump()["resolved_model"] == METADATA + + +def test_ocr_request_without_backend_metadata(load_adapter): + adapter = load_adapter("trocr.trocr", "InferenceModelsTrOCRAdapter") + adapter._model = SimpleNamespace() + adapter.infer = Mock(return_value="text") + + response = adapter.infer_from_request( + TrOCRInferenceRequest.model_validate({"image": IMAGE}) + ) + + assert "resolved_model" not in response.model_dump(exclude_none=True) + + +@pytest.mark.usefixtures("segmentation_backends") +@pytest.mark.parametrize( + "module_name,class_name,request_class,embedding_result", + [ + ( + "sam.segment_anything", + "InferenceModelsSAMAdapter", + SamEmbeddingRequest, + (np.zeros((1, 1, 1, 1)), None), + ), + ( + "sam2.segment_anything2", + "InferenceModelsSAM2Adapter", + Sam2EmbeddingRequest, + (None, None, "image-id"), + ), + ( + "sam3.visual_segmentation", + "InferenceModelsSAM3InteractiveAdapter", + Sam2EmbeddingRequest, + (None, None, "image-id"), + ), + ], +) +def test_sam_embedding_request_metadata( + load_adapter, module_name, class_name, request_class, embedding_result +): + adapter = load_adapter(module_name, class_name) + adapter.embed_image = Mock(return_value=embedding_result) + + response = adapter.infer_from_request(request_class(image=IMAGE)) + + assert response.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.usefixtures("segmentation_backends") +@pytest.mark.parametrize( + "module_name,class_name", + [ + ("sam2.segment_anything2", "InferenceModelsSAM2Adapter"), + ("sam3.visual_segmentation", "InferenceModelsSAM3InteractiveAdapter"), + ], +) +@pytest.mark.parametrize("format", ["json", "rle", "binary"]) +def test_sam_segmentation_response_formats( + load_adapter, module_name, class_name, format +): + adapter = load_adapter(module_name, class_name) + masks = np.zeros((1, 8, 8), dtype=np.float32) + adapter.segment_image = Mock(return_value=(masks, np.array([0.9]), masks)) + request = Sam2SegmentationRequest.model_construct(image=IMAGE, format=format) + + response = adapter.infer_from_request(request) + + if format == "binary": + assert isinstance(response, bytes) + else: + assert response.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.usefixtures("segmentation_backends") +@pytest.mark.parametrize("format", ["json", "binary"]) +def test_sam_segmentation_metadata(load_adapter, format): + adapter = load_adapter("sam.segment_anything", "InferenceModelsSAMAdapter") + masks = np.zeros((1, 8, 8), dtype=np.float32) + adapter.segment_image = Mock(return_value=(masks, masks)) + + response = adapter.infer_from_request( + SamSegmentationRequest.model_validate({"format": format}) + ) + + if format == "binary": + assert isinstance(response, bytes) + else: + assert response.model_dump()["resolved_model"] == METADATA + + +@pytest.mark.usefixtures("segmentation_backends") +def test_sam3_segmentation_request_metadata(load_adapter): + adapter = load_adapter("sam3.segment_anything3", "InferenceModelsSAM3Adapter") + adapter.segment_image = Mock( + return_value=Sam3SegmentationResponse(prompt_results=[], time=0.1) + ) + request = Sam3SegmentationRequest.model_validate( + {"image": IMAGE, "prompts": [{"text": "cat"}]} + ) + + response = adapter.infer_from_request(request) + + assert response.model_dump()["resolved_model"] == METADATA diff --git a/workflows/roboflow_workflows/core_steps/common/inference_response_entities.py b/workflows/roboflow_workflows/core_steps/common/inference_response_entities.py index 07d9179b9f..ada13086b7 100644 --- a/workflows/roboflow_workflows/core_steps/common/inference_response_entities.py +++ b/workflows/roboflow_workflows/core_steps/common/inference_response_entities.py @@ -36,6 +36,34 @@ class InferenceResponseImage(BaseModel): ) +class ResolvedModel(BaseModel): + """Identity of the model and available package details used for inference. + + Attributes: + model_id (str): ID of the loaded model, using the canonical ID when available. + model_package_id (Optional[str]): ID of the loaded model package. + backend (Optional[str]): Backend of the loaded package. + quantization (Optional[str]): Quantization of the loaded package. + """ + + model_config = ConfigDict(protected_namespaces=()) + model_id: str = Field( + description="ID of the loaded model, using the canonical ID when available." + ) + model_package_id: Optional[str] = Field( + default=None, + description="ID of the package that loaded successfully and produced this result, including after a loading fallback.", + ) + backend: Optional[str] = Field( + default=None, + description="Backend of the loaded package, such as onnx, trt, or torch.", + ) + quantization: Optional[str] = Field( + default=None, + description="Package quantization, such as fp32 or fp16, or unknown when unavailable. This does not specify the input tensor dtype or the precision of every runtime operation.", + ) + + class InferenceResponse(BaseModel): """Base inference response. @@ -43,6 +71,7 @@ class InferenceResponse(BaseModel): inference_id (Optional[str]): Unique identifier of inference frame_id (Optional[int]): The frame id of the image used in inference if the input was a video. time (Optional[float]): The time in seconds it took to produce the predictions including image preprocessing. + resolved_model (Optional[ResolvedModel]): Model identity and available package details for this result. """ model_config = ConfigDict(protected_namespaces=()) @@ -57,6 +86,10 @@ class InferenceResponse(BaseModel): default=None, description="The time in seconds it took to produce the predictions including image preprocessing", ) + resolved_model: Optional[ResolvedModel] = Field( + default=None, + description="Model identity and available package details for this result.", + ) class CvInferenceResponse(InferenceResponse):