diff --git a/inference/core/entities/responses/inference.py b/inference/core/entities/responses/inference.py index 4d338cf580..772813042d 100644 --- a/inference/core/entities/responses/inference.py +++ b/inference/core/entities/responses/inference.py @@ -355,7 +355,13 @@ class AnomalyDetectionResponse(ClassificationInferenceResponse): anomaly_threshold: float = Field( description="Decision threshold fitted on validation images" ) - is_anomalous: bool + anomaly_scale: float = Field( + description="Calibration scale: the anomalous confidence is " + "sigmoid((anomaly_score - anomaly_threshold) / anomaly_scale)" + ) + is_anomalous: bool = Field( + description="Whether the anomalous confidence reached the request confidence" + ) anomaly_map: Optional[AnomalyMapPayload] = Field( default=None, description="Raw local anomaly evidence at the network input resolution", diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index cbb970b84a..37bed16daa 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -5072,11 +5072,17 @@ def legacy_infer_from_request( args = {"keypoint_confidence": keypoint_confidence} elif task_type == "semantic-segmentation": inference_request_type = SemanticSegmentationInferenceRequest + # Leave an omitted confidence unset so anomaly detection models can + # tell it apart from an explicit one and use the model-eval value. + if ( + task_type != "classification" + or "confidence" in request.query_params + ): + args["confidence"] = confidence inference_request = inference_request_type( api_key=api_key, model_id=model_id, image=request_image, - confidence=confidence, iou_threshold=overlap, max_detections=max_detections, visualization_labels=labels, diff --git a/inference/core/interfaces/workflows_models_provider.py b/inference/core/interfaces/workflows_models_provider.py index c11abeaf1e..94d643178d 100644 --- a/inference/core/interfaces/workflows_models_provider.py +++ b/inference/core/interfaces/workflows_models_provider.py @@ -159,7 +159,11 @@ def run_classification( disable_active_learning: Optional[bool] = None, active_learning_target_dataset: Optional[str] = None, inference_kwargs: Optional[Dict[str, Any]] = None, + include_anomaly_map: bool = False, ) -> List[dict]: + # Set only when asked, so the request a classification block builds is + # field-for-field what it was before the anomaly block existed. + anomaly_arguments = {"include_anomaly_map": True} if include_anomaly_map else {} request = ClassificationInferenceRequest( api_key=api_key, model_id=model_id, @@ -168,6 +172,7 @@ def run_classification( disable_active_learning=disable_active_learning, source=_WORKFLOW_SOURCE, active_learning_target_dataset=active_learning_target_dataset, + **anomaly_arguments, ) return self._dump( self._infer(model_id=model_id, request=request, **(inference_kwargs or {})) diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index 4f55a92edc..a07ee1d88f 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -1834,6 +1834,16 @@ def _reshape_classification_confidences( class InferenceModelsAnomalyDetectionAdapter(InferenceModelsClassificationAdapter): """Serves PatchCore and FoundAD through the classification route.""" + def infer_from_request( + self, + request: ClassificationInferenceRequest, + ) -> Union[List[InferenceResponse], InferenceResponse]: + # The classification request defaults confidence to 0.4; a caller who sends + # none gets the model-eval recommendation instead. + if "confidence" not in request.model_fields_set: + request = request.model_copy(update={"confidence": "best"}) + return super().infer_from_request(request) + def postprocess( self, predictions: Any, @@ -1841,7 +1851,9 @@ def postprocess( **kwargs, ) -> List[AnomalyDetectionResponse]: post_processed_predictions = self._model.post_process( - predictions, include_anomaly_map=kwargs.get("include_anomaly_map", False) + predictions, + include_anomaly_map=kwargs.get("include_anomaly_map", False), + confidence=kwargs["confidence"], ) return prepare_anomaly_detection_response( post_processed_predictions, @@ -1855,8 +1867,8 @@ def prepare_anomaly_detection_response( image_sizes: List[Tuple[int, int]], class_names: List[str], ) -> List[AnomalyDetectionResponse]: - # The decision comes from the threshold saved with the model, so both classes - # are always reported and the request confidence does not filter them. + # The request confidence decides the verdict instead of filtering classes, so + # both classes are always reported, the decided one first. batch_confidences = _reshape_classification_confidences( confidence=post_processed_predictions.confidence.cpu(), expected_num_images=len(image_sizes), @@ -1886,6 +1898,7 @@ def prepare_anomaly_detection_response( confidence=class_predictions[0]["confidence"], anomaly_score=image_metadata["anomaly_score"], anomaly_threshold=image_metadata["anomaly_threshold"], + anomaly_scale=image_metadata["anomaly_scale"], is_anomalous=image_metadata["is_anomalous"], anomaly_map=( AnomalyMapPayload(**encode_anomaly_map(anomaly_map)) diff --git a/inference/core/version.py b/inference/core/version.py index 68e6fa3378..46cf645d76 100644 --- a/inference/core/version.py +++ b/inference/core/version.py @@ -1,4 +1,4 @@ -__version__ = "1.8.1" +__version__ = "1.8.2" if __name__ == "__main__": diff --git a/inference_model_manager/inference_model_manager/registry_defaults.py b/inference_model_manager/inference_model_manager/registry_defaults.py index 0be440af0e..f64e38e6f5 100644 --- a/inference_model_manager/inference_model_manager/registry_defaults.py +++ b/inference_model_manager/inference_model_manager/registry_defaults.py @@ -188,8 +188,16 @@ def _p(*dicts: dict) -> dict: _P_SAM2_SEGMENT = _p( _P_SAM_SEGMENT_COMMON, { - "load_from_mask_input_cache": {"type": "bool", "required": False, "default": False}, - "save_to_mask_input_cache": {"type": "bool", "required": False, "default": False}, + "load_from_mask_input_cache": { + "type": "bool", + "required": False, + "default": False, + }, + "save_to_mask_input_cache": { + "type": "bool", + "required": False, + "default": False, + }, }, ) @@ -413,6 +421,7 @@ def _unpack_config(cfg: tuple) -> tuple: _p( _P_IMAGES, {"include_anomaly_map": {"type": "bool", "required": False}}, + _P_CONFIDENCE, _K_PREPROC, ), "validate_images_required", @@ -639,7 +648,13 @@ def _unpack_config(cfg: tuple) -> tuple: True, _p( _P_VLM_PROMPT, - {"enable_thinking": {"type": "bool", "required": False, "default": False}}, + { + "enable_thinking": { + "type": "bool", + "required": False, + "default": False, + } + }, ), "validate_images_and_prompt", "serialize_text", @@ -653,7 +668,13 @@ def _unpack_config(cfg: tuple) -> tuple: True, _p( _P_VLM_PROMPT, - {"images_to_single_prompt": {"type": "bool", "required": False, "default": True}}, + { + "images_to_single_prompt": { + "type": "bool", + "required": False, + "default": True, + } + }, ), "validate_images_and_prompt", "serialize_text", @@ -677,7 +698,11 @@ def _unpack_config(cfg: tuple) -> tuple: "caption", "caption_image", True, - _p(_P_IMAGES, {"granularity": {"type": "str", "required": False}}, _P_MAX_NEW_TOKENS), + _p( + _P_IMAGES, + {"granularity": {"type": "str", "required": False}}, + _P_MAX_NEW_TOKENS, + ), "validate_images_required", "serialize_text", "roboflow-text-v1", @@ -829,7 +854,11 @@ def _unpack_config(cfg: tuple) -> tuple: "caption", "caption", True, - _p(_P_IMAGES, {"length": {"type": "str", "required": False}}, _P_MAX_NEW_TOKENS), + _p( + _P_IMAGES, + {"length": {"type": "str", "required": False}}, + _P_MAX_NEW_TOKENS, + ), "validate_images_required", "serialize_text", "roboflow-text-v1", @@ -921,8 +950,16 @@ def _unpack_config(cfg: tuple) -> tuple: "image": {"type": "image", "required": True}, "bboxes": {"type": "list", "required": True}, "state_dict": {"type": "dict", "required": False}, - "clear_old_points": {"type": "bool", "required": False, "default": True}, - "normalize_coords": {"type": "bool", "required": False, "default": True}, + "clear_old_points": { + "type": "bool", + "required": False, + "default": True, + }, + "normalize_coords": { + "type": "bool", + "required": False, + "default": True, + }, "frame_idx": {"type": "int", "required": False, "default": 0}, }, "validate_passthrough", @@ -1074,9 +1111,16 @@ def lazy_register_by_names(mro_names: list[str]) -> None: continue placeholder = type(name, (), {}) for cfg in config: - action_name, method, default, params, val_name, ser_name, resp_type, aliases = ( - _unpack_config(cfg) - ) + ( + action_name, + method, + default, + params, + val_name, + ser_name, + resp_type, + aliases, + ) = _unpack_config(cfg) registry.register( placeholder, action_name, diff --git a/inference_model_manager/inference_model_manager/serializers_typed.py b/inference_model_manager/inference_model_manager/serializers_typed.py index 573823539d..5f3a7c57d7 100644 --- a/inference_model_manager/inference_model_manager/serializers_typed.py +++ b/inference_model_manager/inference_model_manager/serializers_typed.py @@ -98,7 +98,13 @@ def serialize_detections_compact(output: Any, model: Any) -> dict: # Classification (single-label) # --------------------------------------------------------------------------- -_ANOMALY_FIELDS = ("anomaly_score", "anomaly_threshold", "is_anomalous", "anomaly_map") +_ANOMALY_FIELDS = ( + "anomaly_score", + "anomaly_threshold", + "anomaly_scale", + "is_anomalous", + "anomaly_map", +) def _anomaly_fields(output: Any, index: int = 0) -> dict: diff --git a/inference_model_manager/tests/unit_tests/test_model_manager.py b/inference_model_manager/tests/unit_tests/test_model_manager.py index 9d773f9922..93f855d88a 100644 --- a/inference_model_manager/tests/unit_tests/test_model_manager.py +++ b/inference_model_manager/tests/unit_tests/test_model_manager.py @@ -565,11 +565,19 @@ def forward(self, pre_processed_images, **kwargs): original_sizes_hw=[(8, 8), (8, 8)], ) - def post_process(self, model_results, include_anomaly_map=False, **kwargs): + def post_process( + self, + model_results, + include_anomaly_map=False, + confidence="best", + **kwargs, + ): return post_process_anomaly_scores( model_results, calibration=AnomalyCalibration(threshold=2.0, scale=1.0), include_anomaly_map=include_anomaly_map, + confidence=confidence, + recommended_parameters=None, ) mm = ModelManager() @@ -1100,9 +1108,7 @@ def test_unload_releases_cuda_cache(self, monkeypatch): import inference_model_manager.model_manager as mm_mod calls = [] - monkeypatch.setattr( - mm_mod, "_try_release_cuda_memory", lambda: calls.append(1) - ) + monkeypatch.setattr(mm_mod, "_try_release_cuda_memory", lambda: calls.append(1)) mm = ModelManager() _patch_create_backend(mm, {}) mm.load("model-a", api_key="") @@ -1225,8 +1231,7 @@ def load_model(name): errors.append(exc) threads = [ - threading.Thread(target=load_model, args=(f"new{i}",)) - for i in range(n_new) + threading.Thread(target=load_model, args=(f"new{i}",)) for i in range(n_new) ] for t in threads: t.start() @@ -1465,11 +1470,10 @@ class TestStreamPipeline: @pytest.fixture def segmentation_manager(self, monkeypatch): + from inference_model_manager.registry_defaults import lazy_register from inference_models.models.base.instance_segmentation import ( InstanceSegmentationModel, ) - - from inference_model_manager.registry_defaults import lazy_register from tests.unit_tests.test_stream_pipeline import FakeSegmentationModel lazy_register(InstanceSegmentationModel) @@ -1697,9 +1701,7 @@ def test_unload_shuts_the_pipeline_down(self, segmentation_manager): assert mm.flush_model_stream_pipeline("seg/1") is None def test_unload_resolves_an_in_flight_frame(self, segmentation_manager): - from inference_models.models.base.async_handoff import ( - get_async_response_future, - ) + from inference_models.models.base.async_handoff import get_async_response_future mm, _, models = segmentation_manager mm.load("seg/1", api_key="") @@ -1824,7 +1826,6 @@ def test_pipeline_is_drained_after_the_last_admitted_frame( from inference_models.models.base.async_handoff import ( get_async_response_context_id, ) - from tests.unit_tests.test_stream_pipeline import ( FakeSegmentationModel, queues_of, diff --git a/inference_model_manager/tests/unit_tests/test_registry_defaults.py b/inference_model_manager/tests/unit_tests/test_registry_defaults.py index 7497251809..1bd3dc49e0 100644 --- a/inference_model_manager/tests/unit_tests/test_registry_defaults.py +++ b/inference_model_manager/tests/unit_tests/test_registry_defaults.py @@ -285,7 +285,12 @@ def test_cosmos3_edge_reasoner_action_config(): assert prompt_action[2] is True assert prompt_action[3]["images"] == {"type": "image", "required": True} assert prompt_action[3]["prompt"] == {"type": "str", "required": True} - for param in ("max_new_tokens", "do_sample", "skip_special_tokens", "return_thinking"): + for param in ( + "max_new_tokens", + "do_sample", + "skip_special_tokens", + "return_thinking", + ): assert param in prompt_action[3], param assert prompt_action[3][param]["required"] is False assert prompt_action[3]["max_new_tokens"]["type"] == "int" @@ -364,7 +369,11 @@ def test_model_owned_defaults_not_injected(): def test_per_family_detector_contracts(): - from inference_model_manager.registry_defaults import _K_ISEG, _ACTION_CONFIGS, _unpack_config + from inference_model_manager.registry_defaults import ( + _ACTION_CONFIGS, + _K_ISEG, + _unpack_config, + ) def params_of(key): return _unpack_config(_ACTION_CONFIGS[key][0])[3] @@ -514,8 +523,8 @@ def test_per_family_contracts_have_no_default_keys(): def test_vlm_generation_params(): from inference_model_manager.registry_defaults import ( - _P_VLM_PROMPT, _ACTION_CONFIGS, + _P_VLM_PROMPT, _unpack_config, ) @@ -550,8 +559,8 @@ def test_vllm_proxy_action_configs_equal_the_hf_rows(proxy_name, hf_name): def test_vlm_generation_contracts_match_model_signatures(): from inference_model_manager.registry_defaults import ( - _P_GEN_FLAGS, _ACTION_CONFIGS, + _P_GEN_FLAGS, _unpack_config, ) @@ -581,10 +590,18 @@ def cfgs_of(key): assert g4["enable_thinking"] == {"type": "bool", "required": False} sv = cfgs_of("SmolVLMHF")["prompt"][3] - assert sv["images_to_single_prompt"] == {"type": "bool", "required": False, "default": True} + assert sv["images_to_single_prompt"] == { + "type": "bool", + "required": False, + "default": True, + } q35 = cfgs_of("Qwen35HF")["prompt"][3] - assert q35["enable_thinking"] == {"type": "bool", "required": False, "default": False} + assert q35["enable_thinking"] == { + "type": "bool", + "required": False, + "default": False, + } cosmos = cfgs_of("Cosmos3EdgeReasoner")["prompt"][3] assert set(cosmos) == { @@ -614,7 +631,12 @@ def cfgs_of(key): glm = cfgs_of("GlmOcrHF") for action in ("recognize_text", "recognize_table", "recognize_formula"): p = glm[action][3] - assert set(p) == {"images", "max_new_tokens", "do_sample", "skip_special_tokens"}, action + assert set(p) == { + "images", + "max_new_tokens", + "do_sample", + "skip_special_tokens", + }, action for name, spec in p.items(): assert "default" not in spec, (action, name) gp = glm["prompt"][3] @@ -653,8 +675,14 @@ def test_classification_declares_include_anomaly_map(): params = _unpack_config(_ACTION_CONFIGS["ClassificationModel"][0])[3] - assert set(params) == {"images", "include_anomaly_map", *PREPROC_FLAGS} + assert set(params) == { + "images", + "include_anomaly_map", + "confidence", + *PREPROC_FLAGS, + } assert params["include_anomaly_map"] == {"type": "bool", "required": False} + assert params["confidence"] == {"type": "confidence", "required": False} def test_roboflow_trained_families_declare_pre_processing_flags(): diff --git a/inference_model_manager/tests/unit_tests/test_serializers_typed.py b/inference_model_manager/tests/unit_tests/test_serializers_typed.py index b1b9291945..747f93fec7 100644 --- a/inference_model_manager/tests/unit_tests/test_serializers_typed.py +++ b/inference_model_manager/tests/unit_tests/test_serializers_typed.py @@ -205,6 +205,7 @@ def test_text_single(self): def test_serialize_gaze_compact(): from types import SimpleNamespace + from inference_model_manager.serializers_typed import serialize_gaze_compact out = SimpleNamespace(yaw=[0.1, 0.2], pitch=[-0.3, 0.0]) @@ -218,6 +219,7 @@ def test_serialize_gaze_compact(): def test_serialize_gaze_compact_batch(): from types import SimpleNamespace + from inference_model_manager.serializers_typed import serialize_gaze_compact outs = [ @@ -234,6 +236,7 @@ def test_serialize_gaze_compact_batch(): def test_serialize_structured_ocr_compact(): from types import SimpleNamespace + from inference_model_manager.serializers_typed import ( serialize_structured_ocr_compact, ) @@ -264,6 +267,7 @@ def test_serialize_structured_ocr_compact(): def test_serialize_sam_segmentation_compact(): from types import SimpleNamespace + from inference_model_manager.serializers_typed import ( serialize_sam_segmentation_compact, ) @@ -282,6 +286,7 @@ def test_serialize_sam_segmentation_compact(): def test_serialize_sam_segmentation_compact_single(): from types import SimpleNamespace + from inference_model_manager.serializers_typed import ( serialize_sam_segmentation_compact, ) @@ -388,6 +393,7 @@ def _anomaly(include_map: bool, *, is_anomalous: bool = True): metadata = { "anomaly_score": 3.5, "anomaly_threshold": 2.0, + "anomaly_scale": 0.5, "is_anomalous": is_anomalous, } if include_map: @@ -465,6 +471,8 @@ def _two_image_anomaly_prediction(include_anomaly_map: bool): raw, calibration=AnomalyCalibration(threshold=2.0, scale=1.0), include_anomaly_map=include_anomaly_map, + confidence="default", + recommended_parameters=None, ) return prediction @@ -480,6 +488,7 @@ def test_each_image_keeps_its_own_anomaly_fields(self): assert [first["anomaly_score"], second["anomaly_score"]] == [0.5, 3.0] assert [first["is_anomalous"], second["is_anomalous"]] == [False, True] assert first["anomaly_threshold"] == second["anomaly_threshold"] == 2.0 + assert first["anomaly_scale"] == second["anomaly_scale"] == 1.0 assert np.asarray(first["anomaly_map"]).tolist() == [[0.0, 0.5], [0.5, 0.5]] assert np.asarray(second["anomaly_map"]).tolist() == [[1.0, 2.0], [3.0, 3.0]] diff --git a/inference_models/docs/changelog.md b/inference_models/docs/changelog.md index 7b3e8f182c..265b50ae5f 100644 --- a/inference_models/docs/changelog.md +++ b/inference_models/docs/changelog.md @@ -1,9 +1,10 @@ # Changelog -## Unreleased +## `0.39.5` ### Added +- Anomaly detection models put `anomaly_scale` in `images_metadata[i]`, next to `anomaly_score`, `anomaly_threshold` and `is_anomalous`, so a caller can re-decide against its own threshold with the same rule. - `RoboflowModelMetadata.model_latency_ms` exposes the registry's `modelLatencyMs`. A non-numeric value is read as `None` instead of failing the metadata parse. - `SAM3Torch.segment_with_visual_prompts(...)` accepts `mask_format="rle"` to return @@ -46,6 +47,14 @@ inputs as RGB. - GLM-OCR table recognition sent the formula-recognition prompt. +### Changed + +- Anomaly detection (PatchCore, FoundAD): `is_anomalous` and `class_id` are decided by the request `confidence` — `sigmoid((anomaly_score - anomaly_threshold) / anomaly_scale) >= confidence` — instead of always `anomaly_score >= anomaly_threshold`. `post_process(..., confidence=...)` accepts a number in `[0, 1]`, `"best"` (the model-eval recommendation through `recommended_parameters`, which both models now accept at load) or `"default"` (0.5, the calibrated decision, identical to the previous rule). + +### Fixed + +- PatchCore bounds its nearest-neighbour distance matrix by elements (`NEAREST_NEIGHBOURS_CHUNK_ELEMENTS = 2**26`, 256 MiB in fp32) instead of 4096 rows, and computes it in place. A 640 px model with a ~140k-entry memory bank allocated about 2 GiB per chunk per request and ran out of GPU memory under concurrent requests. + ## `0.39.4` ### Added diff --git a/inference_models/docs/models/foundad-anomaly-detection.md b/inference_models/docs/models/foundad-anomaly-detection.md index bd303e448e..c0b7cee41f 100644 --- a/inference_models/docs/models/foundad-anomaly-detection.md +++ b/inference_models/docs/models/foundad-anomaly-detection.md @@ -38,7 +38,7 @@ The package contains the DINOv3 encoder and the predictor. Loading does not down |---------|-----------| | **Training** | ✅ Train custom models on Roboflow | | **Upload Weights** | ❌ Not supported | -| **Workflows** | ✅ Use in [Workflows](https://docs.roboflow.com/workflows) via Classification block | +| **Workflows** | ✅ Use in [Workflows](https://docs.roboflow.com/workflows) via the Anomaly Detection Model block | | **Self-Hosting** | ✅ Deploy with `inference-models` | ## Usage Example @@ -53,11 +53,12 @@ model = AutoModel.from_pretrained( ) image = cv2.imread("path/to/image.jpg") -prediction = model(image, include_anomaly_map=True) +prediction = model(image, include_anomaly_map=True, confidence=0.5) result = prediction.images_metadata[0] print(model.class_names[prediction.class_id[0].item()]) # "normal" or "anomalous" print(result["anomaly_score"], result["anomaly_threshold"], result["is_anomalous"]) +print(result["anomaly_scale"]) # sigmoid((score - threshold) / scale) is the anomalous confidence heatmap = result["anomaly_map"] # float32 array of shape (image_size, image_size) overlay = cv2.resize(heatmap, (image.shape[1], image.shape[0])) # stretch onto the image ``` @@ -66,11 +67,15 @@ overlay = cv2.resize(heatmap, (image.shape[1], image.shape[0])) # stretch onto The model returns a `ClassificationPrediction` over the fixed classes `normal` and `anomalous`: -- `class_id` follows the threshold saved with the model: `anomalous` when `anomaly_score >= anomaly_threshold`. -- `confidence` is a monotonic transform of the score margin. The threshold maps to exactly `0.5`, so the - argmax always agrees with `class_id`. It is not an estimated defect probability. +- `confidence` is the anomalous confidence `sigmoid((anomaly_score - anomaly_threshold) / anomaly_scale)`, + with the threshold and scale calibrated at training, so it is exactly `0.5` at the saved threshold. It is + not an estimated defect probability. +- `class_id` is `anomalous` when that confidence reaches the `confidence` passed to the model: a number in + `[0, 1]`, `"best"` (the model-eval recommendation, the default) or `"default"` (`0.5`, which reproduces the + decision calibrated at training). The argmax of `confidence` agrees with `class_id` only at `0.5`. - `images_metadata[i]` holds `anomaly_score` (raw score, larger means more anomalous), `anomaly_threshold`, - `is_anomalous` and, when `include_anomaly_map=True` is passed, `anomaly_map`. + `anomaly_scale`, `is_anomalous` and, when `include_anomaly_map=True` is passed, `anomaly_map`. A caller can + re-decide against its own threshold with the same rule. The score of an image does not depend on the other images in the batch. The heatmap is local evidence for the score, not a supervised segmentation mask. It is returned at the network input resolution (a square of the diff --git a/inference_models/docs/models/patchcore-anomaly-detection.md b/inference_models/docs/models/patchcore-anomaly-detection.md index 02d93974ac..2951444e7f 100644 --- a/inference_models/docs/models/patchcore-anomaly-detection.md +++ b/inference_models/docs/models/patchcore-anomaly-detection.md @@ -38,7 +38,7 @@ The package contains the full WideResNet50 backbone and the memory bank. Loading |---------|-----------| | **Training** | ✅ Train custom models on Roboflow | | **Upload Weights** | ❌ Not supported | -| **Workflows** | ✅ Use in [Workflows](https://docs.roboflow.com/workflows) via Classification block | +| **Workflows** | ✅ Use in [Workflows](https://docs.roboflow.com/workflows) via the Anomaly Detection Model block | | **Self-Hosting** | ✅ Deploy with `inference-models` | ## Usage Example @@ -53,11 +53,12 @@ model = AutoModel.from_pretrained( ) image = cv2.imread("path/to/image.jpg") -prediction = model(image, include_anomaly_map=True) +prediction = model(image, include_anomaly_map=True, confidence=0.5) result = prediction.images_metadata[0] print(model.class_names[prediction.class_id[0].item()]) # "normal" or "anomalous" print(result["anomaly_score"], result["anomaly_threshold"], result["is_anomalous"]) +print(result["anomaly_scale"]) # sigmoid((score - threshold) / scale) is the anomalous confidence heatmap = result["anomaly_map"] # float32 array of shape (image_size, image_size) overlay = cv2.resize(heatmap, (image.shape[1], image.shape[0])) # stretch onto the image ``` @@ -66,11 +67,15 @@ overlay = cv2.resize(heatmap, (image.shape[1], image.shape[0])) # stretch onto The model returns a `ClassificationPrediction` over the fixed classes `normal` and `anomalous`: -- `class_id` follows the threshold saved with the model: `anomalous` when `anomaly_score >= anomaly_threshold`. -- `confidence` is a monotonic transform of the score margin. The threshold maps to exactly `0.5`, so the - argmax always agrees with `class_id`. It is not an estimated defect probability. +- `confidence` is the anomalous confidence `sigmoid((anomaly_score - anomaly_threshold) / anomaly_scale)`, + with the threshold and scale calibrated at training, so it is exactly `0.5` at the saved threshold. It is + not an estimated defect probability. +- `class_id` is `anomalous` when that confidence reaches the `confidence` passed to the model: a number in + `[0, 1]`, `"best"` (the model-eval recommendation, the default) or `"default"` (`0.5`, which reproduces the + decision calibrated at training). The argmax of `confidence` agrees with `class_id` only at `0.5`. - `images_metadata[i]` holds `anomaly_score` (raw score, larger means more anomalous), `anomaly_threshold`, - `is_anomalous` and, when `include_anomaly_map=True` is passed, `anomaly_map`. + `anomaly_scale`, `is_anomalous` and, when `include_anomaly_map=True` is passed, `anomaly_map`. A caller can + re-decide against its own threshold with the same rule. The score of an image does not depend on the other images in the batch. The heatmap is local evidence for the score, not a supervised segmentation mask. It is returned at the network input resolution (a square of the diff --git a/inference_models/inference_models/models/common/anomaly_detection.py b/inference_models/inference_models/models/common/anomaly_detection.py index 432e888f8c..3259f23d85 100644 --- a/inference_models/inference_models/models/common/anomaly_detection.py +++ b/inference_models/inference_models/models/common/anomaly_detection.py @@ -19,10 +19,16 @@ from pydantic import BaseModel, ConfigDict, Field, PositiveInt, ValidationError from inference_models import ClassificationPrediction -from inference_models.entities import ColorFormat +from inference_models.entities import ColorFormat, Confidence from inference_models.errors import CorruptedModelPackageError, ModelInputError +from inference_models.models.common.roboflow.post_processing import ConfidenceFilter +from inference_models.weights_providers.entities import RecommendedParameters ANOMALY_CLASS_NAMES = ["normal", "anomalous"] +ANOMALOUS_CLASS_ID = 1 +# The anomalous confidence is exactly 0.5 at the saved threshold, so 0.5 reproduces +# the decision calibrated during training. +ANOMALY_DEFAULT_CONFIDENCE = 0.5 CORRUPTED_PACKAGE_HELP_URL = "https://inference-models.roboflow.com/errors/model-loading/#corruptedmodelpackageerror" MODEL_INPUT_HELP_URL = ( "https://inference-models.roboflow.com/errors/input-validation/#modelinputerror" @@ -220,6 +226,8 @@ def post_process_anomaly_scores( model_results: AnomalyRawPrediction, calibration: AnomalyCalibration, include_anomaly_map: bool, + confidence: Confidence, + recommended_parameters: Optional[RecommendedParameters], ) -> ClassificationPrediction: """Turn raw anomaly scores into the shared classification prediction. @@ -228,36 +236,47 @@ def post_process_anomaly_scores( maps for one batch, as produced by the model forward pass. calibration (AnomalyCalibration): Threshold and scale saved with the model. include_anomaly_map (bool): If True, each image's metadata carries its map. + confidence (Confidence): Minimum anomalous confidence for the anomalous + verdict: a number in ``[0, 1]``, ``"best"`` for the model-eval + recommendation, or ``"default"`` for 0.5 (the saved threshold). + recommended_parameters (Optional[RecommendedParameters]): Model-eval + recommendations ``"best"`` resolves against. Returns: - ClassificationPrediction over ``normal`` and ``anomalous``. ``class_id`` - follows the saved threshold and ``confidence`` maps the threshold to - exactly 0.5. ``images_metadata[i]`` holds ``anomaly_score``, - ``anomaly_threshold``, ``is_anomalous`` and, when requested, - ``anomaly_map``: a float32 array of shape ``(image_size, image_size)`` - in network input coordinates, covering the whole (square-resized) image. + ClassificationPrediction over ``normal`` and ``anomalous``. The anomalous + confidence is ``sigmoid((score - threshold) / scale)``, exactly 0.5 at the + saved threshold, and ``class_id`` is anomalous when it reaches + ``confidence``. ``images_metadata[i]`` holds ``anomaly_score``, + ``anomaly_threshold`` (the saved threshold), ``anomaly_scale``, + ``is_anomalous`` and, when requested, ``anomaly_map``: a float32 array of + shape ``(image_size, image_size)`` in network input coordinates, covering + the whole (square-resized) image. Raises: CorruptedModelPackageError: If the model produced non-finite values. + ModelInputError: If ``confidence`` is not a number in ``[0, 1]``, + ``"best"`` or ``"default"``. """ + minimum_confidence = resolve_anomalous_confidence( + confidence=confidence, recommended_parameters=recommended_parameters + ) scores = model_results.scores.double() if not torch.isfinite(scores).all() or not torch.isfinite(model_results.maps).all(): raise CorruptedModelPackageError( message="Anomaly detection model produced non-finite scores.", help_url=CORRUPTED_PACKAGE_HELP_URL, ) - # The threshold maps to exactly 0.5, so the decision below and the argmax of - # the two class confidences always agree. margin = (scores - calibration.threshold) / calibration.scale anomalous_confidence = torch.sigmoid( margin.clamp(-MAX_CONFIDENCE_MARGIN, MAX_CONFIDENCE_MARGIN) ) - is_anomalous = scores >= calibration.threshold + is_anomalous = anomalous_confidence >= minimum_confidence images_metadata = [] for index in range(len(model_results.original_sizes_hw)): metadata = { "anomaly_score": scores[index].item(), "anomaly_threshold": calibration.threshold, + "anomaly_scale": calibration.scale, "is_anomalous": bool(is_anomalous[index].item()), } if include_anomaly_map: @@ -273,3 +292,35 @@ def post_process_anomaly_scores( ).float(), images_metadata=images_metadata, ) + + +def resolve_anomalous_confidence( + confidence: Confidence, + recommended_parameters: Optional[RecommendedParameters], +) -> float: + """The anomalous confidence an image must reach to be called anomalous. + + Raises: + ModelInputError: If ``confidence`` is not a number in ``[0, 1]``, + ``"best"`` or ``"default"``. + """ + if isinstance(confidence, (int, float)) and not isinstance(confidence, bool): + if not 0 <= confidence <= 1: + raise ModelInputError( + message=f"Anomaly confidence must be between 0 and 1, got {confidence}.", + help_url=MODEL_INPUT_HELP_URL, + ) + confidence = float(confidence) + elif confidence not in ("best", "default"): + raise ModelInputError( + message='Anomaly confidence must be a number, "best" or "default".', + help_url=MODEL_INPUT_HELP_URL, + ) + threshold = ConfidenceFilter( + confidence=confidence, + recommended_parameters=recommended_parameters, + default_confidence=ANOMALY_DEFAULT_CONFIDENCE, + ).get_threshold(ANOMALY_CLASS_NAMES) + if isinstance(threshold, torch.Tensor): + return float(threshold[ANOMALOUS_CLASS_ID]) + return threshold diff --git a/inference_models/inference_models/models/foundad/foundad_anomaly_detection_torch.py b/inference_models/inference_models/models/foundad/foundad_anomaly_detection_torch.py index 0dcaad95f3..3e4eeb6939 100644 --- a/inference_models/inference_models/models/foundad/foundad_anomaly_detection_torch.py +++ b/inference_models/inference_models/models/foundad/foundad_anomaly_detection_torch.py @@ -8,7 +8,7 @@ from inference_models import ClassificationModel, ClassificationPrediction from inference_models.configuration import DEFAULT_DEVICE -from inference_models.entities import ColorFormat +from inference_models.entities import ColorFormat, Confidence from inference_models.errors import CorruptedModelPackageError from inference_models.models.common.anomaly_detection import ( ANOMALY_CLASS_NAMES, @@ -21,6 +21,7 @@ pre_process_anomaly_images, ) from inference_models.models.common.model_packages import get_model_package_contents +from inference_models.weights_providers.entities import RecommendedParameters # Settings of the FoundAD recipe used by Roboflow training # (https://github.com/ymxlzgy/FoundAD). @@ -152,6 +153,7 @@ def from_pretrained( cls, model_name_or_path: str, device: torch.device = DEFAULT_DEVICE, + recommended_parameters: Optional[RecommendedParameters] = None, **kwargs, ) -> "FoundADForAnomalyDetectionTorch": model_package_content = get_model_package_contents( @@ -183,17 +185,24 @@ def from_pretrained( weights_only=True, )["state"] model.load_state_dict(state) - return cls(model=model.to(device).eval(), manifest=manifest, device=device) + return cls( + model=model.to(device).eval(), + manifest=manifest, + device=device, + recommended_parameters=recommended_parameters, + ) def __init__( self, model: FoundADModel, manifest: AnomalyModelManifest, device: torch.device, + recommended_parameters: Optional[RecommendedParameters] = None, ): self._model = model self._manifest = manifest self._device = device + self.recommended_parameters = recommended_parameters @property def class_names(self) -> List[str]: @@ -233,10 +242,13 @@ def post_process( self, model_results: AnomalyRawPrediction, include_anomaly_map: bool = False, + confidence: Confidence = "best", **kwargs, ) -> ClassificationPrediction: return post_process_anomaly_scores( model_results=model_results, calibration=self._manifest.calibration, include_anomaly_map=include_anomaly_map, + confidence=confidence, + recommended_parameters=self.recommended_parameters, ) diff --git a/inference_models/inference_models/models/patchcore/patchcore_anomaly_detection_torch.py b/inference_models/inference_models/models/patchcore/patchcore_anomaly_detection_torch.py index 208f8ff3a8..5e3b63289d 100644 --- a/inference_models/inference_models/models/patchcore/patchcore_anomaly_detection_torch.py +++ b/inference_models/inference_models/models/patchcore/patchcore_anomaly_detection_torch.py @@ -8,7 +8,7 @@ from inference_models import ClassificationModel, ClassificationPrediction from inference_models.configuration import DEFAULT_DEVICE -from inference_models.entities import ColorFormat +from inference_models.entities import ColorFormat, Confidence from inference_models.errors import CorruptedModelPackageError from inference_models.models.common.anomaly_detection import ( ANOMALY_CLASS_NAMES, @@ -21,6 +21,7 @@ pre_process_anomaly_images, ) from inference_models.models.common.model_packages import get_model_package_contents +from inference_models.weights_providers.entities import RecommendedParameters # Settings of the PatchCore recipe used by Roboflow training # (https://github.com/amazon-science/patchcore-inspection). @@ -28,8 +29,10 @@ EMBEDDING_DIMENSION = 1024 MAP_SMOOTHING_SIGMA = 4.0 MAP_SMOOTHING_RADIUS = 16 # scipy.ndimage.gaussian_filter default: truncate=4.0 -# Bounds the (patches x memory bank) distance matrix held in memory at once. -NEAREST_NEIGHBOURS_CHUNK_SIZE = 4096 +# Bounds the (patches x memory bank) distance matrix held in memory at once, in +# elements (256 MiB in fp32). A 640px model has 6400 patches per image and a memory +# bank of ~10^5 entries, so a row count alone would let a single request take GiBs. +NEAREST_NEIGHBOURS_CHUNK_ELEMENTS = 2**26 class PatchCoreModel(nn.Module): @@ -97,12 +100,16 @@ def embed(self, images: torch.Tensor) -> Tuple[torch.Tensor, Tuple[int, int]]: def score_patches(self, embeddings: torch.Tensor) -> torch.Tensor: """Mean squared L2 distance to the nearest memory bank entries.""" scores = [] - for chunk in embeddings.split(NEAREST_NEIGHBOURS_CHUNK_SIZE): - distances = ( - (chunk**2).sum(dim=1, keepdim=True) - - 2 * chunk @ self.memory_bank.T - + self.memory_bank_squared_norms[None] - ).clamp(min=0) + chunk_size = max(1, NEAREST_NEIGHBOURS_CHUNK_ELEMENTS // len(self.memory_bank)) + for chunk in embeddings.split(chunk_size): + # In place, so the chunk's distance matrix is the only one held. + distances = torch.addmm( + self.memory_bank_squared_norms[None], + chunk, + self.memory_bank.T, + alpha=-2, + ) + distances.add_((chunk**2).sum(dim=1, keepdim=True)).clamp_(min=0) nearest = distances.topk(self.neighbors, dim=1, largest=False).values scores.append(nearest.mean(dim=1)) return torch.cat(scores) @@ -162,6 +169,7 @@ def from_pretrained( cls, model_name_or_path: str, device: torch.device = DEFAULT_DEVICE, + recommended_parameters: Optional[RecommendedParameters] = None, **kwargs, ) -> "PatchCoreForAnomalyDetectionTorch": model_package_content = get_model_package_contents( @@ -193,17 +201,24 @@ def from_pretrained( image_size=manifest.config.image_size, ) model.backbone.load_state_dict(state["backbone"]) - return cls(model=model.to(device).eval(), manifest=manifest, device=device) + return cls( + model=model.to(device).eval(), + manifest=manifest, + device=device, + recommended_parameters=recommended_parameters, + ) def __init__( self, model: PatchCoreModel, manifest: AnomalyModelManifest, device: torch.device, + recommended_parameters: Optional[RecommendedParameters] = None, ): self._model = model self._manifest = manifest self._device = device + self.recommended_parameters = recommended_parameters @property def class_names(self) -> List[str]: @@ -243,10 +258,13 @@ def post_process( self, model_results: AnomalyRawPrediction, include_anomaly_map: bool = False, + confidence: Confidence = "best", **kwargs, ) -> ClassificationPrediction: return post_process_anomaly_scores( model_results=model_results, calibration=self._manifest.calibration, include_anomaly_map=include_anomaly_map, + confidence=confidence, + recommended_parameters=self.recommended_parameters, ) diff --git a/inference_models/pyproject.toml b/inference_models/pyproject.toml index 661960563a..d33b0a8d71 100644 --- a/inference_models/pyproject.toml +++ b/inference_models/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "inference-models" -version = "0.39.5rc2" +version = "0.39.5" description = "The new inference engine for Computer Vision models" readme = "README.md" requires-python = ">=3.10,<3.14" diff --git a/inference_models/tests/unit_tests/models/common/test_anomaly_detection.py b/inference_models/tests/unit_tests/models/common/test_anomaly_detection.py index 6b748c0edc..91bc696cbd 100644 --- a/inference_models/tests/unit_tests/models/common/test_anomaly_detection.py +++ b/inference_models/tests/unit_tests/models/common/test_anomaly_detection.py @@ -17,6 +17,7 @@ post_process_anomaly_scores, pre_process_anomaly_images, ) +from inference_models.weights_providers.entities import RecommendedParameters IMAGE_SIZE = 32 @@ -149,7 +150,11 @@ def test_score_at_threshold_is_anomalous_with_confidence_one_half() -> None: calibration = AnomalyCalibration(threshold=2.0, scale=0.5) prediction = post_process_anomaly_scores( - raw_prediction([1.0, 2.0, 1e9]), calibration, include_anomaly_map=False + raw_prediction([1.0, 2.0, 1e9]), + calibration, + include_anomaly_map=False, + confidence="default", + recommended_parameters=None, ) assert prediction.class_id.tolist() == [0, 1, 1] @@ -163,14 +168,95 @@ def test_score_at_threshold_is_anomalous_with_confidence_one_half() -> None: ] assert prediction.images_metadata[0]["anomaly_score"] == 1.0 assert prediction.images_metadata[0]["anomaly_threshold"] == 2.0 + assert prediction.images_metadata[0]["anomaly_scale"] == 0.5 assert "anomaly_map" not in prediction.images_metadata[0] +@pytest.mark.parametrize( + "confidence, expected_is_anomalous", + [ + (0.5, [False, True, True]), + (0.25, [True, True, True]), + (0.75, [False, False, True]), + ], +) +def test_confidence_moves_the_decision_and_keeps_the_saved_threshold( + confidence: float, expected_is_anomalous: list +) -> None: + calibration = AnomalyCalibration(threshold=2.0, scale=0.5) + + # Anomalous confidences: sigmoid(-0.4) ~ 0.40, sigmoid(0) = 0.5, sigmoid(2) ~ 0.88. + prediction = post_process_anomaly_scores( + raw_prediction([1.8, 2.0, 3.0]), + calibration, + include_anomaly_map=False, + confidence=confidence, + recommended_parameters=None, + ) + + assert [m["is_anomalous"] for m in prediction.images_metadata] == ( + expected_is_anomalous + ) + assert prediction.class_id.tolist() == [int(v) for v in expected_is_anomalous] + assert {m["anomaly_threshold"] for m in prediction.images_metadata} == {2.0} + + +@pytest.mark.parametrize( + "recommended_parameters, expected_is_anomalous", + [ + (None, [False, True]), + (RecommendedParameters(confidence=0.7), [False, False]), + ( + RecommendedParameters( + confidence=0.7, per_class_confidence={"anomalous": 0.3} + ), + [True, True], + ), + ], +) +def test_best_confidence_follows_model_eval_recommendation( + recommended_parameters, expected_is_anomalous: list +) -> None: + calibration = AnomalyCalibration(threshold=2.0, scale=0.5) + + prediction = post_process_anomaly_scores( + raw_prediction([1.8, 2.0]), + calibration, + include_anomaly_map=False, + confidence="best", + recommended_parameters=recommended_parameters, + ) + + assert [m["is_anomalous"] for m in prediction.images_metadata] == ( + expected_is_anomalous + ) + + +@pytest.mark.parametrize("confidence", [-0.1, 1.5, True, "high"]) +def test_invalid_confidence_is_rejected(confidence) -> None: + calibration = AnomalyCalibration(threshold=2.0, scale=0.5) + + with pytest.raises(ModelInputError, match="confidence"): + post_process_anomaly_scores( + raw_prediction([1.0]), + calibration, + include_anomaly_map=False, + confidence=confidence, + recommended_parameters=None, + ) + + def test_anomaly_map_is_returned_at_network_resolution() -> None: calibration = AnomalyCalibration(threshold=2.0, scale=0.5) raw = raw_prediction([1.0], size_hw=(10, 20)) - prediction = post_process_anomaly_scores(raw, calibration, include_anomaly_map=True) + prediction = post_process_anomaly_scores( + raw, + calibration, + include_anomaly_map=True, + confidence="default", + recommended_parameters=None, + ) anomaly_map = prediction.images_metadata[0]["anomaly_map"] # The map keeps the network's square geometry whatever the image size was. @@ -184,5 +270,9 @@ def test_non_finite_scores_are_rejected() -> None: with pytest.raises(CorruptedModelPackageError, match="non-finite"): post_process_anomaly_scores( - raw_prediction([float("nan")]), calibration, include_anomaly_map=False + raw_prediction([float("nan")]), + calibration, + include_anomaly_map=False, + confidence="default", + recommended_parameters=None, ) diff --git a/inference_models/tests/unit_tests/models/test_patchcore_torch.py b/inference_models/tests/unit_tests/models/test_patchcore_torch.py index d09d5b5a68..57fecadb31 100644 --- a/inference_models/tests/unit_tests/models/test_patchcore_torch.py +++ b/inference_models/tests/unit_tests/models/test_patchcore_torch.py @@ -87,9 +87,18 @@ def save_package( return str(package_dir) +@pytest.mark.parametrize( + "chunk_elements", + [patchcore_anomaly_detection_torch.NEAREST_NEIGHBOURS_CHUNK_ELEMENTS, 120, 10], +) def test_patch_scores_are_mean_squared_distances_to_nearest_memory_entries( - tiny_backbone: None, + tiny_backbone: None, monkeypatch: pytest.MonkeyPatch, chunk_elements: int ) -> None: + monkeypatch.setattr( + patchcore_anomaly_detection_torch, + "NEAREST_NEIGHBOURS_CHUNK_ELEMENTS", + chunk_elements, + ) torch.manual_seed(0) memory_bank = torch.randn(50, EMBEDDING_DIMENSION) embeddings = torch.randn(7, EMBEDDING_DIMENSION) diff --git a/inference_models/uv.lock b/inference_models/uv.lock index 88965d7af1..598384852e 100644 --- a/inference_models/uv.lock +++ b/inference_models/uv.lock @@ -38,6 +38,8 @@ conflicts = [[ { package = "inference-models", extra = "onnx-jp6-cu126" }, ]] +[manifest] + [[manifest.dependency-metadata]] name = "nvidia-cuda-runtime-cu12" @@ -1349,7 +1351,7 @@ wheels = [ [[package]] name = "inference-models" -version = "0.39.5rc2" +version = "0.39.5" source = { virtual = "." } dependencies = [ { name = "accelerate" }, @@ -4524,6 +4526,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] +[[package]] +name = "pkgconfig" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/fd/0adde075cd3bfecd557bc7d757e00e231d34d8a6edb4c8d1642759254c21/pkgconfig-1.6.0.tar.gz", hash = "sha256:4a5a6631ce937fafac457104a40d558785a658bbdca5c49b6295bc3fd651907f", size = 5691, upload-time = "2026-03-06T11:26:01.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/6f/f7ec07fba48f07c555cc4099481df644fbbc12067879072c17ac229f6556/pkgconfig-1.6.0-py3-none-any.whl", hash = "sha256:98e71754855e9563838d952a160eb577edabb57782e49853edb5381927e6bea1", size = 7086, upload-time = "2026-03-06T11:26:07.688Z" }, +] + [[package]] name = "platformdirs" version = "4.3.8" @@ -5105,6 +5116,7 @@ version = "2.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, + { name = "pkgconfig" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7b/88/f73dae807ec68b228fba72507105e3ba80a561dc0bade0004ce24fd118fc/pyvips-2.2.3.tar.gz", hash = "sha256:43bceced0db492654c93008246a58a508e0373ae1621116b87b322f2ac72212f", size = 56626, upload-time = "2024-04-28T11:19:58.158Z" } diff --git a/inference_server/inference_server/handlers/classification/introspection.py b/inference_server/inference_server/handlers/classification/introspection.py index c008e5dcf3..ce59786aac 100644 --- a/inference_server/inference_server/handlers/classification/introspection.py +++ b/inference_server/inference_server/handlers/classification/introspection.py @@ -1,6 +1,11 @@ from __future__ import annotations -from inference_model_manager.registry_defaults import _K_PREPROC, _P_IMAGES, _p +from inference_model_manager.registry_defaults import ( + _K_PREPROC, + _P_CONFIDENCE, + _P_IMAGES, + _p, +) from inference_server.framework.entities import ModelInterfaceDescription @@ -10,6 +15,7 @@ def get_classification_interface() -> ModelInterfaceDescription: params=_p( _P_IMAGES, {"include_anomaly_map": {"type": "bool", "required": False}}, + _P_CONFIDENCE, _K_PREPROC, ), output_schema={ diff --git a/inference_server/inference_server/legacy/entities.py b/inference_server/inference_server/legacy/entities.py index c396c3254d..4e7eebfb49 100644 --- a/inference_server/inference_server/legacy/entities.py +++ b/inference_server/inference_server/legacy/entities.py @@ -1678,7 +1678,8 @@ class AnomalyDetectionResponse(ClassificationInferenceResponse): Attributes: anomaly_score (float): Raw anomaly score; larger means more anomalous. anomaly_threshold (float): Decision threshold fitted on validation images. - is_anomalous (bool): Whether the score reached the threshold. + anomaly_scale (float): Calibration scale of the anomalous confidence. + is_anomalous (bool): Whether the anomalous confidence reached the request confidence. anomaly_map (Optional[List[List[float]]]): Raw local anomaly evidence in original image coordinates. """ @@ -1688,7 +1689,13 @@ class AnomalyDetectionResponse(ClassificationInferenceResponse): anomaly_threshold: float = Field( description="Decision threshold fitted on validation images" ) - is_anomalous: bool = Field(description="Whether the score reached the threshold") + anomaly_scale: float = Field( + description="Calibration scale: the anomalous confidence is " + "sigmoid((anomaly_score - anomaly_threshold) / anomaly_scale)" + ) + is_anomalous: bool = Field( + description="Whether the anomalous confidence reached the request confidence" + ) anomaly_map: Optional[List[List[float]]] = Field( default=None, description="Raw local anomaly evidence in original image coordinates", diff --git a/inference_server/inference_server/legacy/router.py b/inference_server/inference_server/legacy/router.py index a168eb2159..b1cab49c3d 100644 --- a/inference_server/inference_server/legacy/router.py +++ b/inference_server/inference_server/legacy/router.py @@ -21,8 +21,15 @@ from inference_model_manager.registry_defaults import IMAGE_EMBEDDINGS_ACTION from inference_sdk.http.utils.aliases import resolve_roboflow_model_alias -from inference_server import configuration, platform_http, server_identity, telemetry +from inference_server import ( + configuration, + pingback, + platform_http, + server_identity, + telemetry, +) from inference_server.dependencies import get_model_manager +from inference_server.hosted.common import service_secret_is_valid from inference_server.legacy.action_recognition import ( ACTION_RECOGNITION_TASK, classify_video, @@ -145,8 +152,6 @@ encode_image_to_jpeg_bytes, render_visualization, ) -from inference_server import pingback -from inference_server.hosted.common import service_secret_is_valid from inference_server.prometheus import measure_inference from inference_server.routing import ( IMAGE_EMBEDDINGS, @@ -1132,12 +1137,15 @@ async def legacy_infer_from_request( extra_args = {"keypoint_confidence": keypoint_confidence} elif route.task_type == "semantic-segmentation": request_type = SemanticSegmentationInferenceRequest + # Leave an omitted confidence unset so anomaly detection models can tell it + # apart from an explicit one and use the model-eval value. + if route.task_type != "classification" or "confidence" in request.query_params: + extra_args["confidence"] = confidence try: inference_request = request_type( api_key=resolved_key, model_id=model_id, image=request_image, - confidence=confidence, iou_threshold=overlap, max_detections=max_detections, visualization_labels=labels, diff --git a/inference_server/inference_server/legacy/translation.py b/inference_server/inference_server/legacy/translation.py index a342c09505..495a13cec8 100644 --- a/inference_server/inference_server/legacy/translation.py +++ b/inference_server/inference_server/legacy/translation.py @@ -110,7 +110,13 @@ def roboflow_confidence(value: Any) -> Optional[Union[float, str]]: def build_task_params(task_type: str, action: str, request: Any, route: Route) -> dict: params: dict = {} confidence = roboflow_confidence(getattr(request, "confidence", None)) - if confidence is not None: + # Single-label classifiers ignore confidence in the model; anomaly detection + # models decide with it, and use the model-eval value when the caller sent none + # (the request field defaults to 0.4). + caller_sent_confidence = task_type != "classification" or ( + "confidence" in getattr(request, "model_fields_set", ()) + ) + if confidence is not None and caller_sent_confidence: params["confidence"] = confidence for field in _DISABLE_PREPROC_FIELDS: if getattr(request, field, False): @@ -446,6 +452,7 @@ def _repack_anomaly_detection_response( confidence=class_predictions[0]["confidence"], anomaly_score=metadata["anomaly_score"], anomaly_threshold=metadata["anomaly_threshold"], + anomaly_scale=metadata["anomaly_scale"], is_anomalous=metadata["is_anomalous"], anomaly_map=( np.asarray(anomaly_map).tolist() if anomaly_map is not None else None diff --git a/inference_server/inference_server/workflows/models_provider.py b/inference_server/inference_server/workflows/models_provider.py index f3d076b2e3..baf23c2e33 100644 --- a/inference_server/inference_server/workflows/models_provider.py +++ b/inference_server/inference_server/workflows/models_provider.py @@ -8,7 +8,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union import torch -from inference_sdk.http.utils.aliases import resolve_roboflow_model_alias from roboflow_workflows import environment from roboflow_workflows.prototypes.models_provider import ( UNSET, @@ -27,6 +26,7 @@ get_async_response_future, ) from inference_models.utils import model_blob_cache +from inference_sdk.http.utils.aliases import resolve_roboflow_model_alias from inference_server import pingback, telemetry from inference_server.gateway import _load_failure from inference_server.legacy.action_recognition import ( @@ -323,7 +323,11 @@ def run_classification( disable_active_learning: Optional[bool] = None, active_learning_target_dataset: Optional[str] = None, inference_kwargs: Optional[Dict[str, Any]] = None, + include_anomaly_map: bool = False, ) -> List[dict]: + # Set only when asked, so the request a classification block builds is + # field-for-field what it was before the anomaly block existed. + anomaly_arguments = {"include_anomaly_map": True} if include_anomaly_map else {} request = ClassificationInferenceRequest( api_key=api_key, model_id=model_id, @@ -332,6 +336,7 @@ def run_classification( source=_WORKFLOW_SOURCE, active_learning_target_dataset=active_learning_target_dataset, confidence=confidence, + **anomaly_arguments, ) return self._dump( self._run_cv(model_id, request, api_key, extra_params=inference_kwargs) diff --git a/inference_server/inference_server/workflows/tensor_native.py b/inference_server/inference_server/workflows/tensor_native.py index 331bb4221a..0458b3c55d 100644 --- a/inference_server/inference_server/workflows/tensor_native.py +++ b/inference_server/inference_server/workflows/tensor_native.py @@ -48,7 +48,11 @@ "object-detection": _DETECTION_PARAMS | _COLOR_FORMAT_PARAM, "instance-segmentation": _DETECTION_PARAMS | _COLOR_FORMAT_PARAM, "keypoint-detection": _DETECTION_PARAMS | _COLOR_FORMAT_PARAM, - "classification": _DETECTION_PARAMS | _COLOR_FORMAT_PARAM, + # Anomaly detection models (PatchCore, FoundAD) are served as classification; + # the flag asks them for the anomaly map, as on the legacy request path. + "classification": _DETECTION_PARAMS + | _COLOR_FORMAT_PARAM + | frozenset({"include_anomaly_map"}), "multi-label-classification": _DETECTION_PARAMS | _COLOR_FORMAT_PARAM, "semantic-segmentation": frozenset({"confidence"}) | _COLOR_FORMAT_PARAM, "depth-estimation": _COLOR_FORMAT_PARAM, @@ -84,12 +88,20 @@ } +# Prediction fields that stay host-side: a model fills them with numpy arrays +# (e.g. the anomaly map in `ClassificationPrediction.images_metadata`), and the +# tensor blocks read them as numpy, exactly as they do with an in-process model. +_HOST_SIDE_FIELDS = frozenset({"images_metadata"}) + + # NOTE: bfloat16 tensors were widened to float32 on the wire; the original precision is not restored. def numpy_to_tensors(result: Any, device: Any) -> Any: if isinstance(result, np.ndarray): return torch.as_tensor(result, device=device) if dataclasses.is_dataclass(result) and not isinstance(result, type): for field in dataclasses.fields(result): + if field.name in _HOST_SIDE_FIELDS: + continue object.__setattr__( result, field.name, diff --git a/inference_server/tests/unit_tests/legacy/test_translation_classification.py b/inference_server/tests/unit_tests/legacy/test_translation_classification.py index 35bad6cc1a..eddfae7485 100644 --- a/inference_server/tests/unit_tests/legacy/test_translation_classification.py +++ b/inference_server/tests/unit_tests/legacy/test_translation_classification.py @@ -5,9 +5,9 @@ import numpy as np import pytest import torch -from inference_models.models.base.classification import ClassificationPrediction from PIL import Image +from inference_models.models.base.classification import ClassificationPrediction from inference_server.legacy.bridge import Route from inference_server.legacy.entities import ( AnomalyDetectionResponse, @@ -38,6 +38,7 @@ def test_anomaly_metadata_repacks_as_anomaly_detection_response(): { "anomaly_score": 1.5, "anomaly_threshold": 1.0, + "anomaly_scale": 0.5, "is_anomalous": True, "anomaly_map": np.array([[0.1, 0.2], [0.3, 0.4]], dtype=np.float32), } @@ -57,6 +58,7 @@ def test_anomaly_metadata_repacks_as_anomaly_detection_response(): assert [p["class"] for p in d["predictions"]] == ["anomalous", "normal"] assert d["anomaly_score"] == 1.5 assert d["anomaly_threshold"] == 1.0 + assert d["anomaly_scale"] == 0.5 assert d["is_anomalous"] is True assert d["anomaly_map"] == [ [pytest.approx(0.1), pytest.approx(0.2)], @@ -69,7 +71,12 @@ def test_anomaly_metadata_without_map_leaves_anomaly_map_out(): class_id=torch.tensor([0]), confidence=torch.tensor([[0.7, 0.3]]), images_metadata=[ - {"anomaly_score": 0.5, "anomaly_threshold": 1.0, "is_anomalous": False} + { + "anomaly_score": 0.5, + "anomaly_threshold": 1.0, + "anomaly_scale": 0.5, + "is_anomalous": False, + } ], ) resp = repack_prediction( @@ -114,6 +121,20 @@ def test_build_task_params_forwards_include_anomaly_map_for_classification(): } +def test_build_task_params_leaves_an_omitted_classification_confidence_unset(): + # The request defaults confidence to 0.4; an anomaly model must not decide at + # it, so only a confidence the caller sent reaches the model. + request = ClassificationInferenceRequest(model_id="ds/1", image=IMG) + route = _anomaly_route() + + assert "confidence" not in build_task_params( + "classification", "infer", request, route + ) + assert build_task_params("multi-label-classification", "infer", request, route) == { + "confidence": 0.4 + } + + def test_build_task_params_forwards_disable_preproc_flags_for_classification(): request = ClassificationInferenceRequest( model_id="ds/1", image=IMG, disable_preproc_grayscale=True diff --git a/inference_server/tests/unit_tests/test_handler_classification.py b/inference_server/tests/unit_tests/test_handler_classification.py index 59cca2151a..71e4f01742 100644 --- a/inference_server/tests/unit_tests/test_handler_classification.py +++ b/inference_server/tests/unit_tests/test_handler_classification.py @@ -27,6 +27,8 @@ def _anomaly_prediction(include_anomaly_map: bool) -> ClassificationPrediction: raw, calibration=AnomalyCalibration(threshold=2.0, scale=1.0), include_anomaly_map=include_anomaly_map, + confidence="default", + recommended_parameters=None, ) return prediction diff --git a/inference_server/tests/unit_tests/workflows/test_models_provider.py b/inference_server/tests/unit_tests/workflows/test_models_provider.py index a9a47a1084..1676ba12fe 100644 --- a/inference_server/tests/unit_tests/workflows/test_models_provider.py +++ b/inference_server/tests/unit_tests/workflows/test_models_provider.py @@ -13,7 +13,6 @@ ) from inference_models.errors import ModelInputError, ModelNotFoundError - from inference_server.gateway import ModelManagerGateway from inference_server.legacy.bridge import ( LegacyModelBridge, @@ -351,6 +350,29 @@ def test_classification_inference_kwargs_override_action_params(): assert b.calls[0][2]["confidence"] == 0.9 +def test_classification_forwards_anomaly_map_flag_and_confidence_to_the_model(): + b = FakeSyncBridge() + b.routes["cls/1"] = Route( + model_id="cls/1", + registry_id="cls/1", + task_type="classification", + action="infer", + actions={"infer"}, + class_names=["cat"], + ) + b.predictions[("cls/1", "infer")] = SimpleNamespace( + confidence=np.array([0.9]), class_id=np.array([0]) + ) + GatewayModelsProvider(b, api_key=None).run_classification( + "cls/1", + [{"type": "numpy_object", "value": np.zeros((4, 6, 3), np.uint8)}], + confidence=0.7, + include_anomaly_map=True, + ) + assert b.calls[0][2]["include_anomaly_map"] is True + assert b.calls[0][2]["confidence"] == 0.7 + + def test_get_class_names_and_contains(): p = GatewayModelsProvider(_od_bridge(), api_key=None) assert p.get_class_names("ds/1") == ["cat"] and "ds/1" in p and "x/1" not in p @@ -1238,12 +1260,11 @@ def __init__(self, pipeline): self.unloaded = False def infer(self, route, api_key, action, images, params, record=True): + from inference_model_manager.model_manager import ModelManager from inference_models.models.base.async_handoff import ( get_async_response_context_id, ) - from inference_model_manager.model_manager import ModelManager - self.calls.append( (route.model_id, action, dict(params), [i.data for i in images]) ) diff --git a/inference_server/tests/unit_tests/workflows/test_tensor_native.py b/inference_server/tests/unit_tests/workflows/test_tensor_native.py index 1b1e982aae..1e5c97ab91 100644 --- a/inference_server/tests/unit_tests/workflows/test_tensor_native.py +++ b/inference_server/tests/unit_tests/workflows/test_tensor_native.py @@ -121,6 +121,83 @@ def test_classification_is_one_batched_prediction(): ) +def test_native_params_forwards_include_anomaly_map_for_classification(): + params = native_params( + "classification", + {"confidence": 0.5, "include_anomaly_map": True, "input_color_format": "bgr"}, + ) + assert params == { + "confidence": 0.5, + "include_anomaly_map": True, + "input_color_format": "bgr", + } + + +def test_native_params_drops_include_anomaly_map_for_other_task_types(): + params = native_params( + "object-detection", + {"confidence": 0.5, "include_anomaly_map": True, "input_color_format": "bgr"}, + ) + assert "include_anomaly_map" not in params + + +def test_numpy_to_tensors_keeps_images_metadata_host_side(): + from inference_models.models.base.classification import ClassificationPrediction + + anomaly_map = np.zeros((4, 4), dtype=np.float32) + restored = numpy_to_tensors( + ClassificationPrediction( + class_id=np.array([1]), + confidence=np.array([[0.2, 0.8]], dtype=np.float32), + images_metadata=[{"anomaly_score": 1.5, "anomaly_map": anomaly_map}], + ), + device=None, + ) + assert isinstance(restored.class_id, torch.Tensor) + assert isinstance(restored.confidence, torch.Tensor) + assert restored.images_metadata[0]["anomaly_map"] is anomaly_map + + +def test_classification_forwards_anomaly_map_flag_and_keeps_the_map_as_numpy(): + from inference_models.models.base.classification import ClassificationPrediction + + bridge = FakeSyncBridge() + bridge.routes["anomaly/1"] = Route( + model_id="anomaly/1", + registry_id="anomaly/1", + task_type="classification", + action="infer", + actions={"infer"}, + class_names=["normal", "anomalous"], + ) + anomaly_map = np.ones((8, 8), dtype=np.float32) + bridge.predictions[("anomaly/1", "infer")] = ClassificationPrediction( + class_id=np.array([1]), + confidence=np.array([[0.3, 0.7]], dtype=np.float32), + images_metadata=[ + { + "anomaly_score": 2.0, + "anomaly_threshold": 1.0, + "anomaly_scale": 0.5, + "is_anomalous": True, + "anomaly_map": anomaly_map, + } + ], + ) + out = GatewayModelsProvider(bridge, api_key="k").run_tensor_native_inference( + model_id="anomaly/1", + images=[np.zeros((2, 2, 3), np.uint8)], + input_color_format="bgr", + include_anomaly_map=True, + confidence=0.5, + ) + assert bridge.calls[0][2]["include_anomaly_map"] is True + assert bridge.calls[0][2]["confidence"] == 0.5 + assert isinstance(out, ClassificationPrediction) + assert isinstance(out.class_id, torch.Tensor) + assert out.images_metadata[0]["anomaly_map"] is anomaly_map + + def test_multi_label_classification_is_one_prediction_per_image(): from inference_models.models.base.classification import ( MultiLabelClassificationPrediction, diff --git a/requirements/requirements.cpu.txt b/requirements/requirements.cpu.txt index 4f53f93faa..5e311683ea 100644 --- a/requirements/requirements.cpu.txt +++ b/requirements/requirements.cpu.txt @@ -1,3 +1,3 @@ onnxruntime>=1.15.1,<1.22.0 nvidia-ml-py<13.0.0 -inference-models[torch-cpu,onnx-cpu]==0.39.5rc2 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu,onnx-cpu]==0.39.5 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.gpu.cu13.txt b/requirements/requirements.gpu.cu13.txt index d8422cf1d4..b87d626190 100644 --- a/requirements/requirements.gpu.cu13.txt +++ b/requirements/requirements.gpu.cu13.txt @@ -4,6 +4,6 @@ # is a CUDA 12 build), so the Dockerfile installs it in a dedicated step from # Microsoft's onnxruntime-cuda-13 feed. A plain pin here would silently # resolve to the CUDA 12 wheel. -inference-models[torch-cu130]==0.39.5rc2 +inference-models[torch-cu130]==0.39.5 pynvvideocodec~=2.1.0; platform_machine == "x86_64" onnxruntime-gpu>=1.27.0,<1.30.0 diff --git a/requirements/requirements.gpu.txt b/requirements/requirements.gpu.txt index 9763e4d5ea..bd7802f176 100644 --- a/requirements/requirements.gpu.txt +++ b/requirements/requirements.gpu.txt @@ -1,3 +1,3 @@ onnxruntime-gpu>=1.15.1,<1.22.0 -inference-models[torch-cu124,onnx-cu12]==0.39.5rc2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cu124,onnx-cu12]==0.39.5 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt pynvvideocodec>=2.1.0,<3.0.0; platform_machine == "x86_64" diff --git a/requirements/requirements.jetson.txt b/requirements/requirements.jetson.txt index 912879d821..963e7f13f8 100644 --- a/requirements/requirements.jetson.txt +++ b/requirements/requirements.jetson.txt @@ -1,3 +1,3 @@ pypdfium2>=4.11.0,<5.0.0 PyYAML~=6.0.0 -inference-models==0.39.5rc2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models==0.39.5 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.vino.txt b/requirements/requirements.vino.txt index 791da50a3d..2eda4e6873 100644 --- a/requirements/requirements.vino.txt +++ b/requirements/requirements.vino.txt @@ -1,2 +1,2 @@ onnxruntime-openvino>=1.15.0,<1.22.0 -inference-models[torch-cpu]==0.39.5rc2 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu]==0.39.5 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.workflows.txt b/requirements/requirements.workflows.txt index 97d0fa5719..31fe3cf4fd 100644 --- a/requirements/requirements.workflows.txt +++ b/requirements/requirements.workflows.txt @@ -1,2 +1,2 @@ # keep in sync with workflows/pyproject.toml version -roboflow-workflows[enterprise]==0.2.11rc1 +roboflow-workflows[enterprise]==0.2.11 diff --git a/tests/inference/hosted_platform_tests/conftest.py b/tests/inference/hosted_platform_tests/conftest.py index 99ae0b640e..c5638b1301 100644 --- a/tests/inference/hosted_platform_tests/conftest.py +++ b/tests/inference/hosted_platform_tests/conftest.py @@ -151,6 +151,8 @@ class PlatformEnvironment(Enum): "yolov8n-pose-640": "yolov8n-pose-640", "rfdetr-od": "rfdetr-nano", "rfdetr-is": "rfdetr-seg-nano", + "anomaly-foundad": "paul-guerrie-tang1/anomaly-det-algbq-1-foundad-t1", + "anomaly-patchcore": "paul-guerrie-tang1/anomaly-det-algbq-1-patchcore-t2", }, PlatformEnvironment.ROBOFLOW_STAGING_LAMBDA: { "object-detection": "eye-detection/35", @@ -161,6 +163,8 @@ class PlatformEnvironment(Enum): "yolov8n-pose-640": "microsoft-coco-pose/1", "rfdetr-od": "rfdetr-nano", "rfdetr-is": "asl-instance-seg/160", + "anomaly-foundad": "paul-guerrie/anomaly-det-1-foundad-t1", + "anomaly-patchcore": "paul-guerrie/anomaly-det-1-patchcore-t2", }, } MODELS_TO_BE_USED[PlatformEnvironment.ROBOFLOW_STAGING_SERVERLESS] = MODELS_TO_BE_USED[ @@ -261,6 +265,13 @@ def multi_class_classification_model_id( return MODELS_TO_BE_USED[platform_environment]["multi_class_classification"] +@pytest.fixture(scope="session", params=["anomaly-foundad", "anomaly-patchcore"]) +def anomaly_detection_model_id( + platform_environment: PlatformEnvironment, request: pytest.FixtureRequest +) -> str: + return MODELS_TO_BE_USED[platform_environment][request.param] + + @pytest.fixture(scope="session") def detection_model_id(platform_environment: PlatformEnvironment) -> str: return MODELS_TO_BE_USED[platform_environment]["object-detection"] diff --git a/tests/inference/hosted_platform_tests/test_anomaly_detection_models.py b/tests/inference/hosted_platform_tests/test_anomaly_detection_models.py new file mode 100644 index 0000000000..ea10d7898c --- /dev/null +++ b/tests/inference/hosted_platform_tests/test_anomaly_detection_models.py @@ -0,0 +1,123 @@ +import math +import os + +import cv2 +import numpy as np +import pytest + +from inference_sdk import InferenceConfiguration, InferenceHTTPClient +from tests.inference.hosted_platform_tests.conftest import ( + ROBOFLOW_API_KEY, + apply_auth_mode, +) + +# Images of the anomaly reference project, far from the threshold on every model. +ASSETS_DIR = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "..", + "..", + "workflows", + "integration_tests", + "execution", + "assets", + ) +) +ANOMALOUS_IMAGE = cv2.imread(os.path.join(ASSETS_DIR, "anomaly_anomalous.jpg")) +NORMAL_IMAGE = cv2.imread(os.path.join(ASSETS_DIR, "anomaly_normal.jpg")) + + +def _client(service_url: str, auth_mode: str, api_v0: bool, **configuration): + client = InferenceHTTPClient(api_url=service_url, api_key=ROBOFLOW_API_KEY) + if api_v0: + client.select_api_v0() + client.configure(InferenceConfiguration(**configuration)) + return apply_auth_mode(client, auth_mode) + + +def _anomalous_confidence(response: dict) -> float: + margin = (response["anomaly_score"] - response["anomaly_threshold"]) / response[ + "anomaly_scale" + ] + return 1 / (1 + math.exp(-margin)) + + +@pytest.mark.flaky(retries=4, delay=1) +@pytest.mark.parametrize("api_v0", [True, False], ids=["v0", "v1"]) +def test_anomaly_model_decides_at_the_calibrated_threshold_by_default( + classification_service_url: str, + anomaly_detection_model_id: str, + auth_mode: str, + api_v0: bool, +) -> None: + # given + client = _client(classification_service_url, auth_mode, api_v0) + + # when + anomalous, normal = ( + client.infer(image, model_id=anomaly_detection_model_id) + for image in (ANOMALOUS_IMAGE, NORMAL_IMAGE) + ) + + # then + assert anomalous["is_anomalous"] is True and anomalous["top"] == "anomalous" + assert normal["is_anomalous"] is False and normal["top"] == "normal" + for response in (anomalous, normal): + assert response["anomaly_scale"] > 0 + assert "anomaly_map" not in response or response["anomaly_map"] is None + anomalous_confidence = _anomalous_confidence(response) + served = {p["class"]: p["confidence"] for p in response["predictions"]} + assert served["anomalous"] == pytest.approx(anomalous_confidence, abs=1e-4) + assert response["is_anomalous"] is (anomalous_confidence >= 0.5) + + +@pytest.mark.flaky(retries=4, delay=1) +@pytest.mark.parametrize("api_v0", [True, False], ids=["v0", "v1"]) +def test_anomaly_model_verdict_follows_the_request_confidence( + classification_service_url: str, + anomaly_detection_model_id: str, + auth_mode: str, + api_v0: bool, +) -> None: + # given + default_response = _client(classification_service_url, auth_mode, api_v0).infer( + NORMAL_IMAGE, model_id=anomaly_detection_model_id + ) + anomalous_confidence = _anomalous_confidence(default_response) + below = _client( + classification_service_url, + auth_mode, + api_v0, + confidence_threshold=round(anomalous_confidence / 2, 4), + ) + + # when + response = below.infer(NORMAL_IMAGE, model_id=anomaly_detection_model_id) + + # then: the threshold in effect never moves, only the verdict does + assert default_response["is_anomalous"] is False + assert response["is_anomalous"] is True and response["top"] == "anomalous" + assert response["anomaly_threshold"] == default_response["anomaly_threshold"] + assert response["anomaly_score"] == default_response["anomaly_score"] + + +@pytest.mark.flaky(retries=4, delay=1) +def test_anomaly_model_serves_the_map_at_network_resolution_on_request( + classification_service_url: str, + anomaly_detection_model_id: str, + auth_mode: str, +) -> None: + # given + client = _client( + classification_service_url, auth_mode, False, include_anomaly_map=True + ) + + # when + response = client.infer(ANOMALOUS_IMAGE, model_id=anomaly_detection_model_id) + + # then + anomaly_map = response["anomaly_map"] + assert isinstance(anomaly_map, np.ndarray) + assert anomaly_map.dtype == np.float32 + assert anomaly_map.ndim == 2 and anomaly_map.shape[0] == anomaly_map.shape[1] + assert float(anomaly_map.max()) > 0 diff --git a/tests/inference/unit_tests/core/interfaces/http/test_legacy_http_route_forwards_anomaly_map.py b/tests/inference/unit_tests/core/interfaces/http/test_legacy_http_route_forwards_anomaly_map.py index 2cfd6fa2f5..7104d82a3e 100644 --- a/tests/inference/unit_tests/core/interfaces/http/test_legacy_http_route_forwards_anomaly_map.py +++ b/tests/inference/unit_tests/core/interfaces/http/test_legacy_http_route_forwards_anomaly_map.py @@ -25,3 +25,31 @@ def test_legacy_classification_route_forwards_include_anomaly_map( assert response.status_code == 200, response.text inference_request = model_manager.infer_from_request_sync.call_args.args[1] assert inference_request.include_anomaly_map is include_anomaly_map + + +def test_legacy_classification_route_leaves_omitted_confidence_unset( + monkeypatch, +) -> None: + interface, model_manager = _build_interface(monkeypatch, task_type="classification") + + with TestClient(interface.app) as client: + omitted = client.post( + "/dummy-dataset/1", + params={"api_key": "query-api-key", "image": "https://example.com/a.jpg"}, + ) + omitted_request = model_manager.infer_from_request_sync.call_args.args[1] + explicit = client.post( + "/dummy-dataset/1", + params={ + "api_key": "query-api-key", + "image": "https://example.com/a.jpg", + "confidence": "0.4", + }, + ) + explicit_request = model_manager.infer_from_request_sync.call_args.args[1] + + assert omitted.status_code == 200, omitted.text + assert explicit.status_code == 200, explicit.text + assert "confidence" not in omitted_request.model_fields_set + assert "confidence" in explicit_request.model_fields_set + assert explicit_request.confidence == 0.4 diff --git a/tests/inference/unit_tests/core/interfaces/test_workflows_models_provider.py b/tests/inference/unit_tests/core/interfaces/test_workflows_models_provider.py index 2a24f2dd08..e80d400a32 100644 --- a/tests/inference/unit_tests/core/interfaces/test_workflows_models_provider.py +++ b/tests/inference/unit_tests/core/interfaces/test_workflows_models_provider.py @@ -426,6 +426,41 @@ def test_run_semantic_segmentation_still_rejects_an_explicit_none_confidence() - manager.infer_from_request_sync.assert_not_called() +def test_run_classification_requests_the_anomaly_map_when_asked() -> None: + """The anomaly detection block asks for the heatmap through the port; the + request field defaults to False so plain classification is unchanged.""" + from inference.core.entities.responses.inference import ( + ClassificationInferenceResponse, + ) + + manager = manager_returning( + ClassificationInferenceResponse( + image=InferenceResponseImage(width=10, height=20), + predictions=[], + top="normal", + confidence=0.9, + ) + ) + ModelManagerModelsProvider(manager).run_classification( + model_id="m/1", + images=IMAGES, + api_key="k", + confidence="default", + include_anomaly_map=True, + ) + request = captured_request(manager) + assert request.include_anomaly_map is True + assert "include_anomaly_map" in request.model_fields_set + + manager.infer_from_request_sync.reset_mock() + ModelManagerModelsProvider(manager).run_classification( + model_id="m/1", images=IMAGES, api_key="k", confidence="default" + ) + request = captured_request(manager) + assert request.include_anomaly_map is False + assert "include_anomaly_map" not in request.model_fields_set + + def test_run_classification_forwards_extra_inference_kwargs() -> None: """multi_label v2/v3 pass `confidence` to the model call as well as into the request; the port carries it as `inference_kwargs`.""" diff --git a/tests/inference/unit_tests/models/test_anomaly_detection.py b/tests/inference/unit_tests/models/test_anomaly_detection.py index a3d57291a5..d4e4a7f102 100644 --- a/tests/inference/unit_tests/models/test_anomaly_detection.py +++ b/tests/inference/unit_tests/models/test_anomaly_detection.py @@ -30,6 +30,7 @@ def prediction(anomalous_confidence=0.5, is_anomalous=True, anomaly_map=None): metadata = { "anomaly_score": 2.0, "anomaly_threshold": 2.0, + "anomaly_scale": 0.5, "is_anomalous": is_anomalous, } if anomaly_map is not None: @@ -94,6 +95,7 @@ def predict(): assert body["anomaly_score"] == 2.0 assert body["is_anomalous"] is True assert body["anomaly_threshold"] == 2.0 + assert body["anomaly_scale"] == 0.5 assert body["anomaly_map"]["shape"] == [10, 20] assert body["anomaly_map"]["dtype"] == "float32" # The map is served at network resolution, never at the image size. @@ -102,17 +104,24 @@ def predict(): ) -def test_request_and_sdk_forward_optional_map_to_the_model(): +def anomaly_adapter() -> InferenceModelsAnomalyDetectionAdapter: model = object.__new__(InferenceModelsAnomalyDetectionAdapter) model.class_names = CLASS_NAMES model._model = Mock(resolved_model=None) model._model.post_process.return_value = prediction() + return model + + +def png_image() -> dict: image = BytesIO() Image.new("RGB", (20, 10)).save(image, format="PNG") + return {"type": "base64", "value": base64.b64encode(image.getvalue()).decode()} + + +def test_request_and_sdk_forward_optional_map_to_the_model(): + model = anomaly_adapter() request = ClassificationInferenceRequest( - model_id="defects/1", - image={"type": "base64", "value": base64.b64encode(image.getvalue()).decode()}, - include_anomaly_map=True, + model_id="defects/1", image=png_image(), include_anomaly_map=True ) response = model.infer_from_request(request) assert response.top == "anomalous" @@ -121,7 +130,10 @@ def test_request_and_sdk_forward_optional_map_to_the_model(): bgr_images = model._model.pre_process.call_args.args[0] assert bgr_images[0].shape == (10, 20, 3) assert model._model.pre_process.call_args.kwargs["input_color_format"] == "bgr" - assert model._model.post_process.call_args.kwargs == {"include_anomaly_map": True} + assert model._model.post_process.call_args.kwargs == { + "include_anomaly_map": True, + "confidence": "best", + } assert ( InferenceConfiguration(include_anomaly_map=True).to_classification_parameters()[ "include_anomaly_map" @@ -132,3 +144,15 @@ def test_request_and_sdk_forward_optional_map_to_the_model(): "include_anomaly_map" not in InferenceConfiguration().to_classification_parameters() ) + + +def test_explicit_request_confidence_decides_the_verdict(): + model = anomaly_adapter() + # 0.4 is also the request default; sent explicitly it must not become "best". + request = ClassificationInferenceRequest( + model_id="defects/1", image=png_image(), confidence=0.4 + ) + + model.infer_from_request(request) + + assert model._model.post_process.call_args.kwargs["confidence"] == 0.4 diff --git a/tests/workflows/integration_tests/execution/assets/anomaly_anomalous.jpg b/tests/workflows/integration_tests/execution/assets/anomaly_anomalous.jpg new file mode 100644 index 0000000000..8c4cd73146 Binary files /dev/null and b/tests/workflows/integration_tests/execution/assets/anomaly_anomalous.jpg differ diff --git a/tests/workflows/integration_tests/execution/assets/anomaly_normal.jpg b/tests/workflows/integration_tests/execution/assets/anomaly_normal.jpg new file mode 100644 index 0000000000..42d1176295 Binary files /dev/null and b/tests/workflows/integration_tests/execution/assets/anomaly_normal.jpg differ diff --git a/tests/workflows/integration_tests/execution/assets/image_credits.txt b/tests/workflows/integration_tests/execution/assets/image_credits.txt index 635034fc9d..3f837d8689 100644 --- a/tests/workflows/integration_tests/execution/assets/image_credits.txt +++ b/tests/workflows/integration_tests/execution/assets/image_credits.txt @@ -4,4 +4,5 @@ dogs.jpg: https://www.pexels.com/photo/brown-and-white-dogs-sitting-on-field-356 multi-fruit.jpg: https://www.freepik.com/free-photo/front-close-view-organic-nutrition-source-fresh-bananas-bundle-red-apples-orange-with-stem-dark-background_17119128.htm multi_line_text.jpg: https://www.pexels.com/photo/illuminated-qoute-board-2255441/ asl_image.jpg: https://universe.roboflow.com/david-lee-d0rhs/american-sign-language-letters -car.jpg: https://pixabay.com/photos/car-vehicle-sports-car-auto-63930/ \ No newline at end of file +car.jpg: https://pixabay.com/photos/car-vehicle-sports-car-auto-63930/ +anomaly_anomalous.jpg, anomaly_normal.jpg: https://universe.roboflow.com/paul-guerrie-tang1/anomaly-det-algbq diff --git a/tests/workflows/integration_tests/execution/conftest.py b/tests/workflows/integration_tests/execution/conftest.py index 206121f661..28bfc682a5 100644 --- a/tests/workflows/integration_tests/execution/conftest.py +++ b/tests/workflows/integration_tests/execution/conftest.py @@ -55,6 +55,16 @@ def multi_line_text_image() -> np.ndarray: return cv2.imread(os.path.join(ASSETS_DIR, "multi_line_text.jpg")) +@pytest.fixture(scope="function") +def anomalous_image() -> np.ndarray: + return cv2.imread(os.path.join(ASSETS_DIR, "anomaly_anomalous.jpg")) + + +@pytest.fixture(scope="function") +def normal_image() -> np.ndarray: + return cv2.imread(os.path.join(ASSETS_DIR, "anomaly_normal.jpg")) + + @pytest.fixture(scope="function") def stitch_left_image() -> np.ndarray: return cv2.imread(os.path.join(ASSETS_DIR, "stitch", "v_left.jpeg")) diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_anomaly_detection.py b/tests/workflows/integration_tests/execution/test_workflow_with_anomaly_detection.py new file mode 100644 index 0000000000..03cf4e93f9 --- /dev/null +++ b/tests/workflows/integration_tests/execution/test_workflow_with_anomaly_detection.py @@ -0,0 +1,218 @@ +import numpy as np +import pytest + +from inference.core.env import ( + ENABLE_TENSOR_DATA_REPRESENTATION, + USE_INFERENCE_MODELS, + WORKFLOWS_MAX_CONCURRENT_STEPS, +) +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.core_steps.common.serializers_tensor import ( + serialise_native_classification, +) +from inference.core.workflows.execution_engine.core import ExecutionEngine +from tests.workflows.integration_tests.execution.workflows_gallery_collector.decorators import ( + add_to_workflows_gallery, +) + +pytestmark = pytest.mark.skipif( + not USE_INFERENCE_MODELS, + reason="PatchCore and FoundAD load only under the inference_models runtime", +) + +# Public models of the anomaly reference project, trained on the same data. +FOUNDAD_MODEL_ID = "paul-guerrie-tang1/anomaly-det-algbq-1-foundad-t1" +PATCHCORE_MODEL_ID = "paul-guerrie-tang1/anomaly-det-algbq-1-patchcore-t2" + +WORKFLOW_DEFINITION = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + { + "type": "WorkflowParameter", + "name": "model_id", + "default_value": FOUNDAD_MODEL_ID, + }, + {"type": "WorkflowParameter", "name": "confidence", "default_value": 0.5}, + {"type": "WorkflowParameter", "name": "anomaly_threshold"}, + ], + "steps": [ + { + "type": "roboflow_core/anomaly_detection_model@v1", + "name": "model", + "images": "$inputs.image", + "model_id": "$inputs.model_id", + "confidence": "$inputs.confidence", + "anomaly_threshold": "$inputs.anomaly_threshold", + }, + { + "type": "roboflow_core/anomaly_heatmap_visualization@v1", + "name": "heatmap", + "image": "$inputs.image", + "anomaly_map": "$steps.model.anomaly_map", + "anomaly_threshold": "$steps.model.anomaly_decision_threshold", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.model.predictions", + }, + { + "type": "JsonField", + "name": "is_anomalous", + "selector": "$steps.model.is_anomalous", + }, + { + "type": "JsonField", + "name": "anomaly_score", + "selector": "$steps.model.anomaly_score", + }, + { + "type": "JsonField", + "name": "anomaly_threshold", + "selector": "$steps.model.anomaly_threshold", + }, + { + "type": "JsonField", + "name": "anomaly_decision_threshold", + "selector": "$steps.model.anomaly_decision_threshold", + }, + { + "type": "JsonField", + "name": "heatmap", + "selector": "$steps.heatmap.image", + }, + ], +} + + +def _engine(model_manager: ModelManager, roboflow_api_key: str) -> ExecutionEngine: + return ExecutionEngine.init( + workflow_definition=WORKFLOW_DEFINITION, + init_parameters={ + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": roboflow_api_key, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + }, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + +def _decision(result: dict): + predictions = result["predictions"] + if ENABLE_TENSOR_DATA_REPRESENTATION: + # The flag ships a native ClassificationPrediction; the serializer restores + # the numpy dict shape. + predictions = serialise_native_classification(prediction=predictions) + return result["is_anomalous"], predictions["top"] + + +@add_to_workflows_gallery( + category="Workflows with visualization blocks", + use_case_title="Workflow with anomaly detection and its heatmap", + workflow_name_in_app="anomaly-detection-heatmap", + use_case_description=""" +This workflow runs an anomaly detection model (FoundAD or PatchCore) and overlays its +anomaly map on the image. The heatmap is anchored at the block's decision threshold, +so the middle of the colour ramp marks the anomalous verdict at any confidence. + """, + workflow_definition=WORKFLOW_DEFINITION, +) +@pytest.mark.parametrize("model_id", [FOUNDAD_MODEL_ID, PATCHCORE_MODEL_ID]) +def test_anomaly_detection_workflow_decides_and_draws_the_heatmap( + model_manager: ModelManager, + anomalous_image: np.ndarray, + normal_image: np.ndarray, + roboflow_api_key: str, + model_id: str, +) -> None: + # given + execution_engine = _engine(model_manager, roboflow_api_key) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [anomalous_image, normal_image], + "model_id": model_id, + } + ) + + # then + assert len(result) == 2, "Expected one output per input image" + assert _decision(result[0]) == (True, "anomalous") + assert _decision(result[1]) == (False, "normal") + for image_result, image in zip(result, (anomalous_image, normal_image)): + # At the default confidence the decision sits at the saved threshold. + assert image_result["anomaly_decision_threshold"] == pytest.approx( + image_result["anomaly_threshold"] + ) + assert image_result["heatmap"].numpy_image.shape == image.shape + assert not np.array_equal(image_result["heatmap"].numpy_image, image) + + +@pytest.mark.parametrize("model_id", [FOUNDAD_MODEL_ID, PATCHCORE_MODEL_ID]) +def test_anomaly_detection_workflow_confidence_moves_the_decision( + model_manager: ModelManager, + anomalous_image: np.ndarray, + normal_image: np.ndarray, + roboflow_api_key: str, + model_id: str, +) -> None: + # given + execution_engine = _engine(model_manager, roboflow_api_key) + + # when: every anomalous confidence reaches 0, none reaches 1 + lenient = execution_engine.run( + runtime_parameters={ + "image": normal_image, + "model_id": model_id, + "confidence": 0.0, + } + ) + strict = execution_engine.run( + runtime_parameters={ + "image": anomalous_image, + "model_id": model_id, + "confidence": 1.0, + } + ) + + # then + assert _decision(lenient[0]) == (True, "anomalous") + assert _decision(strict[0]) == (False, "normal") + assert lenient[0]["anomaly_decision_threshold"] < lenient[0]["anomaly_threshold"] + assert strict[0]["anomaly_decision_threshold"] > strict[0]["anomaly_threshold"] + # Past a decision threshold below zero the whole map is anomalous evidence. + assert lenient[0]["heatmap"].numpy_image.shape == normal_image.shape + + +@pytest.mark.parametrize("model_id", [FOUNDAD_MODEL_ID, PATCHCORE_MODEL_ID]) +def test_anomaly_detection_workflow_threshold_override_composes_with_confidence( + model_manager: ModelManager, + anomalous_image: np.ndarray, + roboflow_api_key: str, + model_id: str, +) -> None: + # given + execution_engine = _engine(model_manager, roboflow_api_key) + baseline = execution_engine.run( + runtime_parameters={"image": anomalous_image, "model_id": model_id} + )[0] + above_the_score = baseline["anomaly_score"] * 2 + + # when + result = execution_engine.run( + runtime_parameters={ + "image": anomalous_image, + "model_id": model_id, + "anomaly_threshold": above_the_score, + } + )[0] + + # then + assert _decision(result) == (False, "normal") + assert result["anomaly_threshold"] == above_the_score + assert result["anomaly_decision_threshold"] == pytest.approx(above_the_score) diff --git a/workflows/CHANGELOG.md b/workflows/CHANGELOG.md index e48d3ffec0..2818338201 100644 --- a/workflows/CHANGELOG.md +++ b/workflows/CHANGELOG.md @@ -14,12 +14,20 @@ Earlier engine changes and migration guidance remain in the See the [engine release rule](../.cursor/rules/execution-engine-version-changelog.mdc) for contributor and maintainer responsibilities. -## Unreleased +## `0.2.11` + +Bundled execution engine: `1.16.1`. ### Added +- Anomaly Detection Model block (`roboflow_core/anomaly_detection_model@v1`): runs a PatchCore or FoundAD model trained on a Roboflow anomaly detection project (served on the classification route). Outputs the classification-style `predictions` (plus `anomaly_score`, `anomaly_threshold`, `anomaly_scale`, `is_anomalous`), the raw `anomaly_score`, the `anomaly_threshold` in effect, the `anomaly_decision_threshold` (the raw score at which the anomalous confidence equals `confidence`), the `is_anomalous` verdict and the `anomaly_map` (a float32 `numpy_array` at the network input resolution; `None` when `include_anomaly_map` is off). `confidence` (default 0.5, the calibrated decision) is the anomalous confidence an image must reach; `anomaly_threshold` overrides the threshold saved with the model and re-decides with the same rule. `ModelsProvider.run_classification` gains `include_anomaly_map`. +- Anomaly Heatmap Visualization block (`roboflow_core/anomaly_heatmap_visualization@v1`): stretches an anomaly map onto its image (bilinear) and blends it as a heatmap on a ramp anchored at its `anomaly_threshold` input, meant to be wired to the model block's `anomaly_decision_threshold`: `t = max(v, 0) / (max(v, 0) + anchor)`; an anchor at or below zero saturates every positive value. `color_map` is the Roboflow app's ramp (`ROBOFLOW`) or an OpenCV map (`JET`, `TURBO`, `INFERNO`, `VIRIDIS`, `HOT`); `opacity` and `copy_image` as the other visualization blocks. - `roboflow_workflows.http_contract` (`describe`, `entities`, `errors`) — the HTTP surface `inference-server` imports to expose workflow routes. +### Changed + +- `inference-sdk>=1.8.0`: the hosted (API v0) request carries `include_anomaly_map` from 1.8.0, which the Anomaly Detection Model block needs in REMOTE execution. The anomaly blocks need `inference-models` 0.39.5 (`anomaly_scale`). + ## `0.2.10` Bundled execution engine: `1.16.1`. diff --git a/workflows/pyproject.toml b/workflows/pyproject.toml index 47a26c9dcf..c74f8f3cf8 100644 --- a/workflows/pyproject.toml +++ b/workflows/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "roboflow-workflows" -version = "0.2.11rc1" +version = "0.2.11" description = "Roboflow Workflows execution engine and block library" readme = "README.md" license = {file = "LICENSE"} @@ -24,7 +24,7 @@ dependencies = [ "torch>=2.0.1,<3.0.0", "torchvision", "inference-models>=0.39.5rc1", - "inference-sdk>=1.7.0", + "inference-sdk>=1.8.0", "msgpack", "websocket-client~=1.9.0", "openai>=1.17.0,<2.0.0", diff --git a/workflows/roboflow_workflows/core_steps/common/anomaly_detection.py b/workflows/roboflow_workflows/core_steps/common/anomaly_detection.py new file mode 100644 index 0000000000..1354ffbbf2 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/common/anomaly_detection.py @@ -0,0 +1,252 @@ +"""Shared helpers for the anomaly detection (PatchCore, FoundAD) model blocks. + +The inference server answers an anomaly model through the classification route +with an ``AnomalyDetectionResponse``: the usual classification fields plus +``anomaly_score`` (raw, unbounded, larger means more anomalous), +``anomaly_threshold`` (the decision boundary saved with the model), +``anomaly_scale`` (the anomalous confidence is +``sigmoid((anomaly_score - anomaly_threshold) / anomaly_scale)``), +``is_anomalous`` (the anomalous confidence reached the request confidence) and, +on request, ``anomaly_map`` - the local evidence at the +network input resolution, serialised as ``{"shape": [h, w], "dtype": +"float32", "data": }``. + +``roboflow_workflows`` cannot import ``inference.core``, so the payload decoder +is duplicated here (it mirrors ``inference_sdk.http.utils.anomaly_maps``). +""" + +import base64 +import binascii +import math +from typing import Any, Dict, List, Mapping, Optional + +import numpy as np + +ANOMALY_SCORE_KEY = "anomaly_score" +ANOMALY_THRESHOLD_KEY = "anomaly_threshold" +ANOMALY_SCALE_KEY = "anomaly_scale" +ANOMALY_DECISION_THRESHOLD_KEY = "anomaly_decision_threshold" +IS_ANOMALOUS_KEY = "is_anomalous" +ANOMALY_MAP_KEY = "anomaly_map" +ANOMALY_METADATA_KEYS = ( + ANOMALY_SCORE_KEY, + ANOMALY_THRESHOLD_KEY, + ANOMALY_SCALE_KEY, + IS_ANOMALOUS_KEY, +) + +ANOMALY_MAP_DTYPE = "float32" +_ANOMALY_MAP_NUMPY_DTYPE = np.dtype(" np.ndarray: + """Decode the ``anomaly_map`` payload of an anomaly detection response. + + Args: + payload (Mapping[str, Any]): Mapping with ``shape`` (``[height, width]``), + ``dtype`` (``"float32"``) and ``data`` (base64 of row-major + little-endian float32 bytes), as the server serialises it. + + Returns: + Writable float32 array of shape ``(height, width)`` at the network input + resolution; stretch it to the image size to overlay it. + + Raises: + ValueError: If the dtype is not float32, the shape is not two positive + ints, the data is not valid base64, or its length does not match + the shape. + """ + if payload.get("dtype") != ANOMALY_MAP_DTYPE: + raise ValueError( + f"Expected anomaly map dtype {ANOMALY_MAP_DTYPE!r}, got " + f"{payload.get('dtype')!r}" + ) + shape = payload.get("shape") + if ( + not isinstance(shape, (list, tuple)) + or len(shape) != 2 + or not all(isinstance(side, int) and side > 0 for side in shape) + ): + raise ValueError(f"Anomaly map shape must be two positive ints, got {shape!r}") + try: + data = base64.b64decode(payload.get("data", ""), validate=True) + except (binascii.Error, ValueError, TypeError) as error: + raise ValueError("Anomaly map payload is not valid base64") from error + expected_bytes = shape[0] * shape[1] * _ANOMALY_MAP_NUMPY_DTYPE.itemsize + if len(data) != expected_bytes: + raise ValueError( + f"Anomaly map payload holds {len(data)} bytes, expected {expected_bytes} " + f"for shape {tuple(shape)}" + ) + values = np.frombuffer(data, dtype=_ANOMALY_MAP_NUMPY_DTYPE).reshape(shape) + return values.astype(np.float32, copy=True) + + +def coerce_anomaly_map(value: Any) -> Optional[np.ndarray]: + """The map as a float32 ``(h, w)`` array, whatever form the response carried. + + Args: + value (Any): ``None`` (map not requested), the serialised payload + mapping (local execution dumps the response as is), an ``np.ndarray`` + (``inference_sdk`` decodes the payload on the remote path) or a + nested float list from an older server. + + Returns: + ``None`` when ``value`` is ``None``, else a float32 ``(h, w)`` array. + + Raises: + ValueError: If the value is not one of the forms above or is not 2D. + """ + if value is None: + return None + if isinstance(value, Mapping): + return decode_anomaly_map(value) + try: + array = np.asarray(value, dtype=np.float32) + except (TypeError, ValueError) as error: + raise ValueError( + f"Anomaly map must be a payload mapping, an array or a nested list, " + f"got {type(value).__name__}" + ) from error + if array.ndim != 2: + raise ValueError(f"Anomaly map must be 2D, got shape {tuple(array.shape)}") + return array + + +def require_anomaly_response(prediction: Dict[str, Any], model_id: str) -> None: + """Fail loudly when a classification response is not an anomaly response. + + Args: + prediction (Dict[str, Any]): The dumped classification response. + model_id (str): The model that produced it, for the error message. + + Raises: + ValueError: If ``anomaly_score``, ``anomaly_threshold``, + ``anomaly_scale`` or ``is_anomalous`` is missing - the model is a + plain classifier, not a PatchCore or FoundAD anomaly model. + """ + missing = [key for key in ANOMALY_METADATA_KEYS if key not in prediction] + if missing: + raise ValueError( + f"Model {model_id!r} did not return an anomaly detection response " + f"(missing {missing}). The Anomaly Detection Model block needs a " + f"PatchCore or FoundAD model trained on an anomaly detection project; " + f"use the Single-Label Classification Model block for other classifiers." + ) + + +def validate_threshold_override(threshold: Optional[float]) -> Optional[float]: + """A finite float, or ``None`` to keep the threshold saved with the model. + + Raises: + ValueError: If ``threshold`` is not a finite number. + """ + if threshold is None: + return None + if isinstance(threshold, bool) or not isinstance(threshold, (int, float)): + raise ValueError( + f"anomaly_threshold must be a number, got {type(threshold).__name__}" + ) + value = float(threshold) + if not math.isfinite(value): + raise ValueError(f"anomaly_threshold must be finite, got {value}") + return value + + +def decision_threshold(threshold: float, scale: float, confidence: float) -> float: + """The raw score at which the anomalous confidence equals ``confidence``. + + ``threshold + scale * logit(confidence)``, so ``threshold`` itself at 0.5. The + logit is clipped like the sigmoid, keeping the result finite at 0 and 1. + """ + if confidence <= 0: + margin = -MAX_CONFIDENCE_MARGIN + elif confidence >= 1: + margin = MAX_CONFIDENCE_MARGIN + else: + margin = math.log(confidence / (1 - confidence)) + margin = max(-MAX_CONFIDENCE_MARGIN, min(MAX_CONFIDENCE_MARGIN, margin)) + return threshold + scale * margin + + +def anomalous_confidence(score: float, threshold: float, scale: float) -> float: + """``sigmoid((score - threshold) / scale)``, exactly 0.5 at ``threshold``.""" + margin = (score - threshold) / scale + margin = max(-MAX_CONFIDENCE_MARGIN, min(MAX_CONFIDENCE_MARGIN, margin)) + return 1.0 / (1.0 + math.exp(-margin)) + + +def redecide_at_threshold( + verdict: Dict[str, Any], threshold: float, confidence: float +) -> float: + """Re-decide a verdict against a caller-supplied threshold, in place. + + The rule is the server's with the override in place of the saved threshold: + ``is_anomalous`` becomes ``sigmoid((anomaly_score - threshold) / + anomaly_scale) >= confidence`` and ``anomaly_threshold`` reports the + override. + + Args: + verdict (Dict[str, Any]): Holds ``anomaly_score`` and ``anomaly_scale``. + threshold (float): The override, already validated. + confidence (float): The anomalous confidence an image must reach. + + Returns: + The anomalous confidence relative to the override. + """ + anomalous = anomalous_confidence( + score=float(verdict[ANOMALY_SCORE_KEY]), + threshold=threshold, + scale=float(verdict[ANOMALY_SCALE_KEY]), + ) + verdict[ANOMALY_THRESHOLD_KEY] = threshold + verdict[IS_ANOMALOUS_KEY] = anomalous >= confidence + return anomalous + + +def apply_threshold_override( + prediction: Dict[str, Any], threshold: Optional[float], confidence: float +) -> Dict[str, Any]: + """Re-decide a dumped anomaly detection response against an override, in place. + + See ``redecide_at_threshold`` for the verdict. ``predictions`` is rebuilt with + both class confidences relative to the override, the decided class first, and + ``top`` / ``confidence`` follow it. With ``threshold`` ``None`` the response + is unchanged: the server already decided against the saved threshold with the + same ``confidence``. + + Args: + prediction (Dict[str, Any]): A dumped anomaly detection response. + threshold (Optional[float]): The override, already validated. + confidence (float): The anomalous confidence an image must reach. + + Returns: + The same dict. + """ + if threshold is None: + return prediction + anomalous = redecide_at_threshold(prediction, threshold, confidence) + class_predictions: List[Dict[str, Any]] = [ + { + "class_id": 0, + "class": NORMAL_CLASS_NAME, + "confidence": round(1 - anomalous, 4), + }, + { + "class_id": 1, + "class": ANOMALOUS_CLASS_NAME, + "confidence": round(anomalous, 4), + }, + ] + if prediction[IS_ANOMALOUS_KEY]: + class_predictions.reverse() + prediction["predictions"] = class_predictions + prediction["top"] = class_predictions[0]["class"] + prediction["confidence"] = class_predictions[0]["confidence"] + return prediction diff --git a/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py b/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py index 4381ac0084..65f1c29fe8 100644 --- a/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py +++ b/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py @@ -4,6 +4,12 @@ import numpy as np import supervision as sv import torch +from roboflow_workflows.core_steps.common.anomaly_detection import ( + ANOMALOUS_CLASS_NAME, + ANOMALY_METADATA_KEYS, + IS_ANOMALOUS_KEY, + NORMAL_CLASS_NAME, +) from roboflow_workflows.core_steps.common.keypoints import real_keypoints_count from roboflow_workflows.core_steps.common.serializers import ( _attach_parent_metadata_to_detection_dict, @@ -600,6 +606,14 @@ def _serialise_classification_model_style( key=lambda item: item["confidence"], reverse=True, ) + # An anomaly model's verdict is decided by the request confidence, not by + # the larger class confidence: the decided class leads, as the numpy + # anomaly block orders it. + if IS_ANOMALOUS_KEY in image_metadata: + individual_classes_predictions = _decided_anomaly_class_first( + individual_classes_predictions, + is_anomalous=bool(image_metadata[IS_ANOMALOUS_KEY]), + ) result["predictions"] = individual_classes_predictions result["top"] = ( individual_classes_predictions[0]["class"] @@ -636,6 +650,12 @@ def _serialise_classification_model_style( result["predictions"] = predictions_dict result["predicted_classes"] = predicted_classes + # The numpy `AnomalyDetectionResponse` dumps its verdict right after the + # classification fields; the anomaly block keeps the raw map out of the dict. + for source_key in ANOMALY_METADATA_KEYS: + if source_key in image_metadata: + result[source_key] = image_metadata[source_key] + # Append order mirrors the numpy block's in-place writes after model_dump: # attach_prediction_type_info, then parent_id, then root_parent_id # (inference_id was already emitted first, matching the dump order). @@ -649,6 +669,15 @@ def _serialise_classification_model_style( return result +def _decided_anomaly_class_first( + class_predictions: List[dict], *, is_anomalous: bool +) -> List[dict]: + decided_class = ANOMALOUS_CLASS_NAME if is_anomalous else NORMAL_CLASS_NAME + decided = [p for p in class_predictions if p["class"] == decided_class] + others = [p for p in class_predictions if p["class"] != decided_class] + return decided + others + + def _serialise_classification_formatter_style( prediction: Union[ClassificationPrediction, MultiLabelClassificationPrediction], image_metadata: dict, diff --git a/workflows/roboflow_workflows/core_steps/loader.py b/workflows/roboflow_workflows/core_steps/loader.py index fbb5fe6549..579fc2bbb9 100644 --- a/workflows/roboflow_workflows/core_steps/loader.py +++ b/workflows/roboflow_workflows/core_steps/loader.py @@ -891,6 +891,14 @@ from roboflow_workflows.core_steps.models.roboflow.multi_class_classification.v3 import ( RoboflowClassificationModelBlockV3, ) +if ENABLE_TENSOR_DATA_REPRESENTATION: + from roboflow_workflows.core_steps.models.roboflow.anomaly_detection.v1_tensor import ( + AnomalyDetectionModelBlockV1, + ) +else: + from roboflow_workflows.core_steps.models.roboflow.anomaly_detection.v1 import ( + AnomalyDetectionModelBlockV1, + ) if ENABLE_TENSOR_DATA_REPRESENTATION: from roboflow_workflows.core_steps.models.roboflow.multi_label_classification.v1_tensor import ( RoboflowMultiLabelClassificationModelBlockV1, @@ -1285,6 +1293,14 @@ ) # Visualizers +if not ENABLE_TENSOR_DATA_REPRESENTATION: + from roboflow_workflows.core_steps.visualizations.anomaly_heatmap.v1 import ( + AnomalyHeatmapVisualizationBlockV1, + ) +else: + from roboflow_workflows.core_steps.visualizations.anomaly_heatmap.v1_tensor import ( + AnomalyHeatmapVisualizationBlockV1, + ) if not ENABLE_TENSOR_DATA_REPRESENTATION: from roboflow_workflows.core_steps.visualizations.background_color.v1 import ( BackgroundColorVisualizationBlockV1, @@ -1796,6 +1812,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: SpaceXAIBlockV2, SpaceXAIBlockV3, CosineSimilarityBlockV1, + AnomalyHeatmapVisualizationBlockV1, BackgroundColorVisualizationBlockV1, BarcodeDetectorBlockV1, BlurVisualizationBlockV1, @@ -1942,6 +1959,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: RoboflowKeypointDetectionModelBlockV3, RoboflowClassificationModelBlockV2, RoboflowClassificationModelBlockV3, + AnomalyDetectionModelBlockV1, RoboflowMultiLabelClassificationModelBlockV2, RoboflowMultiLabelClassificationModelBlockV3, ActionRecognitionModelBlockV1, diff --git a/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/__init__.py b/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/v1.py b/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/v1.py new file mode 100644 index 0000000000..728bb0a823 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/v1.py @@ -0,0 +1,397 @@ +from typing import Any, Dict, List, Literal, Optional, Type, Union + +import numpy as np +from pydantic import ConfigDict, Field +from roboflow_workflows.core_steps.common.anomaly_detection import ( + ANOMALY_DECISION_THRESHOLD_KEY, + ANOMALY_MAP_KEY, + ANOMALY_SCALE_KEY, + ANOMALY_SCORE_KEY, + ANOMALY_THRESHOLD_KEY, + IS_ANOMALOUS_KEY, + apply_threshold_override, + coerce_anomaly_map, + decision_threshold, + require_anomaly_response, + validate_threshold_override, +) +from roboflow_workflows.core_steps.common.entities import StepExecutionMode +from roboflow_workflows.core_steps.common.utils import attach_prediction_type_info +from roboflow_workflows.environment import ( + HOSTED_CLASSIFICATION_URL, + LOCAL_INFERENCE_API_URL, + WORKFLOWS_REMOTE_API_KEY_TRANSPORT, + WORKFLOWS_REMOTE_API_TARGET, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, +) +from roboflow_workflows.execution_engine.constants import ( + INFERENCE_ID_KEY, + PARENT_ID_KEY, + ROOT_PARENT_ID_KEY, +) +from roboflow_workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from roboflow_workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + CLASSIFICATION_PREDICTION_KIND, + FLOAT_KIND, + FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + NUMPY_ARRAY_KIND, + ROBOFLOW_MODEL_ID_KIND, + ROBOFLOW_PROJECT_KIND, + FloatZeroToOne, + ImageInputField, + RoboflowModelField, + Selector, +) +from roboflow_workflows.execution_engine.entities.workload import ( + Discovery, + RuntimeRestriction, + WorkOperation, +) +from roboflow_workflows.prototypes.block import ( + BlockResult, + DependentResource, + WorkflowBlock, + WorkflowBlockManifest, + roboflow_platform_model, + roboflow_platform_project, +) +from roboflow_workflows.prototypes.models_provider import ModelsProvider + +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + +PREDICTION_TYPE = "classification" + +LONG_DESCRIPTION = """ +Run an anomaly detection model (PatchCore or FoundAD) trained on an anomaly +detection project hosted on Roboflow. + +An anomaly model learns what *normal* looks like from normal images only and +scores a new image by how far it departs from that. The block reports: + +- `predictions`: the classification-style response, with `top` being `normal` + or `anomalous` and the two class confidences. The anomalous confidence is + `sigmoid((anomaly_score - anomaly_threshold) / scale)`, with the scale saved + with the model, so it is exactly 0.5 at the threshold. +- `anomaly_score`: the raw image score (unbounded, larger means more anomalous). +- `anomaly_threshold`: the threshold in effect for this image (saved with the + model, or the override). +- `is_anomalous`: whether the anomalous confidence reached `confidence`. +- `anomaly_decision_threshold`: the raw score at which the anomalous confidence + equals `confidence` (`anomaly_threshold` itself at 0.5). Connect it to the + heatmap's `anomaly_threshold` so the middle of its ramp marks the decision. +- `anomaly_map`: the local anomaly evidence as a float32 `(height, width)` + array at the network's input resolution (square), in the same units as + `anomaly_score`. The map covers the whole image as the network saw it, so + stretching it to the image size overlays it. Connect it to the **Anomaly + Heatmap Visualization** block to draw it. Set `include_anomaly_map` to + `False` to skip the map when only the verdict is needed. + +`confidence` sets the operating point without retraining: the default 0.5 +reproduces the decision calibrated during training; lower it to catch more +defects at the cost of more false alarms, raise it to do the opposite. +`anomaly_threshold` replaces the saved threshold on the raw score; the class +confidences are then recomputed relative to it and `confidence` applies to them +the same way. + +You will need to set your Roboflow API key in your Inference environment to use +this block. To learn more about setting your Roboflow API key, [refer to the +Inference documentation](https://inference.roboflow.com/quickstart/configure_api_key/). +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Anomaly Detection Model", + "version": "v1", + "short_description": "Score an image against a model of normal images and locate what differs.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + "search_keywords": [ + "anomaly", + "anomaly detection", + "defect", + "PatchCore", + "FoundAD", + "heatmap", + ], + "ui_manifest": { + "section": "model", + "icon": "far fa-magnifying-glass-chart", + "blockPriority": 2.5, + "inference": True, + }, + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/anomaly_detection_model@v1"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + include_anomaly_map: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Return the local anomaly map at the network input resolution. Disable it when only the verdict and score are needed.", + examples=[True, "$inputs.include_anomaly_map"], + ) + confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.5, + description="Minimum anomalous confidence for an anomalous verdict. 0.5 reproduces the decision calibrated during training; lower it to flag more images as anomalous.", + examples=[0.5, "$inputs.confidence"], + ) + anomaly_threshold: Union[Optional[float], Selector(kind=[FLOAT_KIND])] = Field( + default=None, + description="Threshold on the raw anomaly score, replacing the one saved with the model; the anomalous confidence is recomputed relative to it. Leave empty to use the saved threshold.", + examples=[1.25, "$inputs.anomaly_threshold"], + ) + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag to disable project-level active learning for this block.", + examples=[True, "$inputs.disable_active_learning"], + ) + active_learning_target_dataset: Union[ + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + ] = Field( + default=None, + description="Target dataset for active learning, if enabled.", + examples=["my_project", "$inputs.al_target_project"], + ) + + @classmethod + def get_compatible_task_types(cls) -> Optional[List[str]]: + # Anomaly models are registered under the classification task. + return ["classification"] + + def discover_dependent_resources(self) -> Optional[List[DependentResource]]: + resources = [roboflow_platform_model(model_id=self.model_id)] + if self.disable_active_learning is True: + return resources + if self.active_learning_target_dataset is not None: + resources.append( + roboflow_platform_project( + project_url=self.active_learning_target_dataset + ) + ) + return resources + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="predictions", kind=[CLASSIFICATION_PREDICTION_KIND]), + OutputDefinition(name=ANOMALY_SCORE_KEY, kind=[FLOAT_KIND]), + OutputDefinition(name=ANOMALY_THRESHOLD_KEY, kind=[FLOAT_KIND]), + OutputDefinition(name=ANOMALY_DECISION_THRESHOLD_KEY, kind=[FLOAT_KIND]), + OutputDefinition(name=IS_ANOMALOUS_KEY, kind=[BOOLEAN_KIND]), + OutputDefinition(name=ANOMALY_MAP_KEY, kind=[NUMPY_ARRAY_KIND]), + OutputDefinition(name=INFERENCE_ID_KEY, kind=[INFERENCE_ID_KIND]), + OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + def discover_work_operations(self) -> List[WorkOperation]: + return [WorkOperation.MODEL_INFERENCE] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + +class AnomalyDetectionModelBlockV1(WorkflowBlock): + + def __init__( + self, + model_manager: ModelsProvider, + api_key: Optional[str], + step_execution_mode: StepExecutionMode, + ): + self._model_manager = model_manager + self._api_key = api_key + self._step_execution_mode = step_execution_mode + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + include_anomaly_map: bool, + confidence: float, + anomaly_threshold: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + threshold_override = validate_threshold_override(anomaly_threshold) + if self._step_execution_mode is StepExecutionMode.LOCAL: + return self.run_locally( + images=images, + model_id=model_id, + include_anomaly_map=include_anomaly_map, + confidence=confidence, + anomaly_threshold=threshold_override, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + elif self._step_execution_mode is StepExecutionMode.REMOTE: + return self.run_remotely( + images=images, + model_id=model_id, + include_anomaly_map=include_anomaly_map, + confidence=confidence, + anomaly_threshold=threshold_override, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + else: + raise ValueError( + f"Unknown step execution mode: {self._step_execution_mode}" + ) + + def run_locally( + self, + images: Batch[WorkflowImageData], + model_id: str, + include_anomaly_map: bool, + confidence: float, + anomaly_threshold: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + inference_images = [i.to_inference_format(numpy_preferred=True) for i in images] + self._model_manager.add_model( + model_id=model_id, + api_key=self._api_key, + ) + predictions = self._model_manager.run_classification( + model_id=model_id, + images=inference_images, + api_key=self._api_key, + confidence=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + include_anomaly_map=include_anomaly_map, + ) + return self._post_process_result( + predictions=predictions, + images=images, + model_id=model_id, + confidence=confidence, + anomaly_threshold=anomaly_threshold, + ) + + def run_remotely( + self, + images: Batch[Optional[WorkflowImageData]], + model_id: str, + include_anomaly_map: bool, + confidence: float, + anomaly_threshold: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + api_url = ( + LOCAL_INFERENCE_API_URL + if WORKFLOWS_REMOTE_API_TARGET != "hosted" + else HOSTED_CLASSIFICATION_URL + ) + client = InferenceHTTPClient( + api_url=api_url, + api_key=self._api_key, + ) + if WORKFLOWS_REMOTE_API_TARGET == "hosted": + client.select_api_v0() + client_config = InferenceConfiguration( + api_key_transport=WORKFLOWS_REMOTE_API_KEY_TRANSPORT, + include_anomaly_map=include_anomaly_map, + confidence_threshold=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + max_batch_size=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, + source="workflow-execution", + ) + client.configure(inference_configuration=client_config) + non_empty_inference_images = [i.base64_image for i in images] + predictions = client.infer( + inference_input=non_empty_inference_images, + model_id=model_id, + ) + if not isinstance(predictions, list): + predictions = [predictions] + return self._post_process_result( + predictions=predictions, + images=images, + model_id=model_id, + confidence=confidence, + anomaly_threshold=anomaly_threshold, + ) + + def _post_process_result( + self, + images: Batch[WorkflowImageData], + predictions: List[dict], + model_id: str, + confidence: float, + anomaly_threshold: Optional[float], + ) -> BlockResult: + predictions = attach_prediction_type_info( + predictions=predictions, + prediction_type=PREDICTION_TYPE, + ) + results: List[Dict[str, Any]] = [] + for prediction, image in zip(predictions, images): + require_anomaly_response(prediction, model_id=model_id) + # The raw map leaves the response dict: it is its own output, and the + # dict is what reaches sinks and JSON outputs. + anomaly_map: Optional[np.ndarray] = coerce_anomaly_map( + prediction.pop(ANOMALY_MAP_KEY, None) + ) + prediction = apply_threshold_override( + prediction, anomaly_threshold, confidence + ) + prediction[PARENT_ID_KEY] = image.parent_metadata.parent_id + prediction[ROOT_PARENT_ID_KEY] = ( + image.workflow_root_ancestor_metadata.parent_id + ) + results.append( + { + "predictions": prediction, + ANOMALY_SCORE_KEY: float(prediction[ANOMALY_SCORE_KEY]), + ANOMALY_THRESHOLD_KEY: float(prediction[ANOMALY_THRESHOLD_KEY]), + ANOMALY_DECISION_THRESHOLD_KEY: decision_threshold( + threshold=float(prediction[ANOMALY_THRESHOLD_KEY]), + scale=float(prediction[ANOMALY_SCALE_KEY]), + confidence=confidence, + ), + IS_ANOMALOUS_KEY: bool(prediction[IS_ANOMALOUS_KEY]), + ANOMALY_MAP_KEY: anomaly_map, + INFERENCE_ID_KEY: prediction.get(INFERENCE_ID_KEY), + "model_id": model_id, + } + ) + return results diff --git a/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/v1_tensor.py b/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/v1_tensor.py new file mode 100644 index 0000000000..33ba19f189 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/models/roboflow/anomaly_detection/v1_tensor.py @@ -0,0 +1,557 @@ +"""Tensor-native sibling of `roboflow_core/anomaly_detection_model@v1`. + +Under ENABLE_TENSOR_DATA_REPRESENTATION this block emits native +``inference_models.ClassificationPrediction`` objects under +``TENSOR_NATIVE_CLASSIFICATION_PREDICTION_KIND`` instead of the legacy +classification response dict. ``images_metadata[0]`` carries the anomaly +verdict (``anomaly_score``, ``anomaly_threshold``, ``anomaly_scale``, +``is_anomalous``) next to +the usual producer contract, and ``serialise_native_classification`` writes +those keys into the flag-off dict shape. + +- LOCAL: ``ModelsProvider.run_tensor_native_inference`` returns ONE batched + ``ClassificationPrediction`` whose ``images_metadata[i]`` holds the per-image + verdict and, when requested, the raw ``anomaly_map`` ndarray. The block fans + it out into single-row predictions and moves the map to its own output. +- REMOTE: the ``AnomalyDetectionResponse`` dicts are rebuilt into native + predictions; the map arrives decoded by ``inference_sdk`` or as the wire + payload. +""" + +import uuid +from time import perf_counter +from typing import Any, Dict, List, Literal, Optional, Type, Union + +import numpy as np +import torch +from pydantic import ConfigDict, Field +from roboflow_workflows.core_steps.common.anomaly_detection import ( + ANOMALOUS_CLASS_NAME, + ANOMALY_DECISION_THRESHOLD_KEY, + ANOMALY_MAP_KEY, + ANOMALY_METADATA_KEYS, + ANOMALY_SCALE_KEY, + ANOMALY_SCORE_KEY, + ANOMALY_THRESHOLD_KEY, + IS_ANOMALOUS_KEY, + NORMAL_CLASS_NAME, + apply_threshold_override, + coerce_anomaly_map, + decision_threshold, + redecide_at_threshold, + require_anomaly_response, + validate_threshold_override, +) +from roboflow_workflows.core_steps.common.entities import StepExecutionMode +from roboflow_workflows.environment import ( + HOSTED_CLASSIFICATION_URL, + LOCAL_INFERENCE_API_URL, + WORKFLOWS_IMAGE_TENSOR_DEVICE, + WORKFLOWS_REMOTE_API_KEY_TRANSPORT, + WORKFLOWS_REMOTE_API_TARGET, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, +) +from roboflow_workflows.execution_engine.constants import ( + CLASS_NAMES_KEY, + CLASSIFICATION_STYLE_KEY, + CLASSIFICATION_STYLE_MODEL, + IMAGE_DIMENSIONS_KEY, + INFERENCE_ID_KEY, + PARENT_ID_KEY, + PREDICTION_TYPE_KEY, + ROOT_PARENT_ID_KEY, +) +from roboflow_workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from roboflow_workflows.execution_engine.entities.tensor_native_types import ( + TENSOR_NATIVE_CLASSIFICATION_PREDICTION_KIND, +) +from roboflow_workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + FLOAT_KIND, + FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, + INFERENCE_ID_KIND, + NUMPY_ARRAY_KIND, + ROBOFLOW_MODEL_ID_KIND, + ROBOFLOW_PROJECT_KIND, + FloatZeroToOne, + ImageInputField, + RoboflowModelField, + Selector, +) +from roboflow_workflows.execution_engine.entities.workload import ( + Discovery, + RuntimeRestriction, + WorkOperation, +) +from roboflow_workflows.prototypes.block import ( + BlockResult, + DependentResource, + WorkflowBlock, + WorkflowBlockManifest, + roboflow_platform_model, + roboflow_platform_project, +) +from roboflow_workflows.prototypes.models_provider import ModelsProvider + +from inference_models.models.base.classification import ClassificationPrediction +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + +PREDICTION_TYPE = "classification" + +LONG_DESCRIPTION = """ +Run an anomaly detection model (PatchCore or FoundAD) trained on an anomaly +detection project hosted on Roboflow. + +An anomaly model learns what *normal* looks like from normal images only and +scores a new image by how far it departs from that. The block reports: + +- `predictions`: the classification-style response, with `top` being `normal` + or `anomalous` and the two class confidences. The anomalous confidence is + `sigmoid((anomaly_score - anomaly_threshold) / scale)`, with the scale saved + with the model, so it is exactly 0.5 at the threshold. +- `anomaly_score`: the raw image score (unbounded, larger means more anomalous). +- `anomaly_threshold`: the threshold in effect for this image (saved with the + model, or the override). +- `is_anomalous`: whether the anomalous confidence reached `confidence`. +- `anomaly_decision_threshold`: the raw score at which the anomalous confidence + equals `confidence` (`anomaly_threshold` itself at 0.5). Connect it to the + heatmap's `anomaly_threshold` so the middle of its ramp marks the decision. +- `anomaly_map`: the local anomaly evidence as a float32 `(height, width)` + array at the network's input resolution (square), in the same units as + `anomaly_score`. The map covers the whole image as the network saw it, so + stretching it to the image size overlays it. Connect it to the **Anomaly + Heatmap Visualization** block to draw it. Set `include_anomaly_map` to + `False` to skip the map when only the verdict is needed. + +`confidence` sets the operating point without retraining: the default 0.5 +reproduces the decision calibrated during training; lower it to catch more +defects at the cost of more false alarms, raise it to do the opposite. +`anomaly_threshold` replaces the saved threshold on the raw score; the class +confidences are then recomputed relative to it and `confidence` applies to them +the same way. + +You will need to set your Roboflow API key in your Inference environment to use +this block. To learn more about setting your Roboflow API key, [refer to the +Inference documentation](https://inference.roboflow.com/quickstart/configure_api_key/). +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Anomaly Detection Model", + "version": "v1", + "short_description": "Score an image against a model of normal images and locate what differs.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + "search_keywords": [ + "anomaly", + "anomaly detection", + "defect", + "PatchCore", + "FoundAD", + "heatmap", + ], + "ui_manifest": { + "section": "model", + "icon": "far fa-magnifying-glass-chart", + "blockPriority": 2.5, + "inference": True, + }, + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/anomaly_detection_model@v1"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + include_anomaly_map: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Return the local anomaly map at the network input resolution. Disable it when only the verdict and score are needed.", + examples=[True, "$inputs.include_anomaly_map"], + ) + confidence: Union[ + FloatZeroToOne, + Selector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.5, + description="Minimum anomalous confidence for an anomalous verdict. 0.5 reproduces the decision calibrated during training; lower it to flag more images as anomalous.", + examples=[0.5, "$inputs.confidence"], + ) + anomaly_threshold: Union[Optional[float], Selector(kind=[FLOAT_KIND])] = Field( + default=None, + description="Threshold on the raw anomaly score, replacing the one saved with the model; the anomalous confidence is recomputed relative to it. Leave empty to use the saved threshold.", + examples=[1.25, "$inputs.anomaly_threshold"], + ) + disable_active_learning: Union[bool, Selector(kind=[BOOLEAN_KIND])] = Field( + default=True, + description="Boolean flag to disable project-level active learning for this block.", + examples=[True, "$inputs.disable_active_learning"], + ) + active_learning_target_dataset: Union[ + Selector(kind=[ROBOFLOW_PROJECT_KIND]), Optional[str] + ] = Field( + default=None, + description="Target dataset for active learning, if enabled.", + examples=["my_project", "$inputs.al_target_project"], + ) + + @classmethod + def get_compatible_task_types(cls) -> Optional[List[str]]: + # Anomaly models are registered under the classification task. + return ["classification"] + + def discover_dependent_resources(self) -> Optional[List[DependentResource]]: + resources = [roboflow_platform_model(model_id=self.model_id)] + if self.disable_active_learning is True: + return resources + if self.active_learning_target_dataset is not None: + resources.append( + roboflow_platform_project( + project_url=self.active_learning_target_dataset + ) + ) + return resources + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="predictions", + kind=[TENSOR_NATIVE_CLASSIFICATION_PREDICTION_KIND], + ), + OutputDefinition(name=ANOMALY_SCORE_KEY, kind=[FLOAT_KIND]), + OutputDefinition(name=ANOMALY_THRESHOLD_KEY, kind=[FLOAT_KIND]), + OutputDefinition(name=ANOMALY_DECISION_THRESHOLD_KEY, kind=[FLOAT_KIND]), + OutputDefinition(name=IS_ANOMALOUS_KEY, kind=[BOOLEAN_KIND]), + OutputDefinition(name=ANOMALY_MAP_KEY, kind=[NUMPY_ARRAY_KIND]), + OutputDefinition(name=INFERENCE_ID_KEY, kind=[INFERENCE_ID_KIND]), + OutputDefinition(name="model_id", kind=[ROBOFLOW_MODEL_ID_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + def discover_work_operations(self) -> List[WorkOperation]: + return [WorkOperation.MODEL_INFERENCE] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + +class AnomalyDetectionModelBlockV1(WorkflowBlock): + + def __init__( + self, + model_manager: ModelsProvider, + api_key: Optional[str], + step_execution_mode: StepExecutionMode, + ): + self._model_manager = model_manager + self._api_key = api_key + self._step_execution_mode = step_execution_mode + + @classmethod + def get_init_parameters(cls) -> List[str]: + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id: str, + include_anomaly_map: bool, + confidence: float, + anomaly_threshold: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + threshold_override = validate_threshold_override(anomaly_threshold) + if self._step_execution_mode is StepExecutionMode.LOCAL: + return self.run_locally( + images=images, + model_id=model_id, + include_anomaly_map=include_anomaly_map, + confidence=confidence, + anomaly_threshold=threshold_override, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + elif self._step_execution_mode is StepExecutionMode.REMOTE: + return self.run_remotely( + images=images, + model_id=model_id, + include_anomaly_map=include_anomaly_map, + confidence=confidence, + anomaly_threshold=threshold_override, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + else: + raise ValueError( + f"Unknown step execution mode: {self._step_execution_mode}" + ) + + def run_locally( + self, + images: Batch[WorkflowImageData], + model_id: str, + include_anomaly_map: bool, + confidence: float, + anomaly_threshold: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + # Feed the representation already materialised on the images: GPU tensors + # (RGB) only when every image in the batch has one, else numpy (BGR). + if all(image.is_tensor_materialised() for image in images): + model_inputs = [image.tensor_image for image in images] + image_color_format = "rgb" + else: + model_inputs = [image.numpy_image for image in images] + image_color_format = "bgr" + self._model_manager.add_model(model_id=model_id, api_key=self._api_key) + inference_start = perf_counter() + batched_prediction: ClassificationPrediction = ( + self._model_manager.run_tensor_native_inference( + model_id=model_id, + images=model_inputs, + input_color_format=image_color_format, + include_anomaly_map=include_anomaly_map, + confidence=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + ) + ) + inference_time = perf_counter() - inference_start + class_names = _class_names_map(self._model_manager.get_class_names(model_id)) + model_metadata = batched_prediction.images_metadata or [] + if len(model_metadata) != len(images): + raise ValueError( + f"Model {model_id!r} did not return an anomaly detection prediction: " + f"expected per-image metadata for {len(images)} images, got " + f"{len(model_metadata)}. The Anomaly Detection Model block needs a " + f"PatchCore or FoundAD model." + ) + results: List[dict] = [] + for index, (image, image_model_metadata) in enumerate( + zip(images, model_metadata) + ): + require_anomaly_response(image_model_metadata, model_id=model_id) + verdict = {key: image_model_metadata[key] for key in ANOMALY_METADATA_KEYS} + confidence_row = batched_prediction.confidence[index : index + 1] + if anomaly_threshold is not None: + anomalous = redecide_at_threshold( + verdict, anomaly_threshold, confidence + ) + confidence_row = torch.tensor( + [[1 - anomalous, anomalous]], dtype=torch.float32 + ) + anomaly_map = coerce_anomaly_map(image_model_metadata.get(ANOMALY_MAP_KEY)) + inference_id = str(uuid.uuid4()) + image_metadata = _build_image_metadata( + image=image, + class_names=class_names, + inference_id=inference_id, + verdict=verdict, + inference_time=inference_time, + ) + prediction = ClassificationPrediction( + class_id=_decided_class_id(verdict).to(WORKFLOWS_IMAGE_TENSOR_DEVICE), + confidence=confidence_row.to(WORKFLOWS_IMAGE_TENSOR_DEVICE), + images_metadata=[image_metadata], + ) + results.append( + _result( + prediction=prediction, + verdict=verdict, + confidence=confidence, + anomaly_map=anomaly_map, + inference_id=inference_id, + model_id=model_id, + ) + ) + return results + + def run_remotely( + self, + images: Batch[Optional[WorkflowImageData]], + model_id: str, + include_anomaly_map: bool, + confidence: float, + anomaly_threshold: Optional[float], + disable_active_learning: Optional[bool], + active_learning_target_dataset: Optional[str], + ) -> BlockResult: + api_url = ( + LOCAL_INFERENCE_API_URL + if WORKFLOWS_REMOTE_API_TARGET != "hosted" + else HOSTED_CLASSIFICATION_URL + ) + client = InferenceHTTPClient( + api_url=api_url, + api_key=self._api_key, + ) + if WORKFLOWS_REMOTE_API_TARGET == "hosted": + client.select_api_v0() + client_config = InferenceConfiguration( + api_key_transport=WORKFLOWS_REMOTE_API_KEY_TRANSPORT, + include_anomaly_map=include_anomaly_map, + confidence_threshold=confidence, + disable_active_learning=disable_active_learning, + active_learning_target_dataset=active_learning_target_dataset, + max_batch_size=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_BATCH_SIZE, + max_concurrent_requests=WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS, + source="workflow-execution", + ) + client.configure(inference_configuration=client_config) + non_empty_inference_images = [i.base64_image for i in images] + responses = client.infer( + inference_input=non_empty_inference_images, + model_id=model_id, + ) + if not isinstance(responses, list): + responses = [responses] + results: List[dict] = [] + for image, response in zip(images, responses): + require_anomaly_response(response, model_id=model_id) + anomaly_map = coerce_anomaly_map(response.pop(ANOMALY_MAP_KEY, None)) + response = apply_threshold_override(response, anomaly_threshold, confidence) + verdict = {key: response[key] for key in ANOMALY_METADATA_KEYS} + inference_id = response.get(INFERENCE_ID_KEY) or str(uuid.uuid4()) + prediction = _native_prediction_from_response( + image=image, + response=response, + verdict=verdict, + inference_id=inference_id, + ) + results.append( + _result( + prediction=prediction, + verdict=verdict, + confidence=confidence, + anomaly_map=anomaly_map, + inference_id=inference_id, + model_id=model_id, + ) + ) + return results + + +def _class_names_map(class_names: List[str]) -> Dict[int, str]: + return {index: name for index, name in enumerate(class_names)} + + +def _decided_class_id(verdict: Dict[str, Any]) -> torch.Tensor: + # The two-class convention of the anomaly models: 0 is normal, 1 is anomalous. + class_id = 1 if bool(verdict[IS_ANOMALOUS_KEY]) else 0 + return torch.tensor([class_id], dtype=torch.long) + + +def _result( + prediction: ClassificationPrediction, + verdict: Dict[str, Any], + confidence: float, + anomaly_map: Optional[np.ndarray], + inference_id: str, + model_id: str, +) -> dict: + return { + "predictions": prediction, + ANOMALY_SCORE_KEY: float(verdict[ANOMALY_SCORE_KEY]), + ANOMALY_THRESHOLD_KEY: float(verdict[ANOMALY_THRESHOLD_KEY]), + ANOMALY_DECISION_THRESHOLD_KEY: decision_threshold( + threshold=float(verdict[ANOMALY_THRESHOLD_KEY]), + scale=float(verdict[ANOMALY_SCALE_KEY]), + confidence=confidence, + ), + IS_ANOMALOUS_KEY: bool(verdict[IS_ANOMALOUS_KEY]), + ANOMALY_MAP_KEY: anomaly_map, + INFERENCE_ID_KEY: inference_id, + "model_id": model_id, + } + + +def _build_image_metadata( + image: WorkflowImageData, + class_names: Dict[int, str], + inference_id: str, + verdict: Dict[str, Any], + inference_time: Optional[float] = None, +) -> dict: + height, width = image._read_shape_without_materialization() + metadata = { + CLASS_NAMES_KEY: class_names, + CLASSIFICATION_STYLE_KEY: CLASSIFICATION_STYLE_MODEL, + PREDICTION_TYPE_KEY: PREDICTION_TYPE, + IMAGE_DIMENSIONS_KEY: [height, width], + INFERENCE_ID_KEY: inference_id, + PARENT_ID_KEY: image.parent_metadata.parent_id, + ROOT_PARENT_ID_KEY: image.workflow_root_ancestor_metadata.parent_id, + # No classification confidence cutoff: an anomaly response always reports + # both classes, whatever their confidence. + ANOMALY_SCORE_KEY: float(verdict[ANOMALY_SCORE_KEY]), + ANOMALY_THRESHOLD_KEY: float(verdict[ANOMALY_THRESHOLD_KEY]), + ANOMALY_SCALE_KEY: float(verdict[ANOMALY_SCALE_KEY]), + IS_ANOMALOUS_KEY: bool(verdict[IS_ANOMALOUS_KEY]), + } + if inference_time is not None: + metadata["time"] = inference_time + return metadata + + +def _native_prediction_from_response( + image: WorkflowImageData, + response: dict, + verdict: Dict[str, Any], + inference_id: str, +) -> ClassificationPrediction: + """Rebuild a native single-row ``ClassificationPrediction`` from an + ``AnomalyDetectionResponse`` dict. Both classes are always present in the + response, so the dense confidence vector is complete.""" + class_names: Dict[int, str] = {} + confidence_by_id: Dict[int, float] = {} + for entry in response.get("predictions", []) or []: + class_id = int(entry["class_id"]) + class_names[class_id] = str(entry.get("class", class_id)) + confidence_by_id[class_id] = float(entry.get("confidence", 0.0)) + num_classes = (max(class_names.keys()) + 1) if class_names else 2 + for class_id, default_name in enumerate((NORMAL_CLASS_NAME, ANOMALOUS_CLASS_NAME)): + class_names.setdefault(class_id, default_name) + for class_id in range(num_classes): + class_names.setdefault(class_id, str(class_id)) + confidence_vector = [confidence_by_id.get(i, 0.0) for i in range(num_classes)] + image_metadata = _build_image_metadata( + image=image, + class_names=class_names, + inference_id=inference_id, + verdict=verdict, + inference_time=response.get("time"), + ) + return ClassificationPrediction( + class_id=_decided_class_id(verdict).to(WORKFLOWS_IMAGE_TENSOR_DEVICE), + confidence=torch.tensor( + [confidence_vector], + dtype=torch.float32, + device=WORKFLOWS_IMAGE_TENSOR_DEVICE, + ), + images_metadata=[image_metadata], + ) diff --git a/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/__init__.py b/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/v1.py b/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/v1.py new file mode 100644 index 0000000000..df755c53ff --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/v1.py @@ -0,0 +1,202 @@ +from typing import List, Literal, Optional, Type, Union + +import numpy as np +from pydantic import ConfigDict, Field +from roboflow_workflows.core_steps.visualizations.common.annotators.anomaly_heatmap import ( + AnomalyHeatmapAnnotator, +) +from roboflow_workflows.core_steps.visualizations.common.base import ( + OUTPUT_IMAGE_KEY, + VisualizationBlock, + VisualizationManifest, +) +from roboflow_workflows.execution_engine.entities.base import WorkflowImageData +from roboflow_workflows.execution_engine.entities.types import ( + FLOAT_KIND, + FLOAT_ZERO_TO_ONE_KIND, + NUMPY_ARRAY_KIND, + STRING_KIND, + FloatZeroToOne, + Selector, +) +from roboflow_workflows.execution_engine.entities.workload import ( + Discovery, + RuntimeRestriction, + WorkOperation, +) +from roboflow_workflows.prototypes.block import ( + BlockResult, + DependentResource, + WorkflowBlockManifest, +) + +TYPE: str = "roboflow_core/anomaly_heatmap_visualization@v1" +SHORT_DESCRIPTION = ( + "Overlay an anomaly detection model's anomaly map on the image as a heatmap." +) +LONG_DESCRIPTION = """ +Draw the anomaly map of an **Anomaly Detection Model** (PatchCore or FoundAD) over +the image it was computed for, so the parts of the image that depart from what the +model learned as normal light up. + +## How This Block Works + +1. Takes the image and the model's `anomaly_map` and `anomaly_threshold` as input +2. Stretches the map (served at the network's square input resolution) onto the image +3. Places every map value on a fixed scale anchored to the threshold: + `t = value / (value + threshold)`, so zero is transparent, the threshold sits at + the middle of the ramp, and the ramp saturates as the evidence grows +4. Colours the scale with the chosen `color_map` and blends it over the image at + the chosen `opacity` + +The scale depends only on the model's threshold, never on the image, so the same +colour means the same evidence on every frame and normal images show their map too. +The default `ROBOFLOW` ramp is the yellow-to-red ramp the Roboflow app draws in its +model preview, with transparency that fades out where the evidence is low. The +OpenCV ramps (`JET`, `TURBO`, `INFERNO`, `VIRIDIS`, `HOT`) are opaque colour maps +over the same scale; use `opacity` to see the image through them. + +Pass the model block's `anomaly_decision_threshold` output so the middle of the ramp +marks the block's decision (its threshold and `confidence`); at the default confidence +of 0.5 it equals `anomaly_threshold`. + +## Common Use Cases + +- **Defect localisation**: show an operator where an anomalous part differs from normal +- **Model review**: compare what the model reacts to against the labelled defect +- **Reports and alerts**: attach the heatmap to a notification or dataset upload + +## Connecting to Other Blocks + +The annotated image can be connected to other visualization blocks (for example the +Classification Label Visualization to print the verdict on top), data storage blocks, +webhook and notification blocks, or video outputs. +""" + + +class AnomalyHeatmapManifest(VisualizationManifest): + type: Literal[f"{TYPE}"] + model_config = ConfigDict( + json_schema_extra={ + "name": "Anomaly Heatmap Visualization", + "version": "v1", + "short_description": SHORT_DESCRIPTION, + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "visualization", + "search_keywords": [ + "anomaly", + "anomaly detection", + "heatmap", + "defect", + "PatchCore", + "FoundAD", + ], + "ui_manifest": { + "section": "visualization", + "icon": "far fa-fire", + "blockPriority": 3, + "warnings": [ + { + "property": "copy_image", + "value": False, + "message": "This setting will mutate its input image. If the input is used by other blocks, it may cause unexpected behavior.", + } + ], + }, + } + ) + + anomaly_map: Selector(kind=[NUMPY_ARRAY_KIND]) = Field( # type: ignore + description="The anomaly map from the Anomaly Detection Model block: raw local anomaly evidence at the network input resolution.", + examples=["$steps.anomaly_model.anomaly_map"], + ) + anomaly_threshold: Union[float, Selector(kind=[FLOAT_KIND])] = Field( # type: ignore + description="The raw anomaly score that anchors the colour ramp; it maps to the middle of the ramp. Connect the model block's anomaly_decision_threshold output so the ramp follows the block's decision.", + examples=["$steps.anomaly_model.anomaly_decision_threshold", 1.25], + ) + color_map: Union[ + Literal["ROBOFLOW", "JET", "TURBO", "INFERNO", "VIRIDIS", "HOT"], + Selector(kind=[STRING_KIND]), + ] = Field( # type: ignore + default="ROBOFLOW", + description="Colour ramp for the heatmap. ROBOFLOW is the Roboflow app's yellow-to-red ramp, transparent where the evidence is low; the others are opaque OpenCV colour maps.", + examples=["ROBOFLOW", "JET", "$inputs.color_map"], + ) + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + default=0.75, + description="Opacity of the heatmap overlay, from 0.0 (invisible) to 1.0 (the ramp's own transparency only).", + examples=[0.75, "$inputs.opacity"], + ) + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + def discover_work_operations(self) -> List[WorkOperation]: + return [WorkOperation.VISUALIZATION] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + def discover_dependent_resources(self) -> List[DependentResource]: + return [] + + +class AnomalyHeatmapVisualizationBlockV1(VisualizationBlock): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.annotatorCache = {} + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return AnomalyHeatmapManifest + + def getAnnotator( + self, + color_map: str, + opacity: float, + ) -> AnomalyHeatmapAnnotator: + key = "_".join(map(str, [color_map, opacity])) + if key not in self.annotatorCache: + self.annotatorCache[key] = AnomalyHeatmapAnnotator( + color_map=color_map, + opacity=opacity, + ) + return self.annotatorCache[key] + + def run( + self, + image: WorkflowImageData, + copy_image: bool, + anomaly_map: Optional[np.ndarray], + anomaly_threshold: float, + color_map: str, + opacity: float, + ) -> BlockResult: + if anomaly_map is None: + raise ValueError( + "Anomaly Heatmap Visualization received no anomaly map. Enable " + "`include_anomaly_map` on the Anomaly Detection Model block that " + "feeds this step." + ) + annotator = self.getAnnotator(color_map, opacity) + scene = image.numpy_image + if copy_image: + scene = scene.copy() + else: + image.declare_numpy_image_mutated() + annotated_image = annotator.annotate( + scene=scene, + anomaly_map=np.asarray(anomaly_map, dtype=np.float32), + threshold=float(anomaly_threshold), + ) + return { + OUTPUT_IMAGE_KEY: WorkflowImageData.copy_and_replace( + origin_image_data=image, numpy_image=annotated_image + ) + } diff --git a/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/v1_tensor.py b/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/v1_tensor.py new file mode 100644 index 0000000000..f5a0cbcc58 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/visualizations/anomaly_heatmap/v1_tensor.py @@ -0,0 +1,216 @@ +"""Tensor-native sibling of `roboflow_core/anomaly_heatmap_visualization@v1`. + +Under ENABLE_TENSOR_DATA_REPRESENTATION the `numpy_array` kind may carry a +``torch.Tensor``; the map is materialised to numpy and drawn with the same +annotator as the numpy sibling. +""" + +from typing import Any, List, Literal, Optional, Type, Union + +import numpy as np +import torch +from pydantic import ConfigDict, Field +from roboflow_workflows.core_steps.visualizations.common.annotators.anomaly_heatmap import ( + AnomalyHeatmapAnnotator, +) +from roboflow_workflows.core_steps.visualizations.common.base_tensor import ( + OUTPUT_IMAGE_KEY, + VisualizationBlock, + VisualizationManifest, +) +from roboflow_workflows.execution_engine.entities.base import WorkflowImageData +from roboflow_workflows.execution_engine.entities.types import ( + FLOAT_KIND, + FLOAT_ZERO_TO_ONE_KIND, + NUMPY_ARRAY_KIND, + STRING_KIND, + FloatZeroToOne, + Selector, +) +from roboflow_workflows.execution_engine.entities.workload import ( + Discovery, + RuntimeRestriction, + WorkOperation, +) +from roboflow_workflows.prototypes.block import ( + BlockResult, + DependentResource, + WorkflowBlockManifest, +) + +TYPE: str = "roboflow_core/anomaly_heatmap_visualization@v1" +SHORT_DESCRIPTION = ( + "Overlay an anomaly detection model's anomaly map on the image as a heatmap." +) +LONG_DESCRIPTION = """ +Draw the anomaly map of an **Anomaly Detection Model** (PatchCore or FoundAD) over +the image it was computed for, so the parts of the image that depart from what the +model learned as normal light up. + +## How This Block Works + +1. Takes the image and the model's `anomaly_map` and `anomaly_threshold` as input +2. Stretches the map (served at the network's square input resolution) onto the image +3. Places every map value on a fixed scale anchored to the threshold: + `t = value / (value + threshold)`, so zero is transparent, the threshold sits at + the middle of the ramp, and the ramp saturates as the evidence grows +4. Colours the scale with the chosen `color_map` and blends it over the image at + the chosen `opacity` + +The scale depends only on the model's threshold, never on the image, so the same +colour means the same evidence on every frame and normal images show their map too. +The default `ROBOFLOW` ramp is the yellow-to-red ramp the Roboflow app draws in its +model preview, with transparency that fades out where the evidence is low. The +OpenCV ramps (`JET`, `TURBO`, `INFERNO`, `VIRIDIS`, `HOT`) are opaque colour maps +over the same scale; use `opacity` to see the image through them. + +Pass the model block's `anomaly_decision_threshold` output so the middle of the ramp +marks the block's decision (its threshold and `confidence`); at the default confidence +of 0.5 it equals `anomaly_threshold`. + +## Common Use Cases + +- **Defect localisation**: show an operator where an anomalous part differs from normal +- **Model review**: compare what the model reacts to against the labelled defect +- **Reports and alerts**: attach the heatmap to a notification or dataset upload + +## Connecting to Other Blocks + +The annotated image can be connected to other visualization blocks (for example the +Classification Label Visualization to print the verdict on top), data storage blocks, +webhook and notification blocks, or video outputs. +""" + + +class AnomalyHeatmapManifest(VisualizationManifest): + type: Literal[f"{TYPE}"] + model_config = ConfigDict( + json_schema_extra={ + "name": "Anomaly Heatmap Visualization", + "version": "v1", + "short_description": SHORT_DESCRIPTION, + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "visualization", + "search_keywords": [ + "anomaly", + "anomaly detection", + "heatmap", + "defect", + "PatchCore", + "FoundAD", + ], + "ui_manifest": { + "section": "visualization", + "icon": "far fa-fire", + "blockPriority": 3, + "warnings": [ + { + "property": "copy_image", + "value": False, + "message": "This setting will mutate its input image. If the input is used by other blocks, it may cause unexpected behavior.", + } + ], + }, + } + ) + + anomaly_map: Selector(kind=[NUMPY_ARRAY_KIND]) = Field( # type: ignore + description="The anomaly map from the Anomaly Detection Model block: raw local anomaly evidence at the network input resolution.", + examples=["$steps.anomaly_model.anomaly_map"], + ) + anomaly_threshold: Union[float, Selector(kind=[FLOAT_KIND])] = Field( # type: ignore + description="The raw anomaly score that anchors the colour ramp; it maps to the middle of the ramp. Connect the model block's anomaly_decision_threshold output so the ramp follows the block's decision.", + examples=["$steps.anomaly_model.anomaly_decision_threshold", 1.25], + ) + color_map: Union[ + Literal["ROBOFLOW", "JET", "TURBO", "INFERNO", "VIRIDIS", "HOT"], + Selector(kind=[STRING_KIND]), + ] = Field( # type: ignore + default="ROBOFLOW", + description="Colour ramp for the heatmap. ROBOFLOW is the Roboflow app's yellow-to-red ramp, transparent where the evidence is low; the others are opaque OpenCV colour maps.", + examples=["ROBOFLOW", "JET", "$inputs.color_map"], + ) + opacity: Union[FloatZeroToOne, Selector(kind=[FLOAT_ZERO_TO_ONE_KIND])] = Field( # type: ignore + default=0.75, + description="Opacity of the heatmap overlay, from 0.0 (invisible) to 1.0 (the ramp's own transparency only).", + examples=[0.75, "$inputs.opacity"], + ) + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + def discover_work_operations(self) -> List[WorkOperation]: + return [WorkOperation.VISUALIZATION] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + def discover_dependent_resources(self) -> List[DependentResource]: + return [] + + +class AnomalyHeatmapVisualizationBlockV1(VisualizationBlock): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.annotatorCache = {} + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return AnomalyHeatmapManifest + + def getAnnotator( + self, + color_map: str, + opacity: float, + ) -> AnomalyHeatmapAnnotator: + key = "_".join(map(str, [color_map, opacity])) + if key not in self.annotatorCache: + self.annotatorCache[key] = AnomalyHeatmapAnnotator( + color_map=color_map, + opacity=opacity, + ) + return self.annotatorCache[key] + + def run( + self, + image: WorkflowImageData, + copy_image: bool, + anomaly_map: Optional[Any], + anomaly_threshold: float, + color_map: str, + opacity: float, + ) -> BlockResult: + if anomaly_map is None: + raise ValueError( + "Anomaly Heatmap Visualization received no anomaly map. Enable " + "`include_anomaly_map` on the Anomaly Detection Model block that " + "feeds this step." + ) + annotator = self.getAnnotator(color_map, opacity) + scene = image.numpy_image + if copy_image: + scene = scene.copy() + else: + image.declare_numpy_image_mutated() + annotated_image = annotator.annotate( + scene=scene, + anomaly_map=_as_numpy_map(anomaly_map), + threshold=float(anomaly_threshold), + ) + return { + OUTPUT_IMAGE_KEY: WorkflowImageData.copy_and_replace( + origin_image_data=image, numpy_image=annotated_image + ) + } + + +def _as_numpy_map(anomaly_map: Any) -> np.ndarray: + if isinstance(anomaly_map, torch.Tensor): + return anomaly_map.detach().cpu().numpy().astype(np.float32, copy=False) + return np.asarray(anomaly_map, dtype=np.float32) diff --git a/workflows/roboflow_workflows/core_steps/visualizations/common/annotators/anomaly_heatmap.py b/workflows/roboflow_workflows/core_steps/visualizations/common/annotators/anomaly_heatmap.py new file mode 100644 index 0000000000..522a59897f --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/visualizations/common/annotators/anomaly_heatmap.py @@ -0,0 +1,177 @@ +"""Colourise an anomaly map and blend it over an image. + +Anomaly map values share the units of the model's anomaly score and are +unbounded above, so every value is placed on a fixed saturating scale anchored +to the model's threshold: ``t = v / (v + threshold)``. Zero is transparent, the +threshold sits at the ramp's midpoint, and the ramp saturates as values grow. A +threshold at or below zero (a decision threshold at a very low confidence) is the +limit of that scale: every positive value is past it, so the ramp saturates. +The scale depends only on the model, never on the image, so the same colour +means the same evidence on every image and normal images show their map too. +This is local anomaly evidence, not a defect mask. + +The ``ROBOFLOW`` ramp reproduces the stops the Roboflow app draws in its +inference preview; the other ramps are OpenCV colour maps over the same scale. +""" + +from typing import Dict, List, Tuple + +import cv2 +import numpy as np + +ROBOFLOW_COLOR_MAP = "ROBOFLOW" + +# Stops on the [0, 1) scale, where 0.5 is the saved threshold: (position, (r, g, b), alpha). +ROBOFLOW_COLOR_STOPS: List[Tuple[float, Tuple[int, int, int], float]] = [ + (0.0, (255, 214, 0), 0.0), + (0.25, (255, 214, 0), 0.55), + (0.5, (255, 122, 0), 0.85), + (1.0, (220, 30, 30), 1.0), +] + +OPENCV_COLOR_MAPS: Dict[str, int] = { + "JET": cv2.COLORMAP_JET, + "TURBO": cv2.COLORMAP_TURBO, + "INFERNO": cv2.COLORMAP_INFERNO, + "VIRIDIS": cv2.COLORMAP_VIRIDIS, + "HOT": cv2.COLORMAP_HOT, +} + +SUPPORTED_COLOR_MAPS = (ROBOFLOW_COLOR_MAP, *OPENCV_COLOR_MAPS) + + +def normalize_anomaly_map(anomaly_map: np.ndarray, threshold: float) -> np.ndarray: + """Map raw anomaly values onto ``[0, 1]``: 0 at or below zero, 0.5 at the threshold. + + Args: + anomaly_map (np.ndarray): Raw map in the units of the anomaly score. + threshold (float): The decision threshold anchoring the ramp. At or below + zero every positive value is past it and maps to 1, the limit of + ``v / (v + threshold)`` as the threshold falls to zero. + + Returns: + A float32 array of the same shape with values in ``[0, 1]``. + + Raises: + ValueError: If ``threshold`` is not finite. + """ + if not np.isfinite(threshold): + raise ValueError( + f"Anomaly threshold must be finite to anchor the heatmap ramp, " + f"got {threshold!r}" + ) + positive = np.maximum(anomaly_map.astype(np.float32, copy=False), 0.0) + if threshold <= 0: + return (positive > 0).astype(np.float32) + normalized = positive / (positive + np.float32(threshold)) + return normalized.astype(np.float32, copy=False) + + +def roboflow_ramp(normalized: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Colour a normalised map with the Roboflow app's yellow-to-red ramp. + + Args: + normalized (np.ndarray): Values in ``[0, 1]`` (clipped otherwise). + + Returns: + ``(bgr, alpha)``: a uint8 ``(h, w, 3)`` BGR image and a float32 ``(h, w)`` + alpha in ``[0, 1]`` interpolated from the ramp stops. + """ + t = np.clip(normalized, 0.0, 1.0).astype(np.float32, copy=False) + positions = [stop[0] for stop in ROBOFLOW_COLOR_STOPS] + channels = [] + for channel_index in range(3): + channel_values = [stop[1][channel_index] for stop in ROBOFLOW_COLOR_STOPS] + channels.append(np.interp(t, positions, channel_values)) + alphas = [stop[2] for stop in ROBOFLOW_COLOR_STOPS] + alpha = np.interp(t, positions, alphas).astype(np.float32) + rgb = np.stack(channels, axis=-1) + bgr = np.rint(rgb[..., ::-1]).astype(np.uint8) + return bgr, alpha + + +def colorize_anomaly_map( + normalized: np.ndarray, color_map: str +) -> Tuple[np.ndarray, np.ndarray]: + """Colour a normalised map with the named ramp. + + Args: + normalized (np.ndarray): Values in ``[0, 1]``. + color_map (str): ``"ROBOFLOW"`` or one of the OpenCV names in + ``OPENCV_COLOR_MAPS``. + + Returns: + ``(bgr, alpha)`` as :func:`roboflow_ramp`. OpenCV ramps are opaque + (alpha 1 everywhere); the blend opacity alone controls their strength. + + Raises: + ValueError: If ``color_map`` is not supported. + """ + name = str(color_map).upper() + if name == ROBOFLOW_COLOR_MAP: + return roboflow_ramp(normalized) + if name not in OPENCV_COLOR_MAPS: + raise ValueError( + f"Unsupported anomaly heatmap color_map {color_map!r}; " + f"expected one of {list(SUPPORTED_COLOR_MAPS)}" + ) + scaled = np.rint(np.clip(normalized, 0.0, 1.0) * 255.0).astype(np.uint8) + bgr = cv2.applyColorMap(scaled, OPENCV_COLOR_MAPS[name]) + alpha = np.ones(normalized.shape, dtype=np.float32) + return bgr, alpha + + +class AnomalyHeatmapAnnotator: + """Blend a colourised anomaly map over a BGR scene.""" + + def __init__(self, color_map: str, opacity: float): + if str(color_map).upper() not in SUPPORTED_COLOR_MAPS: + raise ValueError( + f"Unsupported anomaly heatmap color_map {color_map!r}; " + f"expected one of {list(SUPPORTED_COLOR_MAPS)}" + ) + if not 0.0 <= opacity <= 1.0: + raise ValueError(f"opacity must be within [0, 1], got {opacity!r}") + self.color_map = str(color_map).upper() + self.opacity = float(opacity) + + def annotate( + self, scene: np.ndarray, anomaly_map: np.ndarray, threshold: float + ) -> np.ndarray: + """Draw the map onto ``scene`` in place and return it. + + Args: + scene (np.ndarray): uint8 BGR ``(h, w, 3)`` or grayscale ``(h, w)`` + image; a grayscale scene is promoted to BGR. + anomaly_map (np.ndarray): 2D raw map at any resolution; it is + stretched to the scene with bilinear interpolation. + threshold (float): The threshold anchoring the ramp. + + Returns: + The annotated uint8 BGR image. + + Raises: + ValueError: If the map is not 2D or the threshold is invalid. + """ + anomaly_map = np.asarray(anomaly_map, dtype=np.float32) + if anomaly_map.ndim != 2: + raise ValueError( + f"Anomaly map must be 2D, got shape {tuple(anomaly_map.shape)}" + ) + if scene.ndim == 2: + scene = cv2.cvtColor(scene, cv2.COLOR_GRAY2BGR) + height, width = scene.shape[:2] + + normalized = normalize_anomaly_map(anomaly_map, threshold=threshold) + if normalized.shape != (height, width): + normalized = cv2.resize( + normalized, (width, height), interpolation=cv2.INTER_LINEAR + ) + bgr, alpha = colorize_anomaly_map(normalized, color_map=self.color_map) + + weight = (alpha * self.opacity)[..., None] + blended = ( + scene.astype(np.float32) * (1.0 - weight) + bgr.astype(np.float32) * weight + ) + np.copyto(scene, np.rint(blended).astype(np.uint8)) + return scene diff --git a/workflows/roboflow_workflows/prototypes/models_provider.py b/workflows/roboflow_workflows/prototypes/models_provider.py index 991eb33aec..3c66c5ff0a 100644 --- a/workflows/roboflow_workflows/prototypes/models_provider.py +++ b/workflows/roboflow_workflows/prototypes/models_provider.py @@ -105,6 +105,7 @@ def run_classification( disable_active_learning: Optional[bool] = None, active_learning_target_dataset: Optional[str] = None, inference_kwargs: Optional[Dict[str, Any]] = None, + include_anomaly_map: bool = False, ) -> List[dict]: ... def run_image_embeddings( diff --git a/workflows/tests/unit_tests/core_steps/common/test_anomaly_detection.py b/workflows/tests/unit_tests/core_steps/common/test_anomaly_detection.py new file mode 100644 index 0000000000..035e23bebc --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/common/test_anomaly_detection.py @@ -0,0 +1,156 @@ +import base64 + +import numpy as np +import pytest +from roboflow_workflows.core_steps.common.anomaly_detection import ( + apply_threshold_override, + coerce_anomaly_map, + decision_threshold, + decode_anomaly_map, + require_anomaly_response, + validate_threshold_override, +) + + +def _payload(values: np.ndarray) -> dict: + array = np.ascontiguousarray(values, dtype=" None: + # given + values = np.array([[0.0, 1.5], [-2.25, 1e6]], dtype=np.float32) + + # when + decoded = decode_anomaly_map(_payload(values)) + + # then + assert decoded.dtype == np.float32 + assert decoded.flags.writeable + np.testing.assert_array_equal(decoded, values) + + +@pytest.mark.parametrize( + "broken", + [ + {"shape": [2, 2], "dtype": "float16", "data": ""}, + {"shape": [2], "dtype": "float32", "data": ""}, + {"shape": [2, 2], "dtype": "float32", "data": "not base64!!"}, + { + "shape": [2, 2], + "dtype": "float32", + "data": base64.b64encode(b"\0" * 4).decode(), + }, + ], +) +def test_decode_anomaly_map_rejects_malformed_payloads(broken: dict) -> None: + with pytest.raises(ValueError): + decode_anomaly_map(broken) + + +def test_coerce_anomaly_map_accepts_every_wire_form() -> None: + # given + values = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) + + # when / then + assert coerce_anomaly_map(None) is None + np.testing.assert_array_equal(coerce_anomaly_map(_payload(values)), values) + np.testing.assert_array_equal(coerce_anomaly_map(values), values) + np.testing.assert_array_equal(coerce_anomaly_map(values.tolist()), values) + with pytest.raises(ValueError): + coerce_anomaly_map(np.zeros((2, 2, 2), dtype=np.float32)) + + +def test_require_anomaly_response_names_the_model_and_missing_keys() -> None: + with pytest.raises(ValueError, match="my/1") as error: + require_anomaly_response({"top": "cat", "confidence": 0.9}, model_id="my/1") + assert "anomaly_score" in str(error.value) + + +@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), "0.5", True]) +def test_validate_threshold_override_rejects_non_finite_or_non_numbers(invalid) -> None: + with pytest.raises(ValueError): + validate_threshold_override(invalid) + + +def test_validate_threshold_override_keeps_none_and_casts_ints() -> None: + assert validate_threshold_override(None) is None + assert validate_threshold_override(2) == 2.0 + + +def _normal_response() -> dict: + # The model said normal at its saved threshold of 2.0 with scale 0.5. + return { + "predictions": [ + {"class": "normal", "class_id": 0, "confidence": 0.7311}, + {"class": "anomalous", "class_id": 1, "confidence": 0.2689}, + ], + "top": "normal", + "confidence": 0.7311, + "anomaly_score": 1.5, + "anomaly_threshold": 2.0, + "anomaly_scale": 0.5, + "is_anomalous": False, + } + + +def test_apply_threshold_override_re_decides_and_recomputes_confidences() -> None: + # given + response = _normal_response() + + # when: relative to 1.0 the anomalous confidence is sigmoid(1) ~ 0.7311 + result = apply_threshold_override(response, 1.0, confidence=0.5) + + # then + assert result is response + assert result["is_anomalous"] is True + assert result["anomaly_threshold"] == 1.0 + assert result["top"] == "anomalous" + assert result["confidence"] == 0.7311 + assert result["predictions"] == [ + {"class_id": 1, "class": "anomalous", "confidence": 0.7311}, + {"class_id": 0, "class": "normal", "confidence": 0.2689}, + ] + + +def test_confidence_applies_relative_to_the_threshold_override() -> None: + # when: the same override, but the anomalous confidence must reach 0.8 + result = apply_threshold_override(_normal_response(), 1.0, confidence=0.8) + + # then + assert result["is_anomalous"] is False + assert result["anomaly_threshold"] == 1.0 + assert result["top"] == "normal" + assert result["confidence"] == 0.2689 + + +def test_apply_threshold_override_without_override_is_a_no_op() -> None: + # given + response = _normal_response() + + # when + result = apply_threshold_override(_normal_response(), None, confidence=0.1) + + # then + assert result == response + + +@pytest.mark.parametrize( + "confidence, expected", + [ + (0.5, 2.0), + (0.8, 2.0 + 0.5 * 1.3862944), + (0.0, 2.0 - 0.5 * 60), + (1.0, 2.0 + 0.5 * 60), + ], +) +def test_decision_threshold_is_where_the_anomalous_confidence_meets_confidence( + confidence: float, expected: float +) -> None: + assert decision_threshold(threshold=2.0, scale=0.5, confidence=confidence) == ( + pytest.approx(expected) + ) diff --git a/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/__init__.py b/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_hosted_api_resolution.py b/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_hosted_api_resolution.py new file mode 100644 index 0000000000..1656992121 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_hosted_api_resolution.py @@ -0,0 +1,23 @@ +import pytest + +from tests.unit_tests.core_steps.models.roboflow._hosted_api_resolution import ( + assert_hosted_selects_v0, + assert_non_hosted_uses_local, +) + +FAMILY = "anomaly_detection" +HOSTED_URL_ATTR = "HOSTED_CLASSIFICATION_URL" +BLOCK_CLASSES = { + "v1": "AnomalyDetectionModelBlockV1", + "v1_tensor": "AnomalyDetectionModelBlockV1", +} + + +@pytest.mark.parametrize("version", list(BLOCK_CLASSES)) +def test_hosted_remote_execution_selects_v0_and_hosted_url(version): + assert_hosted_selects_v0(FAMILY, version, BLOCK_CLASSES[version], HOSTED_URL_ATTR) + + +@pytest.mark.parametrize("version", list(BLOCK_CLASSES)) +def test_non_hosted_remote_execution_uses_local_url_and_no_v0(version): + assert_non_hosted_uses_local(FAMILY, version, BLOCK_CLASSES[version]) diff --git a/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_v1.py b/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_v1.py new file mode 100644 index 0000000000..dcde2bd8c8 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_v1.py @@ -0,0 +1,369 @@ +import base64 +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +from pydantic import ValidationError +from roboflow_workflows.core_steps.common.entities import StepExecutionMode +from roboflow_workflows.core_steps.models.roboflow.anomaly_detection import v1 +from roboflow_workflows.core_steps.models.roboflow.anomaly_detection.v1 import ( + AnomalyDetectionModelBlockV1, + BlockManifest, +) +from roboflow_workflows.execution_engine.entities.base import ( + Batch, + ImageParentMetadata, + WorkflowImageData, +) + +TYPE = "roboflow_core/anomaly_detection_model@v1" +MAP = np.array([[0.0, 1.0], [2.0, 4.0]], dtype=np.float32) + + +def _map_payload(values: np.ndarray = MAP) -> dict: + array = np.ascontiguousarray(values, dtype=" Dict[str, Any]: + is_anomalous = score >= threshold + anomalous_confidence = 0.75 if is_anomalous else 0.25 + classes = [ + {"class": "anomalous", "class_id": 1, "confidence": anomalous_confidence}, + {"class": "normal", "class_id": 0, "confidence": 1 - anomalous_confidence}, + ] + if not is_anomalous: + classes.reverse() + response = { + "inference_id": "iid-1", + "time": 0.01, + "image": {"width": 4, "height": 3}, + "predictions": classes, + "top": classes[0]["class"], + "confidence": classes[0]["confidence"], + "anomaly_score": score, + "anomaly_threshold": threshold, + "anomaly_scale": 0.5, + "is_anomalous": is_anomalous, + } + if anomaly_map is not None: + response["anomaly_map"] = anomaly_map + return response + + +def _image(parent_id: str = "img") -> WorkflowImageData: + return WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id=parent_id), + numpy_image=np.zeros((3, 4, 3), dtype=np.uint8), + ) + + +class _StubProvider: + def __init__(self, responses: List[Dict[str, Any]]): + self.responses = responses + self.add_model = MagicMock() + self.calls: List[Dict[str, Any]] = [] + + def run_classification(self, **kwargs: Any) -> List[dict]: + self.calls.append(kwargs) + return [dict(response) for response in self.responses] + + +def _block(provider: Any, mode: StepExecutionMode) -> AnomalyDetectionModelBlockV1: + return AnomalyDetectionModelBlockV1( + model_manager=provider, api_key="key", step_execution_mode=mode + ) + + +def _run( + block: AnomalyDetectionModelBlockV1, images: List[WorkflowImageData], **overrides +): + kwargs = dict( + model_id="ws/anomaly/1", + include_anomaly_map=True, + confidence=0.5, + anomaly_threshold=None, + disable_active_learning=True, + active_learning_target_dataset=None, + ) + kwargs.update(overrides) + return block.run( + images=Batch.init(content=images, indices=[(i,) for i in range(len(images))]), + **kwargs + ) + + +@pytest.mark.parametrize("images_field_alias", ["images", "image"]) +def test_manifest_validation_when_minimal_config_is_given( + images_field_alias: str, +) -> None: + # given + data = { + "type": TYPE, + "name": "anomaly", + images_field_alias: "$inputs.image", + "model_id": "ws/anomaly/1", + } + + # when + result = BlockManifest.model_validate(data) + + # then + assert result == BlockManifest( + type=TYPE, name="anomaly", images="$inputs.image", model_id="ws/anomaly/1" + ) + assert result.include_anomaly_map is True + assert result.confidence == 0.5 + assert result.anomaly_threshold is None + + +@pytest.mark.parametrize( + "field, value", + [ + ("include_anomaly_map", "$inputs.include_map"), + ("include_anomaly_map", False), + ("confidence", "$inputs.confidence"), + ("confidence", 0.3), + ("anomaly_threshold", "$inputs.threshold"), + ("anomaly_threshold", 1.25), + ("anomaly_threshold", None), + ], +) +def test_manifest_accepts_selectors_and_literals(field: str, value: Any) -> None: + # given + data = { + "type": TYPE, + "name": "anomaly", + "images": "$inputs.image", + "model_id": "a/1", + } + data[field] = value + + # when + result = BlockManifest.model_validate(data) + + # then + assert getattr(result, field) == value + + +@pytest.mark.parametrize("field", ["type", "name", "images", "model_id"]) +def test_manifest_validation_when_required_field_is_missing(field: str) -> None: + # given + data = { + "type": TYPE, + "name": "anomaly", + "images": "$inputs.image", + "model_id": "a/1", + } + del data[field] + + # when / then + with pytest.raises(ValidationError): + BlockManifest.model_validate(data) + + +@pytest.mark.parametrize( + "field, value", + [ + ("anomaly_threshold", "high"), + ("confidence", 1.5), + ("include_anomaly_map", "maybe"), + ("model_id", None), + ], +) +def test_manifest_validation_when_field_has_invalid_type( + field: str, value: Any +) -> None: + # given + data = { + "type": TYPE, + "name": "anomaly", + "images": "$inputs.image", + "model_id": "a/1", + } + data[field] = value + + # when / then + with pytest.raises(ValidationError): + BlockManifest.model_validate(data) + + +def test_manifest_outputs_are_declared_in_contract_order() -> None: + names = [output.name for output in BlockManifest.describe_outputs()] + assert names == [ + "predictions", + "anomaly_score", + "anomaly_threshold", + "anomaly_decision_threshold", + "is_anomalous", + "anomaly_map", + "inference_id", + "model_id", + ] + + +def test_run_locally_forwards_the_map_flag_and_decodes_the_payload() -> None: + # given + provider = _StubProvider([_response(anomaly_map=_map_payload())]) + block = _block(provider, StepExecutionMode.LOCAL) + + # when + result = _run(block, [_image()]) + + # then + provider.add_model.assert_called_once_with(model_id="ws/anomaly/1", api_key="key") + assert provider.calls[0]["include_anomaly_map"] is True + assert provider.calls[0]["confidence"] == 0.5 + assert provider.calls[0]["images"] == [ + {"type": "numpy_object", "value": provider.calls[0]["images"][0]["value"]} + ] + assert len(result) == 1 + assert result[0]["anomaly_score"] == 3.0 + assert result[0]["anomaly_threshold"] == 2.0 + assert result[0]["anomaly_decision_threshold"] == 2.0 + assert result[0]["is_anomalous"] is True + assert result[0]["inference_id"] == "iid-1" + assert result[0]["model_id"] == "ws/anomaly/1" + np.testing.assert_array_equal(result[0]["anomaly_map"], MAP) + assert result[0]["anomaly_map"].dtype == np.float32 + predictions = result[0]["predictions"] + assert "anomaly_map" not in predictions + assert predictions["top"] == "anomalous" + assert predictions["prediction_type"] == "classification" + assert predictions["parent_id"] == "img" + assert predictions["root_parent_id"] == "img" + + +def test_run_locally_without_map_outputs_none() -> None: + # given + provider = _StubProvider([_response()]) + block = _block(provider, StepExecutionMode.LOCAL) + + # when + result = _run(block, [_image()], include_anomaly_map=False) + + # then + assert provider.calls[0]["include_anomaly_map"] is False + assert result[0]["anomaly_map"] is None + + +def test_run_locally_applies_the_threshold_override() -> None: + # given: the model says normal (1.5 < 2.0) + provider = _StubProvider([_response(score=1.5, threshold=2.0)]) + block = _block(provider, StepExecutionMode.LOCAL) + + # when + result = _run(block, [_image()], anomaly_threshold=1.0) + + # then: relative to 1.0 the anomalous confidence is sigmoid(1) ~ 0.7311 + assert result[0]["is_anomalous"] is True + assert result[0]["anomaly_threshold"] == 1.0 + assert result[0]["anomaly_score"] == 1.5 + assert result[0]["predictions"]["top"] == "anomalous" + assert result[0]["predictions"]["confidence"] == 0.7311 + assert result[0]["predictions"]["anomaly_threshold"] == 1.0 + assert result[0]["predictions"]["is_anomalous"] is True + + +def test_run_locally_sends_confidence_and_applies_it_under_an_override() -> None: + # given + provider = _StubProvider([_response(score=1.5, threshold=2.0)]) + block = _block(provider, StepExecutionMode.LOCAL) + + # when: the anomalous confidence relative to 1.0 (~0.73) misses 0.8 + result = _run(block, [_image()], confidence=0.8, anomaly_threshold=1.0) + + # then + assert provider.calls[0]["confidence"] == 0.8 + assert result[0]["is_anomalous"] is False + assert result[0]["predictions"]["top"] == "normal" + # 1.0 + 0.5 * logit(0.8): the score the override needs for an anomalous verdict. + assert result[0]["anomaly_decision_threshold"] == pytest.approx(1.6931, abs=1e-4) + + +def test_run_rejects_a_plain_classifier_response() -> None: + # given + plain = { + "predictions": [{"class": "cat", "class_id": 0, "confidence": 0.9}], + "top": "cat", + "confidence": 0.9, + } + block = _block(_StubProvider([plain]), StepExecutionMode.LOCAL) + + # when / then + with pytest.raises(ValueError, match="ws/anomaly/1"): + _run(block, [_image()]) + + +@pytest.mark.parametrize("invalid", [float("nan"), "0.5"]) +def test_run_rejects_an_invalid_threshold_override(invalid: Any) -> None: + block = _block(_StubProvider([_response()]), StepExecutionMode.LOCAL) + with pytest.raises(ValueError): + _run(block, [_image()], anomaly_threshold=invalid) + + +def test_run_locally_handles_a_batch_in_order() -> None: + # given + provider = _StubProvider([_response(score=5.0), _response(score=0.5)]) + block = _block(provider, StepExecutionMode.LOCAL) + + # when + result = _run(block, [_image("a"), _image("b")]) + + # then + assert [r["is_anomalous"] for r in result] == [True, False] + assert [r["predictions"]["parent_id"] for r in result] == ["a", "b"] + + +@pytest.mark.parametrize( + "served_map", [_map_payload(), MAP.copy()], ids=["payload", "ndarray"] +) +def test_run_remotely_configures_the_client_and_accepts_both_map_forms( + served_map, +) -> None: + # given + block = _block(MagicMock(), StepExecutionMode.REMOTE) + with patch.object(v1, "InferenceHTTPClient") as client_cls: + client = client_cls.return_value + client.infer.return_value = [_response(anomaly_map=served_map)] + + # when + result = _run(block, [_image()], anomaly_threshold=2.5) + + # then + configuration = client.configure.call_args.kwargs["inference_configuration"] + assert configuration.include_anomaly_map is True + assert configuration.confidence_threshold == 0.5 + assert configuration.disable_active_learning is True + assert configuration.source == "workflow-execution" + client.infer.assert_called_once() + assert client.infer.call_args.kwargs["model_id"] == "ws/anomaly/1" + np.testing.assert_array_equal(result[0]["anomaly_map"], MAP) + assert result[0]["anomaly_threshold"] == 2.5 + assert result[0]["is_anomalous"] is True + assert "anomaly_map" not in result[0]["predictions"] + + +def test_run_remotely_wraps_a_single_response() -> None: + # given + block = _block(MagicMock(), StepExecutionMode.REMOTE) + with patch.object(v1, "InferenceHTTPClient") as client_cls: + client_cls.return_value.infer.return_value = _response() + + # when + result = _run(block, [_image()], include_anomaly_map=False) + + # then + assert len(result) == 1 + assert result[0]["anomaly_map"] is None + configuration = client_cls.return_value.configure.call_args.kwargs[ + "inference_configuration" + ] + assert configuration.include_anomaly_map is False diff --git a/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_v1_tensor.py b/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_v1_tensor.py new file mode 100644 index 0000000000..a6c4ab503b --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/models/roboflow/anomaly_detection/test_v1_tensor.py @@ -0,0 +1,266 @@ +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +import torch +from roboflow_workflows.core_steps.common.entities import StepExecutionMode +from roboflow_workflows.core_steps.common.serializers_tensor import ( + serialise_native_classification, +) +from roboflow_workflows.core_steps.models.roboflow.anomaly_detection import v1_tensor +from roboflow_workflows.core_steps.models.roboflow.anomaly_detection.v1 import ( + BlockManifest as NumpyManifest, +) +from roboflow_workflows.core_steps.models.roboflow.anomaly_detection.v1_tensor import ( + AnomalyDetectionModelBlockV1, + BlockManifest, +) +from roboflow_workflows.execution_engine.entities.base import ( + Batch, + ImageParentMetadata, + WorkflowImageData, +) + +from inference_models.models.base.classification import ClassificationPrediction + +MAP = np.array([[0.0, 1.0], [2.0, 4.0]], dtype=np.float32) + + +def _image(parent_id: str = "img") -> WorkflowImageData: + return WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id=parent_id), + numpy_image=np.zeros((3, 4, 3), dtype=np.uint8), + ) + + +def _batched(scores: List[float], threshold: float = 2.0, with_map: bool = True): + metadata = [] + for score in scores: + entry: Dict[str, Any] = { + "anomaly_score": score, + "anomaly_threshold": threshold, + "anomaly_scale": 0.5, + "is_anomalous": score >= threshold, + } + if with_map: + entry["anomaly_map"] = MAP.copy() + metadata.append(entry) + anomalous = torch.tensor([[0.75 if s >= threshold else 0.25] for s in scores]) + return ClassificationPrediction( + class_id=torch.tensor([int(s >= threshold) for s in scores]), + confidence=torch.cat([1 - anomalous, anomalous], dim=1).float(), + images_metadata=metadata, + ) + + +class _StubProvider: + def __init__(self, prediction: ClassificationPrediction): + self.prediction = prediction + self.add_model = MagicMock() + self.calls: List[Dict[str, Any]] = [] + + def run_tensor_native_inference(self, model_id: str, **kwargs: Any): + self.calls.append({"model_id": model_id, **kwargs}) + return self.prediction + + def get_class_names(self, model_id: str) -> List[str]: + return ["normal", "anomalous"] + + +def _block(provider: Any, mode: StepExecutionMode) -> AnomalyDetectionModelBlockV1: + return AnomalyDetectionModelBlockV1( + model_manager=provider, api_key="key", step_execution_mode=mode + ) + + +def _run(block, images: List[WorkflowImageData], **overrides): + kwargs = dict( + model_id="ws/anomaly/1", + include_anomaly_map=True, + confidence=0.5, + anomaly_threshold=None, + disable_active_learning=True, + active_learning_target_dataset=None, + ) + kwargs.update(overrides) + return block.run( + images=Batch.init(content=images, indices=[(i,) for i in range(len(images))]), + **kwargs, + ) + + +def _remote_response(score: float = 3.0, threshold: float = 2.0) -> dict: + is_anomalous = score >= threshold + anomalous_confidence = 0.75 if is_anomalous else 0.25 + classes = [ + {"class": "anomalous", "class_id": 1, "confidence": anomalous_confidence}, + {"class": "normal", "class_id": 0, "confidence": 1 - anomalous_confidence}, + ] + if not is_anomalous: + classes.reverse() + return { + "inference_id": "iid-1", + "time": 0.01, + "image": {"width": 4, "height": 3}, + "predictions": classes, + "top": classes[0]["class"], + "confidence": classes[0]["confidence"], + "anomaly_score": score, + "anomaly_threshold": threshold, + "anomaly_scale": 0.5, + "is_anomalous": is_anomalous, + "anomaly_map": MAP.copy(), + } + + +def test_sibling_manifests_declare_the_same_surface() -> None: + assert set(BlockManifest.model_fields) == set(NumpyManifest.model_fields) + numpy_outputs = [ + (o.name, [k.name for k in o.kind]) for o in NumpyManifest.describe_outputs() + ] + tensor_outputs = [ + (o.name, [k.name for k in o.kind]) for o in BlockManifest.describe_outputs() + ] + assert numpy_outputs == tensor_outputs + + +def test_run_locally_fans_out_the_batched_prediction() -> None: + # given + provider = _StubProvider(_batched([3.0, 0.5])) + block = _block(provider, StepExecutionMode.LOCAL) + + # when + result = _run(block, [_image("a"), _image("b")]) + + # then + provider.add_model.assert_called_once_with(model_id="ws/anomaly/1", api_key="key") + assert provider.calls[0]["include_anomaly_map"] is True + assert provider.calls[0]["input_color_format"] == "bgr" + assert [r["is_anomalous"] for r in result] == [True, False] + assert [r["anomaly_score"] for r in result] == [3.0, 0.5] + assert [r["anomaly_threshold"] for r in result] == [2.0, 2.0] + for entry in result: + np.testing.assert_array_equal(entry["anomaly_map"], MAP) + prediction = entry["predictions"] + assert isinstance(prediction, ClassificationPrediction) + assert prediction.class_id.shape == (1,) + assert prediction.confidence.shape == (1, 2) + metadata = prediction.images_metadata[0] + assert metadata["class_names"] == {0: "normal", 1: "anomalous"} + assert "anomaly_map" not in metadata + assert metadata["inference_id"] == entry["inference_id"] + assert result[0]["predictions"].class_id.tolist() == [1] + assert result[1]["predictions"].class_id.tolist() == [0] + assert result[0]["predictions"].images_metadata[0]["parent_id"] == "a" + assert result[1]["predictions"].images_metadata[0]["parent_id"] == "b" + + +def test_run_locally_without_map_and_with_override() -> None: + # given: the model says normal at 2.0; the override at 1.0 flips it + provider = _StubProvider(_batched([1.5], with_map=False)) + block = _block(provider, StepExecutionMode.LOCAL) + + # when + result = _run(block, [_image()], include_anomaly_map=False, anomaly_threshold=1.0) + + # then + assert provider.calls[0]["include_anomaly_map"] is False + assert provider.calls[0]["confidence"] == 0.5 + assert result[0]["anomaly_map"] is None + assert result[0]["is_anomalous"] is True + assert result[0]["anomaly_threshold"] == 1.0 + assert result[0]["predictions"].class_id.tolist() == [1] + # Both confidences are recomputed relative to the override: sigmoid(1) ~ 0.7311. + assert result[0]["predictions"].confidence[0].tolist() == pytest.approx( + [0.2689, 0.7311], abs=1e-4 + ) + assert result[0]["predictions"].images_metadata[0]["is_anomalous"] is True + + +def test_run_locally_applies_confidence_under_an_override() -> None: + # given + provider = _StubProvider(_batched([1.5], with_map=False)) + block = _block(provider, StepExecutionMode.LOCAL) + + # when: the anomalous confidence relative to 1.0 (~0.73) misses 0.8 + result = _run(block, [_image()], confidence=0.8, anomaly_threshold=1.0) + + # then + assert provider.calls[0]["confidence"] == 0.8 + assert result[0]["is_anomalous"] is False + assert result[0]["predictions"].class_id.tolist() == [0] + assert result[0]["anomaly_decision_threshold"] == pytest.approx(1.6931, abs=1e-4) + + +def test_run_locally_rejects_a_plain_classifier() -> None: + # given: a classifier returns no per-image anomaly metadata + plain = ClassificationPrediction( + class_id=torch.tensor([0]), confidence=torch.tensor([[0.9, 0.1]]) + ) + block = _block(_StubProvider(plain), StepExecutionMode.LOCAL) + + # when / then + with pytest.raises(ValueError, match="ws/anomaly/1"): + _run(block, [_image()]) + + +def test_serialised_local_prediction_matches_the_numpy_dict_shape() -> None: + # given + provider = _StubProvider(_batched([1.5])) + block = _block(provider, StepExecutionMode.LOCAL) + result = _run(block, [_image()], anomaly_threshold=1.0) + + # when + serialised = serialise_native_classification(result[0]["predictions"]) + + # then: the override decided "anomalous" with confidences relative to it + assert list(serialised.keys()) == [ + "inference_id", + "time", + "image", + "predictions", + "top", + "confidence", + "anomaly_score", + "anomaly_threshold", + "anomaly_scale", + "is_anomalous", + "prediction_type", + "parent_id", + "root_parent_id", + ] + assert serialised["image"] == {"width": 4, "height": 3} + assert [p["class"] for p in serialised["predictions"]] == ["anomalous", "normal"] + assert serialised["top"] == "anomalous" + assert serialised["confidence"] == pytest.approx(0.7311, abs=1e-4) + assert serialised["anomaly_score"] == 1.5 + assert serialised["anomaly_threshold"] == 1.0 + assert serialised["is_anomalous"] is True + assert "anomaly_map" not in serialised + + +def test_run_remotely_rebuilds_a_native_prediction() -> None: + # given + block = _block(MagicMock(), StepExecutionMode.REMOTE) + with patch.object(v1_tensor, "InferenceHTTPClient") as client_cls: + client = client_cls.return_value + client.infer.return_value = [_remote_response()] + + # when + result = _run(block, [_image()]) + + # then + configuration = client.configure.call_args.kwargs["inference_configuration"] + assert configuration.include_anomaly_map is True + assert configuration.confidence_threshold == 0.5 + prediction = result[0]["predictions"] + assert isinstance(prediction, ClassificationPrediction) + assert prediction.class_id.tolist() == [1] + assert prediction.confidence.tolist() == [[0.25, 0.75]] + assert prediction.images_metadata[0]["class_names"] == {0: "normal", 1: "anomalous"} + assert prediction.images_metadata[0]["inference_id"] == "iid-1" + np.testing.assert_array_equal(result[0]["anomaly_map"], MAP) + serialised = serialise_native_classification(prediction) + assert serialised["top"] == "anomalous" + assert serialised["anomaly_score"] == 3.0 diff --git a/workflows/tests/unit_tests/core_steps/visualizations/test_anomaly_heatmap.py b/workflows/tests/unit_tests/core_steps/visualizations/test_anomaly_heatmap.py new file mode 100644 index 0000000000..9df1d6d9c6 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/visualizations/test_anomaly_heatmap.py @@ -0,0 +1,309 @@ +import numpy as np +import pytest +import torch +from pydantic import ValidationError +from roboflow_workflows.core_steps.visualizations.anomaly_heatmap.v1 import ( + AnomalyHeatmapManifest, + AnomalyHeatmapVisualizationBlockV1, +) +from roboflow_workflows.core_steps.visualizations.anomaly_heatmap.v1_tensor import ( + AnomalyHeatmapManifest as TensorManifest, +) +from roboflow_workflows.core_steps.visualizations.anomaly_heatmap.v1_tensor import ( + AnomalyHeatmapVisualizationBlockV1 as TensorBlock, +) +from roboflow_workflows.core_steps.visualizations.common.annotators.anomaly_heatmap import ( + AnomalyHeatmapAnnotator, + colorize_anomaly_map, + normalize_anomaly_map, +) +from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, +) + +TYPE = "roboflow_core/anomaly_heatmap_visualization@v1" + + +@pytest.mark.parametrize("manifest_class", [AnomalyHeatmapManifest, TensorManifest]) +@pytest.mark.parametrize("images_field_alias", ["images", "image"]) +def test_manifest_validation_when_valid_manifest_is_given( + manifest_class, images_field_alias: str +) -> None: + # given + data = { + "type": TYPE, + "name": "heatmap", + images_field_alias: "$inputs.image", + "anomaly_map": "$steps.anomaly.anomaly_map", + "anomaly_threshold": "$steps.anomaly.anomaly_threshold", + "color_map": "JET", + "opacity": 0.5, + } + + # when + result = manifest_class.model_validate(data) + + # then + assert result == manifest_class( + type=TYPE, + name="heatmap", + images="$inputs.image", + anomaly_map="$steps.anomaly.anomaly_map", + anomaly_threshold="$steps.anomaly.anomaly_threshold", + color_map="JET", + opacity=0.5, + ) + assert result.copy_image is True + + +def test_manifest_defaults_and_literal_threshold() -> None: + result = AnomalyHeatmapManifest.model_validate( + { + "type": TYPE, + "name": "heatmap", + "image": "$inputs.image", + "anomaly_map": "$steps.anomaly.anomaly_map", + "anomaly_threshold": 1.5, + } + ) + assert result.color_map == "ROBOFLOW" + assert result.opacity == 0.75 + assert result.anomaly_threshold == 1.5 + + +def test_sibling_manifests_declare_the_same_surface() -> None: + assert set(AnomalyHeatmapManifest.model_fields) == set(TensorManifest.model_fields) + assert [o.name for o in AnomalyHeatmapManifest.describe_outputs()] == [ + o.name for o in TensorManifest.describe_outputs() + ] + + +@pytest.mark.parametrize( + "field, value", + [ + ("anomaly_map", "not a selector"), + ("anomaly_threshold", "high"), + ("color_map", "RAINBOW"), + ("opacity", 1.5), + ], +) +def test_manifest_validation_when_field_is_invalid(field: str, value) -> None: + data = { + "type": TYPE, + "name": "heatmap", + "image": "$inputs.image", + "anomaly_map": "$steps.anomaly.anomaly_map", + "anomaly_threshold": 1.5, + } + data[field] = value + with pytest.raises(ValidationError): + AnomalyHeatmapManifest.model_validate(data) + + +@pytest.mark.parametrize("field", ["anomaly_map", "anomaly_threshold"]) +def test_manifest_validation_when_required_field_is_missing(field: str) -> None: + data = { + "type": TYPE, + "name": "heatmap", + "image": "$inputs.image", + "anomaly_map": "$steps.anomaly.anomaly_map", + "anomaly_threshold": 1.5, + } + del data[field] + with pytest.raises(ValidationError): + AnomalyHeatmapManifest.model_validate(data) + + +def test_normalize_anomaly_map_anchors_the_threshold_at_half() -> None: + # given + values = np.array([[-1.0, 0.0], [2.0, 6.0]], dtype=np.float32) + + # when + normalized = normalize_anomaly_map(values, threshold=2.0) + + # then + np.testing.assert_allclose(normalized, [[0.0, 0.0], [0.5, 0.75]]) + with pytest.raises(ValueError): + normalize_anomaly_map(values, threshold=float("nan")) + + +@pytest.mark.parametrize("threshold", [0.0, -0.5]) +def test_normalize_anomaly_map_saturates_past_a_non_positive_threshold( + threshold: float, +) -> None: + # given: a decision threshold at a very low confidence can reach zero + values = np.array([[-1.0, 0.0], [1e-3, 6.0]], dtype=np.float32) + + # when + normalized = normalize_anomaly_map(values, threshold=threshold) + + # then: every positive value is past the decision; zero stays transparent + np.testing.assert_array_equal(normalized, [[0.0, 0.0], [1.0, 1.0]]) + + +def test_roboflow_ramp_hits_its_stops() -> None: + # given + normalized = np.array([[0.0, 0.25, 0.5, 1.0]], dtype=np.float32) + + # when + bgr, alpha = colorize_anomaly_map(normalized, color_map="ROBOFLOW") + + # then: stops are (255,214,0,a0), (255,214,0,.55), (255,122,0,.85), (220,30,30,1) + assert bgr.tolist() == [ + [[0, 214, 255], [0, 214, 255], [0, 122, 255], [30, 30, 220]] + ] + np.testing.assert_allclose(alpha, [[0.0, 0.55, 0.85, 1.0]]) + + +def test_opencv_ramp_is_opaque() -> None: + bgr, alpha = colorize_anomaly_map(np.full((2, 2), 0.5, dtype=np.float32), "JET") + assert bgr.shape == (2, 2, 3) and bgr.dtype == np.uint8 + np.testing.assert_array_equal(alpha, np.ones((2, 2), dtype=np.float32)) + with pytest.raises(ValueError): + colorize_anomaly_map(np.zeros((2, 2), dtype=np.float32), "RAINBOW") + + +def _scene(height: int = 8, width: int = 8) -> np.ndarray: + return np.full((height, width, 3), 100, dtype=np.uint8) + + +def test_annotator_blends_the_threshold_colour_at_the_ramp_midpoint() -> None: + # given: a map at exactly the threshold everywhere, stretched 2x2 -> 8x8 + annotator = AnomalyHeatmapAnnotator(color_map="ROBOFLOW", opacity=1.0) + scene = _scene() + + # when + annotated = annotator.annotate(scene, np.full((2, 2), 2.0, dtype=np.float32), 2.0) + + # then: 100*(1-0.85) + (0,122,255)*0.85 per BGR channel + assert annotated is scene + assert annotated.shape == (8, 8, 3) + expected = np.rint(100 * 0.15 + np.array([0, 122, 255]) * 0.85).astype(np.uint8) + assert annotated[4, 4].tolist() == expected.tolist() + + +def test_annotator_leaves_zero_evidence_untouched_and_scales_by_opacity() -> None: + # given + annotator = AnomalyHeatmapAnnotator(color_map="ROBOFLOW", opacity=0.5) + scene = _scene() + anomaly_map = np.zeros((2, 2), dtype=np.float32) + anomaly_map[1, 1] = 1e9 # saturates the ramp + + # when + annotated = annotator.annotate(scene, anomaly_map, 2.0) + + # then + assert annotated[0, 0].tolist() == [100, 100, 100] + expected = np.rint(100 * 0.5 + np.array([30, 30, 220]) * 0.5).astype(np.uint8) + assert annotated[7, 7].tolist() == expected.tolist() + + +def test_annotator_promotes_grayscale_scenes_and_rejects_bad_maps() -> None: + annotator = AnomalyHeatmapAnnotator(color_map="HOT", opacity=0.5) + annotated = annotator.annotate( + np.zeros((4, 4), dtype=np.uint8), np.ones((2, 2), dtype=np.float32), 1.0 + ) + assert annotated.shape == (4, 4, 3) + with pytest.raises(ValueError): + annotator.annotate(_scene(), np.ones((2, 2, 2), dtype=np.float32), 1.0) + with pytest.raises(ValueError): + AnomalyHeatmapAnnotator(color_map="RAINBOW", opacity=0.5) + + +def _image(scene: np.ndarray) -> WorkflowImageData: + return WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="img"), numpy_image=scene + ) + + +@pytest.mark.parametrize( + "block_class", [AnomalyHeatmapVisualizationBlockV1, TensorBlock] +) +def test_block_copies_the_input_by_default(block_class) -> None: + # given + block = block_class() + scene = _scene() + image = _image(scene) + + # when + result = block.run( + image=image, + copy_image=True, + anomaly_map=np.full((2, 2), 2.0, dtype=np.float32), + anomaly_threshold=2.0, + color_map="ROBOFLOW", + opacity=1.0, + ) + + # then + output = result["image"] + assert output.numpy_image.shape == (8, 8, 3) + assert output.parent_metadata.parent_id == "img" + assert scene[4, 4].tolist() == [100, 100, 100] + assert output.numpy_image[4, 4].tolist() != [100, 100, 100] + assert output.numpy_image is not scene + + +@pytest.mark.parametrize( + "block_class", [AnomalyHeatmapVisualizationBlockV1, TensorBlock] +) +def test_block_mutates_the_input_when_asked(block_class) -> None: + # given + block = block_class() + scene = _scene() + + # when + result = block.run( + image=_image(scene), + copy_image=False, + anomaly_map=np.full((2, 2), 2.0, dtype=np.float32), + anomaly_threshold=2.0, + color_map="JET", + opacity=0.75, + ) + + # then + assert result["image"].numpy_image is scene + assert scene[4, 4].tolist() != [100, 100, 100] + + +@pytest.mark.parametrize( + "block_class", [AnomalyHeatmapVisualizationBlockV1, TensorBlock] +) +def test_block_requires_a_map(block_class) -> None: + with pytest.raises(ValueError, match="include_anomaly_map"): + block_class().run( + image=_image(_scene()), + copy_image=True, + anomaly_map=None, + anomaly_threshold=2.0, + color_map="ROBOFLOW", + opacity=1.0, + ) + + +def test_tensor_block_accepts_a_torch_map() -> None: + # given + scene = _scene() + + # when + result = TensorBlock().run( + image=_image(scene), + copy_image=True, + anomaly_map=torch.full((2, 2), 2.0), + anomaly_threshold=2.0, + color_map="ROBOFLOW", + opacity=1.0, + ) + + # then + expected = np.rint(100 * 0.15 + np.array([0, 122, 255]) * 0.85).astype(np.uint8) + assert result["image"].numpy_image[4, 4].tolist() == expected.tolist() + + +def test_annotator_is_cached_per_style() -> None: + block = AnomalyHeatmapVisualizationBlockV1() + first = block.getAnnotator("ROBOFLOW", 0.5) + assert block.getAnnotator("ROBOFLOW", 0.5) is first + assert block.getAnnotator("JET", 0.5) is not first diff --git a/workflows/uv.lock b/workflows/uv.lock index 0e20ef4d59..f65c0350ee 100644 --- a/workflows/uv.lock +++ b/workflows/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.10, <3.14" resolution-markers = [ "python_full_version >= '3.13' and sys_platform == 'darwin'", @@ -1577,7 +1577,7 @@ wheels = [ [[package]] name = "inference-sdk" -version = "1.7.2" +version = "1.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1594,7 +1594,7 @@ dependencies = [ { name = "urllib3" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/7b/726fc47d541b633c41d7ded6dbad206aee1ec67d4a1df0e6875816d6fcc3/inference_sdk-1.7.2-py3-none-any.whl", hash = "sha256:94711e6e756e26a7c5e62eb58e2110926721df7691ce926d0e219b8352370cb3", size = 105201, upload-time = "2026-09-25T19:24:47.147Z" }, + { url = "https://files.pythonhosted.org/packages/4c/00/6fafb460ef153c2d641e35040d4334264a0f8c86d5ce63219eac319e19b3/inference_sdk-1.8.1-py3-none-any.whl", hash = "sha256:5c7293f174532b3523bafa59aa696b909f315b9bfaf6436300acb58e6986d8ae", size = 108404, upload-time = "2026-10-19T15:57:59.291Z" }, ] [[package]] @@ -2945,6 +2945,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] +[[package]] +name = "pkgconfig" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/fd/0adde075cd3bfecd557bc7d757e00e231d34d8a6edb4c8d1642759254c21/pkgconfig-1.6.0.tar.gz", hash = "sha256:4a5a6631ce937fafac457104a40d558785a658bbdca5c49b6295bc3fd651907f", size = 5691, upload-time = "2026-03-06T11:26:01.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/6f/f7ec07fba48f07c555cc4099481df644fbbc12067879072c17ac229f6556/pkgconfig-1.6.0-py3-none-any.whl", hash = "sha256:98e71754855e9563838d952a160eb577edabb57782e49853edb5381927e6bea1", size = 7086, upload-time = "2026-03-06T11:26:07.688Z" }, +] + [[package]] name = "platformdirs" version = "4.12.1" @@ -3669,6 +3678,7 @@ version = "2.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, + { name = "pkgconfig" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7b/88/f73dae807ec68b228fba72507105e3ba80a561dc0bade0004ce24fd118fc/pyvips-2.2.3.tar.gz", hash = "sha256:43bceced0db492654c93008246a58a508e0373ae1621116b87b322f2ac72212f", size = 56626, upload-time = "2024-04-28T11:19:58.158Z" } @@ -4076,7 +4086,7 @@ wheels = [ [[package]] name = "roboflow-workflows" -version = "0.2.11rc1" +version = "0.2.11" source = { editable = "." } dependencies = [ { name = "anthropic" }, @@ -4151,7 +4161,7 @@ requires-dist = [ { name = "filelock", specifier = ">=3.20.3,<=4.0.0" }, { name = "httpx", specifier = "~=0.28.1" }, { name = "inference-models", specifier = ">=0.39.5rc1" }, - { name = "inference-sdk", specifier = ">=1.7.0" }, + { name = "inference-sdk", specifier = ">=1.8.0" }, { name = "matplotlib" }, { name = "msgpack" }, { name = "networkx", specifier = ">=3.0.0,<4.0.0" },